def test_sync_volumes(refresh, emit_event, compare, save, load, blivet):
init()
refresh.return_value = True
compare.return_value = True
save.return_value = True
emit_event.return_value = True
NS.node_context = node_context.NodeContext
obj = node_context.NodeContext(
node_id="eeebe9f5-6e43-4cf8-b321-37d648eb0510",
fqdn="dhcp12-12.lab.abc.com",
tags=["provisioner/%s" % NS.tendrl_context.integration_id],
ipv4_addr="127.0.0.1")
NS.node_context = obj
load.return_value = obj
sds_sync = importlib.import_module('tendrl.gluster_integration.sds_sync')
raw_data = ini2json.ini_to_dict(
os.path.join(os.path.dirname(__file__), "gluster-state.yaml"))
raw_data_options = ini2json.ini_to_dict(
os.path.join(os.path.dirname(__file__), "gluster-volume-option.yaml"))
sds_sync.Event = MagicMock()
sds_sync.Message = MagicMock()
with patch.object(NS._int.client, "read", read):
with patch.object(brick_device_details,
"update_brick_device_details") as update:
update.return_value = True
with patch.object(brick_utilization,
"brick_utilization") as utilization:
utilization.return_value = 10
sds_sync.sync_volumes(raw_data['Volumes'], 1,
raw_data_options.get('Volume Options'),
10)
def test_parse_continuation_line(self):
body = """[foo]\nbar:1 2 jammy:jam\n foo bar\n boo"""
filename = self._makeFile('pytest', body)
sections = ini2json.ini_to_dict(filename)
assert sections == (
{'foo': {'bar': ['1', '2', 'jammy:jam', 'foo', 'bar', 'boo']}}
)
def test_snapshot_restored(read, call, sleep, load):
load.return_value = maps.NamedDict(
short_name="77deef29-b8e5-4dc5-8247-21e2a409a66a")
sleep.return_value = True
call.return_value = True
keys = maps.NamedDict(key="sub_vol1/dhcp123-12.lab.abc.com:|gluster|b1")
read.return_value = maps.NamedDict(leaves=[keys], value="/gluster/b1")
event = {
"message": {
"volume_name": "GlusterVolume"
},
"event": "SNAPSHOT_RESTORED",
"ts": 1486634392,
"nodeid": "0f5b4b99-fa4a-4b8f-be52-770b42879d67"
}
raw_data = ini2json.ini_to_dict(
os.path.join(os.path.dirname(__file__), "gluster_state.yaml"))
with patch.object(ini2json, "ini_to_dict") as ini_to_dict:
ini_to_dict.return_value = raw_data
obj = Callback()
with patch.object(obj, "volume_remove_brick_force") as rm_brick:
obj.snapshot_restored(event)
rm_brick.assert_called_with({
'event':
'SNAPSHOT_RESTORED',
'ts':
1486634392,
'message': {
'volume': 'GlusterVolume',
'bricks': '/gluster/b1'
},
'nodeid':
'0f5b4b99-fa4a-4b8f-be52-770b42879d67'
})
def test_snapshot_restored(read, call, sleep, load):
load.return_value = maps.NamedDict(
short_name="77deef29-b8e5-4dc5-8247-21e2a409a66a"
)
sleep.return_value = True
call.return_value = True
keys = maps.NamedDict(
key="sub_vol1/dhcp123-12.lab.abc.com:|gluster|b1"
)
read.return_value = maps.NamedDict(leaves=[keys], value="/gluster/b1")
event = {"message": {"volume_name": "GlusterVolume"},
"event": "SNAPSHOT_RESTORED",
"ts": 1486634392,
"nodeid": "0f5b4b99-fa4a-4b8f-be52-770b42879d67"
}
raw_data = ini2json.ini_to_dict(
os.path.join(os.path.dirname(__file__),
"gluster_state.yaml")
)
with patch.object(ini2json, "ini_to_dict") as ini_to_dict:
ini_to_dict.return_value = raw_data
obj = Callback()
with patch.object(obj, "volume_remove_brick_force") as rm_brick:
obj.snapshot_restored(event)
rm_brick.assert_called_with(
{'event': 'SNAPSHOT_RESTORED',
'ts': 1486634392,
'message': {'volume': 'GlusterVolume',
'bricks': '/gluster/b1'
},
'nodeid': '0f5b4b99-fa4a-4b8f-be52-770b42879d67'
}
)
def test_parse_with_bad_format(self):
body = """[foo]\nbar: 1 ;\n 2 3 4 jammy:jam\n"""
filename = self._makeFile('pytest', body)
sections = ini2json.ini_to_dict(filename)
assert sections == (
{'foo': {'bar': ['1', '2', '3', '4', 'jammy:jam']}}
)
def test_parse_without_space(self):
body = """[foo]\nbar:""\n"""
filename = self._makeFile('pytest', body)
sections = ini2json.ini_to_dict(filename)
assert sections == (
{'foo': {'bar': ''}}
)
def test_parse_body_default(self):
body = """[foo]\nbar=1\nbaz=2"\n[DEFAULT]\na=1"""
filename = self._makeFile('pytest', body)
sections = ini2json.ini_to_dict(filename)
assert sections == (
{'foo': {'a': '1', 'bar': '1', 'baz': '2"'}}
)
def _run(self):
Event(
Message(
priority="info",
publisher=NS.publisher_id,
payload={"message": "%s running" % self.__class__.__name__}))
gluster_brick_dir = NS.gluster.objects.\
GlusterBrickDir()
gluster_brick_dir.save()
while not self._complete.is_set():
try:
# Acquire lock before updating the volume details in etcd
# We are blocking till we acquire the lock.
# the lock will live for 60 sec after which it will be released.
lock = etcd.Lock(NS._int.wclient, 'volume')
try:
lock.acquire(blocking=True, lock_ttl=60)
if lock.is_acquired:
# renewing the lock
lock.acquire(lock_ttl=60)
except etcd.EtcdLockExpired:
continue
gevent.sleep(10)
subprocess.call([
'gluster', 'get-state', 'glusterd', 'odir', '/var/run',
'file', 'glusterd-state'
])
raw_data = ini2json.ini_to_dict('/var/run/glusterd-state')
subprocess.call(['rm', '-rf', '/var/run/glusterd-state'])
sync_object = NS.gluster.objects.\
SyncObject(data=json.dumps(raw_data))
sync_object.save()
if "Peers" in raw_data:
index = 1
peers = raw_data["Peers"]
while True:
try:
peer = NS.gluster.\
objects.Peer(
peer_uuid=peers['peer%s.uuid' % index],
hostname=peers['peer%s.primary_hostname' % index],
state=peers['peer%s.state' % index]
)
peer.save()
index += 1
except KeyError:
break
if "Volumes" in raw_data:
index = 1
volumes = raw_data['Volumes']
node_context = NS.node_context.load()
tag_list = list(node_context.tags)
while True:
try:
# Raise alerts for volume state change.
cluster_provisioner = "provisioner/%s" % NS.tendrl_context.integration_id
if cluster_provisioner in tag_list:
try:
stored_volume_status = NS._int.client.read(
"clusters/%s/Volumes/%s/status" %
(NS.tendrl_context.integration_id,
volumes['volume%s.id' % index])).value
current_status = volumes['volume%s.status'
% index]
if current_status != stored_volume_status:
msg = "Status of volume: %s changed from %s to %s" % (
volumes['volume%s.name' % index],
stored_volume_status,
current_status)
instance = "volume_%s" % volumes[
'volume%s.name' % index]
self._emit_event(
"volume_status", current_status,
msg, instance)
except etcd.EtcdKeyNotFound:
pass
volume = NS.gluster.objects.Volume(
vol_id=volumes['volume%s.id' % index],
vol_type=volumes['volume%s.type' % index],
name=volumes['volume%s.name' % index],
transport_type=volumes[
'volume%s.transport_type' % index],
status=volumes['volume%s.status' % index],
brick_count=volumes['volume%s.brickcount' %
index],
snap_count=volumes['volume%s.snap_count' %
index],
stripe_count=volumes['volume%s.stripe_count' %
index],
replica_count=volumes['volume%s.replica_count'
% index],
subvol_count=volumes['volume%s.subvol_count' %
index],
arbiter_count=volumes['volume%s.arbiter_count'
% index],
disperse_count=volumes[
'volume%s.disperse_count' % index],
redundancy_count=volumes[
'volume%s.redundancy_count' % index],
quorum_status=volumes['volume%s.quorum_status'
% index],
snapd_status=volumes[
'volume%s.snapd_svc.online_status' %
index],
snapd_inited=volumes[
'volume%s.snapd_svc.inited' % index],
rebal_id=volumes['volume%s.rebalance.id' %
index],
rebal_status=volumes[
'volume%s.rebalance.status' % index],
rebal_failures=volumes[
'volume%s.rebalance.failures' % index],
rebal_skipped=volumes[
'volume%s.rebalance.skipped' % index],
rebal_lookedup=volumes[
'volume%s.rebalance.lookedup' % index],
rebal_files=volumes['volume%s.rebalance.files'
% index],
rebal_data=volumes['volume%s.rebalance.data' %
index],
)
volume.save()
b_index = 1
# ipv4 address of current node
try:
network_ip = []
networks = NS._int.client.read(
"nodes/%s/Networks" %
NS.node_context.node_id)
for interface in networks.leaves:
key = interface.key.split("/")[-1]
network = NS.tendrl.objects.NodeNetwork(
interface=key).load()
network_ip.extend(network.ipv4)
except etcd.EtcdKeyNotFound as ex:
Event(
ExceptionMessage(
priority="debug",
publisher=NS.publisher_id,
payload={
"message":
"Could not find any ipv4 networks for node %s"
% NS.node_context.node_id,
"exception":
ex
}))
while True:
try:
# Update brick node wise
hostname = volumes[
'volume%s.brick%s.hostname' %
(index, b_index)]
if (NS.node_context.fqdn != hostname) and (
hostname not in network_ip):
b_index += 1
continue
# Raise alerts if the brick path changes
try:
stored_brick_status = NS._int.client.read(
"clusters/%s/Volumes/%s/Bricks/%s/status"
%
(NS.tendrl_context.integration_id,
volumes['volume%s.id' % index],
volumes['volume%s.brick%s.path' %
(index, b_index)].replace(
"/", "_"))).value
current_status = volumes.get(
'volume%s.brick%s.status' %
(index, b_index))
if current_status != stored_brick_status:
msg = "Status of brick: %s under volume %s changed from %s to %s" % (
volumes['volume%s.brick%s.path'
% (index, b_index)],
volumes['volume%s.name' %
index],
stored_brick_status,
current_status)
instance = "volume_%s|brick_%s" % (
volumes['volume%s.name' %
index],
volumes['volume%s.brick%s.path'
% (index, b_index)])
self._emit_event(
"brick_status", current_status,
msg, instance)
except etcd.EtcdKeyNotFound:
pass
brick = NS.gluster\
.objects.Brick(
vol_id=volumes['volume%s.id' % index],
sequence_number=b_index,
path=volumes[
'volume%s.brick%s.path' % (
index, b_index)],
hostname=volumes.get(
'volume%s.brick%s.hostname' % (
index, b_index)),
port=volumes.get(
'volume%s.brick%s.port' % (
index, b_index)),
status=volumes.get(
'volume%s.brick%s.status' % (
index, b_index)),
filesystem_type=volumes.get(
'volume%s.brick%s.filesystem_type' % (
index, b_index)),
mount_opts=volumes.get(
'volume%s.brick%s.mount_options' % (
index, b_index)),
utilization=brick_utilization\
.brick_utilization(
volumes['volume%s.brick%s.path' % (
index, b_index)])
)
brick.save()
b_index += 1
except KeyError:
break
index += 1
except KeyError:
break
# poplate the volume options
reg_ex = re.compile("^volume[0-9]+.options+")
options = {}
for key in volumes.keys():
if reg_ex.match(key):
options[key] = volumes[key]
for key in options.keys():
volname = key.split('.')[0]
vol_id = volumes['%s.id' % volname]
dict1 = {}
for k, v in options.items():
if k.startswith('%s.options' % volname):
dict1['.'.join(k.split(".")[2:])] = v
options.pop(k, None)
vol_options = NS.gluster.objects.\
VolumeOptions(
vol_id=vol_id,
options=dict1
)
vol_options.save()
# Sync the cluster status details
args = ["gstatus", "-o", "json"]
p = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=open(os.devnull, "r"))
stdout, stderr = p.communicate()
if stderr == "" and p.returncode == 0:
# The format of the output from gstatus is as below
# "2017-02-23 18:13:01.944183 {"brick_count": 2,
# bricks_active": 2, "glfs_version": "3.9.0",
# node_count": 2, "nodes_active": 2, "over_commit": "No",
# product_name": "Community", "raw_capacity": 52181536768,
# sh_active": 0, "sh_enabled": 0, "snapshot_count": 0,
# status": "healthy", "usable_capacity": 52181536768,
# used_capacity": 2117836800, "volume_count": 1,
# volume_summary": [{"snapshot_count": 0, "state": "up",
# usable_capacity": 52181536768, "used_capacity": 2117836800,
# volume_name": "vol1"}]}\n"
out_dict = json.loads(stdout[stdout.index('{'):-1])
NS.gluster.objects.Utilization(
raw_capacity=out_dict['raw_capacity'],
usable_capacity=out_dict['usable_capacity'],
used_capacity=out_dict['used_capacity'],
pcnt_used=(out_dict['used_capacity'] * 100 /
out_dict['usable_capacity'])).save()
volume_up_degraded = 0
for item in out_dict['volume_summary']:
if "up(degraded)" in item['state']:
volume_up_degraded = volume_up_degraded + 1
volumes = NS._int.client.read(
"clusters/%s/Volumes" %
NS.tendrl_context.integration_id)
for child in volumes._children:
volume = NS.gluster.objects.Volume(
vol_id=child['key'].split('/')[-1]).load()
if volume.name == item['volume_name']:
NS.gluster.objects.Volume(
vol_id=child['key'].split('/')[-1],
usable_capacity=str(
item['usable_capacity']),
used_capacity=str(item['used_capacity']),
pcnt_used=str(item['used_capacity'] * 100 /
item['usable_capacity']),
vol_type=volume.vol_type,
name=volume.name,
transport_type=volume.transport_type,
status=volume.status,
brick_count=volume.brick_count,
snap_count=volume.snap_count,
stripe_count=volume.stripe_count,
replica_count=volume.replica_count,
subvol_count=volume.subvol_count,
arbiter_count=volume.arbiter_count,
disperse_count=volume.disperse_count,
redundancy_count=volume.redundancy_count,
quorum_status=volume.quorum_status,
snapd_status=volume.snapd_status,
snapd_inited=volume.snapd_inited,
rebal_id=volume.rebal_id,
rebal_status=volume.rebal_status,
rebal_failures=volume.rebal_failures,
rebal_skipped=volume.rebal_skipped,
rebal_lookedup=volume.rebal_lookedup,
rebal_files=volume.rebal_files,
rebal_data=volume.rebal_data).save()
connection_count = None
connection_active = None
# gstatus result:
# Product: Community Capacity: 25.00 GiB(raw bricks)
# Status: HEALTHY 3.00 GiB(raw used)
# Glusterfs: 3.9.0 87.00 GiB(usable from volumes)
# OverCommit: Yes Snapshots: 0
# Nodes : 2/ 2 Volumes: 4 Up
# Self Heal : 0/ 0 0 Up(Degraded)
# Bricks : 7/ 7 0 Up(Partial)
# Connections : 1/ 2 0 Down
# grep result:
# Connections : 0/ 0 1 Down
cmd = cmd_utils.Command('gstatus -s | grep Connections',
True)
out, err, rc = cmd.run()
if not err:
if "Connection" in out:
out = (re.split(r'\s{5,}', out)[0])
out = out.split(":")[1].replace(" ", "").split("/")
connection_active = int(out[0])
connection_count = int(out[1])
NS.gluster.objects.GlobalDetails(
status=out_dict['status'],
volume_up_degraded=volume_up_degraded,
connection_active=connection_active,
connection_count=connection_count).save()
else:
# If gstatus does not return any status details in absence
# of volumes, default set the status and utilization details
# A sample output from gstatus if no volumes in cluster is as below
#
# >gstatus -o json
# This cluster doesn't have any volumes/daemons running.
# The output below shows the current nodes attached to this host.
#
# UUID Hostname State
# 6a48d2f7-3859-4542-86e0-0a8146588f31 {FQDN-1} Connected
# 7380e303-83b6-4728-918f-e99029bc1bce {FQDN-2} Connected
# 751ecb42-da85-4c3d-834d-9824d1ce7fd3 {FQDN-3} Connected
# 388b708c-a86c-4a16-9a6f-f0d53ea79a51 localhost Connected
out_lines = stdout.split('\n')
connected = True
for index in range(4, len(out_lines) - 2):
node_status_det = out_lines[index].split('\t')
if len(node_status_det) > 2:
if node_status_det[2].strip() != 'Connected':
connected = connected and False
if connected:
NS.gluster.objects.GlobalDetails(
status='healthy').save()
else:
NS.gluster.objects.GlobalDetails(
status='unhealthy').save()
NS.gluster.objects.Utilization(raw_capacity=0,
usable_capacity=0,
used_capacity=0,
pcnt_used=0).save()
except Exception as ex:
Event(
ExceptionMessage(priority="error",
publisher=NS.publisher_id,
payload={
"message":
"gluster sds state sync error",
"exception": ex
}))
finally:
lock.release()
Event(
Message(
priority="debug",
publisher=NS.publisher_id,
payload={"message": "%s complete" % self.__class__.__name__}))
def run(self):
logger.log(
"info",
NS.publisher_id,
{"message": "%s running" % self.__class__.__name__}
)
gluster_brick_dir = NS.gluster.objects.GlusterBrickDir()
gluster_brick_dir.save()
cluster = NS.tendrl.objects.Cluster(
integration_id=NS.tendrl_context.integration_id
).load()
if cluster.cluster_network in [None, ""]:
try:
node_networks = NS.tendrl.objects.NodeNetwork().load_all()
cluster.cluster_network = node_networks[0].subnet
cluster.save()
except etcd.EtcdKeyNotFound as ex:
logger.log(
"error",
NS.publisher_id,
{"message": "Failed to sync cluster network details"}
)
_sleep = 0
while not self._complete.is_set():
# To detect out of band deletes
# refresh gluster object inventory at config['sync_interval']
SYNC_TTL = int(NS.config.data.get("sync_interval", 10)) + 100
NS.node_context = NS.node_context.load()
NS.tendrl_context = NS.tendrl_context.load()
if _sleep > 5:
_sleep = int(NS.config.data.get("sync_interval", 10))
else:
_sleep += 1
try:
_cluster = NS.tendrl.objects.Cluster(
integration_id=NS.tendrl_context.integration_id
).load()
if (_cluster.status == "importing" and (
_cluster.current_job['status'] == 'failed')) or \
_cluster.status == "unmanaging" or \
_cluster.status == "set_volume_profiling":
time.sleep(_sleep)
continue
_cnc = NS.tendrl.objects.ClusterNodeContext(
node_id=NS.node_context.node_id
).load()
_cnc.is_managed = "yes"
_cnc.save()
subprocess.call(
[
'gluster',
'get-state',
'glusterd',
'odir',
'/var/run',
'file',
'glusterd-state',
'detail'
]
)
raw_data = ini2json.ini_to_dict(
'/var/run/glusterd-state'
)
subprocess.call(['rm', '-rf', '/var/run/glusterd-state'])
subprocess.call(
[
'gluster',
'get-state',
'glusterd',
'odir',
'/var/run',
'file',
'glusterd-state-vol-opts',
'volumeoptions'
]
)
raw_data_options = ini2json.ini_to_dict(
'/var/run/glusterd-state-vol-opts'
)
subprocess.call(
[
'rm',
'-rf',
'/var/run/glusterd-state-vol-opts'
]
)
sync_object = NS.gluster.objects.\
SyncObject(data=json.dumps(raw_data))
sync_object.save()
if "Peers" in raw_data:
index = 1
peers = raw_data["Peers"]
disconnected_hosts = []
while True:
try:
peer = NS.tendrl.\
objects.GlusterPeer(
peer_uuid=peers['peer%s.uuid' % index],
hostname=peers[
'peer%s.primary_hostname' % index
],
state=peers['peer%s.state' % index],
connected=peers['peer%s.connected' % index]
)
try:
stored_peer_status = None
# find peer detail using hostname
ip = socket.gethostbyname(
peers['peer%s.primary_hostname' % index]
)
node_id = etcd_utils.read(
"/indexes/ip/%s" % ip
).value
stored_peer = NS.tendrl.objects.GlusterPeer(
peer_uuid=peers['peer%s.uuid' % index],
node_id=node_id
).load()
stored_peer_status = stored_peer.connected
current_status = peers[
'peer%s.connected' % index
]
if stored_peer_status and \
current_status != stored_peer_status:
msg = (
"Peer %s in cluster %s "
"is %s"
) % (
peers[
'peer%s.primary_hostname' %
index
],
_cluster.short_name,
current_status
)
instance = "peer_%s" % peers[
'peer%s.primary_hostname' % index
]
event_utils.emit_event(
"peer_status",
current_status,
msg,
instance,
'WARNING'
if current_status != 'Connected'
else 'INFO'
)
# save current status in actual peer
# directory also
stored_peer.connected = current_status
stored_peer.save()
# Disconnected host name to
# raise brick alert
if current_status.lower() == \
"disconnected":
disconnected_hosts.append(
peers[
'peer%s.primary_hostname' %
index
]
)
except etcd.EtcdKeyNotFound:
pass
SYNC_TTL += 5
peer.save(ttl=SYNC_TTL)
index += 1
except KeyError:
break
# Raise an alert for bricks when peer disconnected
# or node goes down
for disconnected_host in disconnected_hosts:
brick_status_alert(
disconnected_host
)
if "Volumes" in raw_data:
# create devicetree using lsblk
devicetree = get_device_tree()
# find lvs
lvs = brick_utilization.get_lvs()
index = 1
volumes = raw_data['Volumes']
total_brick_count = 0
while True:
try:
b_count = sync_volumes(
volumes, index,
raw_data_options.get('Volume Options'),
SYNC_TTL + VOLUME_TTL,
_cluster.short_name,
devicetree,
lvs
)
index += 1
SYNC_TTL += 1
total_brick_count += b_count - 1
except KeyError:
global VOLUME_TTL
# from second sync volume ttl is
# SYNC_TTL + (no.volumes) * 20 +
# (no.of.bricks) * 10 + 160
if index > 1:
volume_count = index - 1
# When all nodes are down we are updating all
# volumes are down, node status TTL is 160,
# So make sure volumes are present in etcd
# while raising volume down alert
VOLUME_TTL = (volume_count * 20) + (
total_brick_count * 10) + 160
break
# populate the volume specific options
reg_ex = re.compile("^volume[0-9]+.options+")
options = {}
for key in volumes.keys():
if reg_ex.match(key):
options[key] = volumes[key]
for key in options.keys():
volname = key.split('.')[0]
vol_id = volumes['%s.id' % volname]
dict1 = {}
for k, v in options.items():
if k.startswith('%s.options' % volname):
dict1['.'.join(k.split(".")[2:])] = v
options.pop(k, None)
volume = NS.tendrl.objects.GlusterVolume(
NS.tendrl_context.integration_id,
vol_id=vol_id
).load()
if volume.options is not None:
dest = dict(volume.options)
dest.update(dict1)
volume.options = dest
volume.save()
# Sync cluster global details
if "provisioner/%s" % NS.tendrl_context.integration_id \
in NS.node_context.tags:
all_volumes = NS.tendrl.objects.GlusterVolume(
NS.tendrl_context.integration_id
).load_all() or []
volumes = []
for volume in all_volumes:
if not str(volume.deleted).lower() == "true" and \
volume.current_job.get('status', '') \
in ['', 'finished', 'failed'] and \
volume.vol_id not in [None, ''] and \
volume.name not in [None, '']:
# only for first sync refresh volume TTL
# It will increase TTL based on no.of volumes
if _cnc.first_sync_done in [None, "no", ""]:
etcd_utils.refresh(
volume.value,
SYNC_TTL + VOLUME_TTL
)
volumes.append(volume)
cluster_status.sync_cluster_status(
volumes, SYNC_TTL + VOLUME_TTL
)
utilization.sync_utilization_details(volumes)
client_connections.sync_volume_connections(volumes)
georep_details.aggregate_session_status()
try:
evt.process_events()
except etcd.EtcdKeyNotFound:
pass
rebalance_status.sync_volume_rebalance_status(volumes)
rebalance_status.sync_volume_rebalance_estimated_time(
volumes
)
snapshots.sync_volume_snapshots(
raw_data['Volumes'],
int(NS.config.data.get(
"sync_interval", 10
)) + len(volumes) * 4
)
# update alert count
update_cluster_alert_count()
# check and enable volume profiling
if "provisioner/%s" % NS.tendrl_context.integration_id in \
NS.node_context.tags:
self._update_volume_profiling()
_cluster = NS.tendrl.objects.Cluster(
integration_id=NS.tendrl_context.integration_id
).load()
if _cluster.exists():
_cluster = _cluster.load()
_cluster.last_sync = str(tendrl_now())
# Mark the first sync done flag
_cnc = NS.tendrl.objects.ClusterNodeContext(
node_id=NS.node_context.node_id
).load()
if _cnc.first_sync_done in [None, "no"]:
_cnc.first_sync_done = "yes"
_cnc.save()
if _cluster.current_job.get(
'status', ''
) in ['', 'finished', 'failed'] and \
_cluster.status in [None, ""]:
_cluster.save()
except Exception as ex:
Event(
ExceptionMessage(
priority="error",
publisher=NS.publisher_id,
payload={"message": "gluster sds state sync error",
"exception": ex
}
)
)
try:
etcd_utils.read(
'/clusters/%s/_sync_now' %
NS.tendrl_context.integration_id
)
continue
except etcd.EtcdKeyNotFound:
pass
time.sleep(_sleep)
logger.log(
"debug",
NS.publisher_id,
{"message": "%s complete" % self.__class__.__name__}
)
def test_parse_missing_section(self):
body = """\nbar=1\nbaz=2"\n[DEFAULT]\na=1i"""
filename = self._makeFile('pytest', body)
with pytest.raises(ini2json.MissingSectionHeaderError):
ini2json.ini_to_dict(filename)
def snapshot_restored(self, event):
time.sleep(self.sync_interval)
message = event["message"]
volume = message['volume_name']
volume_id = ""
bricks_to_remove = []
# get the list of current bricks by running get-state
output_dir = '/var/run/'
output_file = 'glusterd-state-snapshot-%s' % str(uuid.uuid4())
subprocess.call([
'gluster', 'get-state', 'glusterd', 'odir', output_dir, 'file',
output_file, 'detail'
])
raw_data = ini2json.ini_to_dict(output_dir + output_file)
subprocess.call(['rm', '-rf', output_dir + output_file])
index = 1
while True:
try:
current_vol = 'volume%s.name' % index
if raw_data['Volumes'][current_vol] == volume:
current_vol_id = 'volume%s.id' % index
volume_id = raw_data['Volumes'][current_vol_id]
break
except KeyError:
return
index += 1
latest_bricks = []
b_index = 1
while True:
try:
curr_brick = 'volume%s.brick%s.path' % (index, b_index)
brick = raw_data['Volumes'][curr_brick]
b_index += 1
except KeyError:
break
latest_bricks.append(brick)
# get the list of bricks in etcd for this volume
sub_volumes = etcd_utils.read(
"/clusters/{0}/Volumes/{1}/Bricks".format(
NS.tendrl_context.integration_id, volume_id))
for sub_volume in sub_volumes.leaves:
bricks = etcd_utils.read(sub_volume.key)
for brick in bricks.leaves:
fqdn = brick.key.split('/')[-1].split(':')[0]
path = brick.key.split('/')[-1].split(':')[-1][1:]
brick_path = "clusters/{0}/Bricks/"\
"all/{1}/{2}".format(
NS.tendrl_context.integration_id,
fqdn,
path
)
brick_full_path = etcd_utils.read("%s/brick_path" %
brick_path).value
if brick_full_path not in latest_bricks:
bricks_to_remove.append(brick_full_path)
brick_details = {}
brick_details["volume"] = volume
brick_details["bricks"] = " ".join(bricks_to_remove)
event["message"] = brick_details
self.volume_remove_brick_force(event)
def Initialize(self, body):
self.Manager.subprocess = self
self.Manager.ini2json = self
filename = self._makeFile('manager', body)
self.sections = ini2json.ini_to_dict(filename)
def run(self):
# To detect out of band deletes
# refresh gluster object inventory at config['sync_interval']
# Default is 260 seconds
SYNC_TTL = int(NS.config.data.get("sync_interval", 10)) + 250
Event(
Message(
priority="info",
publisher=NS.publisher_id,
payload={"message": "%s running" % self.__class__.__name__}))
gluster_brick_dir = NS.gluster.objects.GlusterBrickDir()
gluster_brick_dir.save()
try:
etcd_utils.read("clusters/%s/"
"cluster_network" %
NS.tendrl_context.integration_id)
except etcd.EtcdKeyNotFound:
try:
node_networks = etcd_utils.read("nodes/%s/Networks" %
NS.node_context.node_id)
# TODO(team) this logic needs to change later
# multiple networks supported for gluster use case
node_network = NS.tendrl.objects.NodeNetwork(
interface=node_networks.leaves.next().key.split(
'/')[-1]).load()
cluster = NS.tendrl.objects.Cluster(
integration_id=NS.tendrl_context.integration_id).load()
cluster.cluster_network = node_network.subnet
cluster.save()
except etcd.EtcdKeyNotFound as ex:
Event(
Message(priority="error",
publisher=NS.publisher_id,
payload={
"message":
"Failed to sync cluster network details"
}))
_sleep = 0
while not self._complete.is_set():
if _sleep > 5:
_sleep = int(NS.config.data.get("sync_interval", 10))
else:
_sleep += 1
try:
_cluster = NS.tendrl.objects.Cluster(
integration_id=NS.tendrl_context.integration_id).load()
if _cluster.import_status == "failed":
continue
try:
NS._int.wclient.write("clusters/%s/"
"sync_status" %
NS.tendrl_context.integration_id,
"in_progress",
prevExist=False)
except (etcd.EtcdAlreadyExist, etcd.EtcdCompareFailed) as ex:
pass
subprocess.call([
'gluster', 'get-state', 'glusterd', 'odir', '/var/run',
'file', 'glusterd-state', 'detail'
])
raw_data = ini2json.ini_to_dict('/var/run/glusterd-state')
subprocess.call(['rm', '-rf', '/var/run/glusterd-state'])
subprocess.call([
'gluster', 'get-state', 'glusterd', 'odir', '/var/run',
'file', 'glusterd-state-vol-opts', 'volumeoptions'
])
raw_data_options = ini2json.ini_to_dict(
'/var/run/glusterd-state-vol-opts')
subprocess.call(
['rm', '-rf', '/var/run/glusterd-state-vol-opts'])
sync_object = NS.gluster.objects.\
SyncObject(data=json.dumps(raw_data))
sync_object.save()
if "Peers" in raw_data:
index = 1
peers = raw_data["Peers"]
while True:
try:
peer = NS.gluster.\
objects.Peer(
peer_uuid=peers['peer%s.uuid' % index],
hostname=peers[
'peer%s.primary_hostname' % index
],
state=peers['peer%s.state' % index],
connected=peers['peer%s.connected' % index]
)
try:
stored_peer_status = NS._int.client.read(
"clusters/%s/Peers/%s/connected" %
(NS.tendrl_context.integration_id,
peers['peer%s.uuid' % index])).value
current_status = peers['peer%s.connected' %
index]
if stored_peer_status != "" and \
current_status != stored_peer_status:
msg = (
"Status of peer: %s in cluster %s "
"changed from %s to %s") % (
peers['peer%s.primary_hostname' %
index],
NS.tendrl_context.integration_id,
stored_peer_status, current_status)
instance = "peer_%s" % peers[
'peer%s.primary_hostname' % index]
event_utils.emit_event(
"peer_status", current_status, msg,
instance, 'WARNING'
if current_status != 'Connected' else
'INFO')
except etcd.EtcdKeyNotFound:
pass
peer.save(ttl=SYNC_TTL)
index += 1
except KeyError:
break
if "Volumes" in raw_data:
index = 1
volumes = raw_data['Volumes']
while True:
try:
sync_volumes(
volumes, index,
raw_data_options.get('Volume Options'))
index += 1
except KeyError:
break
# populate the volume specific options
reg_ex = re.compile("^volume[0-9]+.options+")
options = {}
for key in volumes.keys():
if reg_ex.match(key):
options[key] = volumes[key]
for key in options.keys():
volname = key.split('.')[0]
vol_id = volumes['%s.id' % volname]
dict1 = {}
for k, v in options.items():
if k.startswith('%s.options' % volname):
dict1['.'.join(k.split(".")[2:])] = v
options.pop(k, None)
NS.gluster.objects.VolumeOptions(vol_id=vol_id,
options=dict1).save()
# Sync cluster global details
if "provisioner/%s" % NS.tendrl_context.integration_id \
in NS.node_context.tags:
all_volumes = NS.gluster.objects.Volume().load_all() or []
volumes = []
for volume in all_volumes:
if not str(volume.deleted).lower() == "true":
volumes.append(volume)
cluster_status.sync_cluster_status(volumes)
utilization.sync_utilization_details(volumes)
client_connections.sync_volume_connections(volumes)
georep_details.aggregate_session_status()
evt.process_events()
rebalance_status.sync_volume_rebalance_status(volumes)
rebalance_status.sync_volume_rebalance_estimated_time(
volumes)
snapshots.sync_volume_snapshots(
raw_data['Volumes'],
int(NS.config.data.get("sync_interval", 10)) +
len(volumes) * 10)
_cluster = NS.tendrl.objects.Cluster(
integration_id=NS.tendrl_context.integration_id)
if _cluster.exists():
_cluster = _cluster.load()
_cluster.sync_status = "done"
_cluster.last_sync = str(tendrl_now())
_cluster.is_managed = "yes"
_cluster.save()
# Initialize alert count
try:
alerts_count_key = '/clusters/%s/alert_counters' % (
NS.tendrl_context.integration_id)
etcd_utils.read(alerts_count_key)
except (etcd.EtcdException) as ex:
if type(ex) == etcd.EtcdKeyNotFound:
ClusterAlertCounters(
integration_id=NS.tendrl_context.integration_id
).save()
# check and enable volume profiling
if "provisioner/%s" % NS.tendrl_context.integration_id in \
NS.node_context.tags:
self._enable_disable_volume_profiling()
except Exception as ex:
Event(
ExceptionMessage(priority="error",
publisher=NS.publisher_id,
payload={
"message":
"gluster sds state sync error",
"exception": ex
}))
try:
etcd_utils.read('/clusters/%s/_sync_now' %
NS.tendrl_context.integration_id)
continue
except etcd.EtcdKeyNotFound:
pass
time.sleep(_sleep)
Event(
Message(
priority="debug",
publisher=NS.publisher_id,
payload={"message": "%s complete" % self.__class__.__name__}))
def test_parse_duplicate_section(self):
body = """[foo]\na=1\n[foo]\nbar=1\nbaz=2"""
filename = self._makeFile('pytest', body)
with pytest.raises(ValueError):
ini2json.ini_to_dict(filename)
def test_parse_empty_no_leading_whitespace(self):
body = """rem ember_last = True"""
filename = self._makeFile('pytest', body)
sections = ini2json.ini_to_dict(filename)
assert sections == {}
def test_parse_empty_with_comment_line(self):
body = """# empty with comment line"""
filename = self._makeFile('pytest', body)
sections = ini2json.ini_to_dict(filename)
assert sections == {}
def test_parse_empty_nodefaults(self):
filename = self._makeFile('empty', '')
sections = ini2json.ini_to_dict(filename)
assert sections == {}
def Initialize(self, body):
self.Manager.subprocess = self
self.Manager.ini2json = self
filename = self._makeFile('manager', body)
self.sections = ini2json.ini_to_dict(filename)
def _run(self):
Event(
Message(
priority="info",
publisher=NS.publisher_id,
payload={"message": "%s running" % self.__class__.__name__}
)
)
while not self._complete.is_set():
try:
# Acquire lock before updating the volume details in etcd
# We are blocking till we acquire the lock
lock = etcd.Lock(NS.etcd_orm.client, 'volume')
lock.acquire(blocking=True,lock_ttl=None)
gevent.sleep(3)
subprocess.call(
[
'gluster',
'get-state',
'glusterd',
'odir',
'/var/run',
'file',
'glusterd-state'
]
)
raw_data = ini2json.ini_to_dict('/var/run/glusterd-state')
subprocess.call(['rm', '-rf', '/var/run/glusterd-state'])
NS.sync_object = NS.gluster.objects.\
SyncObject(data=json.dumps(raw_data))
NS.sync_object.save()
if "Peers" in raw_data:
index = 1
peers = raw_data["Peers"]
while True:
try:
NS.peer = NS.gluster.\
objects.Peer(
peer_uuid=peers['peer%s.uuid' % index],
hostname=peers['peer%s.primary_hostname' % index],
state=peers['peer%s.state' % index]
)
NS.peer.save()
index += 1
except KeyError:
break
if "Volumes" in raw_data:
index = 1
volumes = raw_data['Volumes']
while True:
try:
NS.volume = NS.gluster.objects.Volume(
vol_id=volumes[
'volume%s.id' % index
],
vol_type=volumes[
'volume%s.type' % index
],
name=volumes[
'volume%s.name' % index
],
transport_type=volumes[
'volume%s.transport_type' % index
],
status=volumes[
'volume%s.status' % index
],
brick_count=volumes[
'volume%s.brickcount' % index
],
snap_count=volumes[
'volume%s.snap_count' % index
],
stripe_count=volumes[
'volume%s.stripe_count' % index
],
replica_count=volumes[
'volume%s.replica_count' % index
],
subvol_count=volumes[
'volume%s.subvol_count' % index
],
arbiter_count=volumes[
'volume%s.arbiter_count' % index
],
disperse_count=volumes[
'volume%s.disperse_count' % index
],
redundancy_count=volumes[
'volume%s.redundancy_count' % index
],
quorum_status=volumes[
'volume%s.quorum_status' % index
],
snapd_status=volumes[
'volume%s.snapd_svc.online_status' % index
],
snapd_inited=volumes[
'volume%s.snapd_svc.inited' % index
],
rebal_id=volumes[
'volume%s.rebalance.id' % index
],
rebal_status=volumes[
'volume%s.rebalance.status' % index
],
rebal_failures=volumes[
'volume%s.rebalance.failures' % index
],
rebal_skipped=volumes[
'volume%s.rebalance.skipped' % index
],
rebal_lookedup=volumes[
'volume%s.rebalance.lookedup' % index
],
rebal_files=volumes[
'volume%s.rebalance.files' % index
],
rebal_data=volumes[
'volume%s.rebalance.data' % index
],
)
NS.volume.save()
b_index = 1
while True:
try:
NS.brick = NS.gluster\
.objects.Brick(
vol_id=volumes['volume%s.id' % index],
path=volumes[
'volume%s.brick%s.path' % (
index, b_index)],
hostname=volumes.get(
'volume%s.brick%s.hostname' % (
index, b_index)),
port=volumes.get(
'volume%s.brick%s.port' % (
index, b_index)),
status=volumes.get(
'volume%s.brick%s.status' % (
index, b_index)),
filesystem_type=volumes.get(
'volume%s.brick%s.filesystem_type' % (
index, b_index)),
mount_opts=volumes.get(
'volume%s.brick%s.mount_options' % (
index, b_index))
)
NS.brick.save()
b_index += 1
except KeyError:
break
index += 1
except KeyError:
break
# poplate the volume options
reg_ex = re.compile("^volume[0-9]+.options+")
options = {}
for key in volumes.keys():
if reg_ex.match(key):
options[key] = volumes[key]
for key in options.keys():
volname = key.split('.')[0]
vol_id = volumes['%s.id' % volname]
dict1 = {}
for k, v in options.items():
if k.startswith('%s.options' % volname):
dict1['.'.join(k.split(".")[2:])] = v
options.pop(k, None)
NS.vol_options = NS.gluster.objects.\
VolumeOptions(
vol_id=vol_id,
options=dict1
)
NS.vol_options.save()
# Sync the cluster status details
args = ["gstatus", "-o", "json"]
p = subprocess.Popen(
args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=open(os.devnull, "r")
)
stdout, stderr = p.communicate()
if stderr == "" and p.returncode == 0:
# The format of the output from gstatus is as below
# "2017-02-23 18:13:01.944183 {"brick_count": 2,
# bricks_active": 2, "glfs_version": "3.9.0",
# node_count": 2, "nodes_active": 2, "over_commit": "No",
# product_name": "Community", "raw_capacity": 52181536768,
# sh_active": 0, "sh_enabled": 0, "snapshot_count": 0,
# status": "healthy", "usable_capacity": 52181536768,
# used_capacity": 2117836800, "volume_count": 1,
# volume_summary": [{"snapshot_count": 0, "state": "up",
# usable_capacity": 52181536768, "used_capacity": 2117836800,
# volume_name": "vol1"}]}\n"
out_dict = json.loads(stdout[stdout.index('{'): -1])
NS.gluster.objects.GlobalDetails(
status=out_dict['status']
).save()
NS.gluster.objects.Utilization(
raw_capacity=out_dict['raw_capacity'],
usable_capacity=out_dict['usable_capacity'],
used_capacity=out_dict['used_capacity'],
pcnt_used=(out_dict['used_capacity'] * 100 / out_dict['usable_capacity'])
).save()
for item in out_dict['volume_summary']:
volumes = NS.etcd_orm.client.read(
"clusters/%s/Volumes" % NS.tendrl_context.integration_id
)
for child in volumes._children:
volume = NS.gluster.objects.Volume(
vol_id=child['key'].split('/')[-1]
).load()
if volume.name == item['volume_name']:
NS.gluster.objects.Volume(
vol_id=child['key'].split('/')[-1],
usable_capacity=str(item['usable_capacity']),
used_capacity=str(item['used_capacity']),
pcnt_used=str(item['used_capacity'] * 100 / item['usable_capacity']),
vol_type=volume.vol_type,
name=volume.name,
transport_type=volume.transport_type,
status=volume.status,
brick_count=volume.brick_count,
snap_count=volume.snap_count,
stripe_count=volume.stripe_count,
replica_count=volume.replica_count,
subvol_count=volume.subvol_count,
arbiter_count=volume.arbiter_count,
disperse_count=volume.disperse_count,
redundancy_count=volume.redundancy_count,
quorum_status=volume.quorum_status,
snapd_status=volume.snapd_status,
snapd_inited=volume.snapd_inited,
rebal_id=volume.rebal_id,
rebal_status=volume.rebal_status,
rebal_failures=volume.rebal_failures,
rebal_skipped=volume.rebal_skipped,
rebal_lookedup=volume.rebal_lookedup,
rebal_files=volume.rebal_files,
rebal_data=volume.rebal_data
).save()
except Exception as ex:
Event(
ExceptionMessage(
priority="error",
publisher=NS.publisher_id,
payload={"message": "error",
"exception": ex
}
)
)
finally:
lock.release()
Event(
Message(
priority="info",
publisher=NS.publisher_id,
payload={"message": "%s complete" % self.__class__.__name__}
)
)
def test_parse_(self):
body = """[foo]\na=1\n[memo]\n bar=1\nbaz=2"""
filename = self._makeFile('pytest', body)
with pytest.raises(ini2json.ParsingError):
ini2json.ini_to_dict(filename)
def snapshot_restored(self, event):
time.sleep(self.sync_interval)
message = event["message"]
volume = message['volume_name']
volume_id = ""
bricks_to_remove = []
# get the list of current bricks by running get-state
output_dir = '/var/run/'
output_file = 'glusterd-state-snapshot-%s' % str(uuid.uuid4())
subprocess.call(
[
'gluster',
'get-state',
'glusterd',
'odir',
output_dir,
'file',
output_file,
'detail'
]
)
raw_data = ini2json.ini_to_dict(
output_dir + output_file
)
subprocess.call(['rm', '-rf', output_dir + output_file])
index = 1
while True:
try:
current_vol = 'volume%s.name' % index
if raw_data['Volumes'][current_vol] == volume:
current_vol_id = 'volume%s.id' % index
volume_id = raw_data['Volumes'][current_vol_id]
break
except KeyError:
return
index += 1
latest_bricks = []
b_index = 1
while True:
try:
curr_brick = 'volume%s.brick%s.path' % (
index, b_index
)
brick = raw_data['Volumes'][curr_brick]
b_index += 1
except KeyError:
break
latest_bricks.append(brick)
# get the list of bricks in etcd for this volume
sub_volumes = etcd_utils.read(
"/clusters/{0}/Volumes/{1}/Bricks".format(
NS.tendrl_context.integration_id,
volume_id
)
)
for sub_volume in sub_volumes.leaves:
bricks = etcd_utils.read(
sub_volume.key
)
for brick in bricks.leaves:
fqdn = brick.key.split('/')[-1].split(':')[0]
path = brick.key.split('/')[-1].split(':')[-1][1:]
brick_path = "clusters/{0}/Bricks/"\
"all/{1}/{2}".format(
NS.tendrl_context.integration_id,
fqdn,
path
)
brick_full_path = etcd_utils.read(
"%s/brick_path" % brick_path
).value
if brick_full_path not in latest_bricks:
bricks_to_remove.append(brick_full_path)
brick_details = {}
brick_details["volume"] = volume
brick_details["bricks"] = " ".join(bricks_to_remove)
event["message"] = brick_details
self.volume_remove_brick_force(event)
def run(self):
logger.log(
"info",
NS.publisher_id,
{"message": "%s running" % self.__class__.__name__}
)
gluster_brick_dir = NS.gluster.objects.GlusterBrickDir()
gluster_brick_dir.save()
cluster = NS.tendrl.objects.Cluster(
integration_id=NS.tendrl_context.integration_id
).load()
if cluster.cluster_network in [None, ""]:
try:
node_networks = NS.tendrl.objects.NodeNetwork().load_all()
cluster.cluster_network = node_networks[0].subnet
cluster.save()
except etcd.EtcdKeyNotFound as ex:
logger.log(
"error",
NS.publisher_id,
{"message": "Failed to sync cluster network details"}
)
_sleep = 0
while not self._complete.is_set():
# To detect out of band deletes
# refresh gluster object inventory at config['sync_interval']
SYNC_TTL = int(NS.config.data.get("sync_interval", 10)) + 100
NS.node_context = NS.node_context.load()
NS.tendrl_context = NS.tendrl_context.load()
if _sleep > 5:
_sleep = int(NS.config.data.get("sync_interval", 10))
else:
_sleep += 1
try:
_cluster = NS.tendrl.objects.Cluster(
integration_id=NS.tendrl_context.integration_id
).load()
if (_cluster.status == "importing" and
_cluster.current_job['status'] == 'failed') or \
_cluster.status == "unmanaging" or \
_cluster.status == "set_volume_profiling":
continue
_cnc = NS.tendrl.objects.ClusterNodeContext(
node_id=NS.node_context.node_id
).load()
_cnc.is_managed = "yes"
_cnc.save()
subprocess.call(
[
'gluster',
'get-state',
'glusterd',
'odir',
'/var/run',
'file',
'glusterd-state',
'detail'
]
)
raw_data = ini2json.ini_to_dict(
'/var/run/glusterd-state'
)
subprocess.call(['rm', '-rf', '/var/run/glusterd-state'])
subprocess.call(
[
'gluster',
'get-state',
'glusterd',
'odir',
'/var/run',
'file',
'glusterd-state-vol-opts',
'volumeoptions'
]
)
raw_data_options = ini2json.ini_to_dict(
'/var/run/glusterd-state-vol-opts'
)
subprocess.call(
[
'rm',
'-rf',
'/var/run/glusterd-state-vol-opts'
]
)
sync_object = NS.gluster.objects.\
SyncObject(data=json.dumps(raw_data))
sync_object.save()
if "Peers" in raw_data:
index = 1
peers = raw_data["Peers"]
disconnected_hosts = []
while True:
try:
peer = NS.tendrl.\
objects.GlusterPeer(
peer_uuid=peers['peer%s.uuid' % index],
hostname=peers[
'peer%s.primary_hostname' % index
],
state=peers['peer%s.state' % index],
connected=peers['peer%s.connected' % index]
)
try:
stored_peer_status = None
# find peer detail using hostname
ip = socket.gethostbyname(
peers['peer%s.primary_hostname' % index]
)
node_id = etcd_utils.read(
"/indexes/ip/%s" % ip
).value
stored_peer = NS.tendrl.objects.GlusterPeer(
peer_uuid=peers['peer%s.uuid' % index],
node_id=node_id
).load()
stored_peer_status = stored_peer.connected
current_status = peers[
'peer%s.connected' % index
]
if stored_peer_status and \
current_status != stored_peer_status:
msg = (
"Peer %s in cluster %s "
"is %s"
) % (
peers[
'peer%s.primary_hostname' %
index
],
_cluster.short_name,
current_status
)
instance = "peer_%s" % peers[
'peer%s.primary_hostname' % index
]
event_utils.emit_event(
"peer_status",
current_status,
msg,
instance,
'WARNING' if current_status !=
'Connected'
else 'INFO'
)
# save current status in actual peer
# directory also
stored_peer.connected = current_status
stored_peer.save()
# Disconnected host name to
# raise brick alert
if current_status.lower() == \
"disconnected":
disconnected_hosts.append(
peers[
'peer%s.primary_hostname' %
index
]
)
except etcd.EtcdKeyNotFound:
pass
SYNC_TTL += 5
peer.save(ttl=SYNC_TTL)
index += 1
except KeyError:
break
# Raise an alert for bricks when peer disconnected
# or node goes down
for disconnected_host in disconnected_hosts:
brick_status_alert(
disconnected_host
)
if "Volumes" in raw_data:
index = 1
volumes = raw_data['Volumes']
# instantiating blivet class, this will be used for
# getting brick_device_details
b = blivet.Blivet()
# reset blivet during every sync to get latest information
# about storage devices in the machine
b.reset()
devicetree = b.devicetree
total_brick_count = 0
while True:
try:
b_count = sync_volumes(
volumes, index,
raw_data_options.get('Volume Options'),
SYNC_TTL + VOLUME_TTL,
_cluster.short_name,
devicetree
)
index += 1
SYNC_TTL += 1
total_brick_count += b_count - 1
except KeyError:
global VOLUME_TTL
# from second sync volume ttl is
# SYNC_TTL + (no.volumes) * 20 +
# (no.of.bricks) * 10 + 160
if index > 1:
volume_count = index - 1
# When all nodes are down we are updating all
# volumes are down, node status TTL is 160,
# So make sure volumes are present in etcd
# while raising volume down alert
VOLUME_TTL = (volume_count * 20) + (
total_brick_count * 10) + 160
break
# populate the volume specific options
reg_ex = re.compile("^volume[0-9]+.options+")
options = {}
for key in volumes.keys():
if reg_ex.match(key):
options[key] = volumes[key]
for key in options.keys():
volname = key.split('.')[0]
vol_id = volumes['%s.id' % volname]
dict1 = {}
for k, v in options.items():
if k.startswith('%s.options' % volname):
dict1['.'.join(k.split(".")[2:])] = v
options.pop(k, None)
volume = NS.tendrl.objects.GlusterVolume(
NS.tendrl_context.integration_id,
vol_id=vol_id
).load()
if volume.options is not None:
dest = dict(volume.options)
dest.update(dict1)
volume.options = dest
volume.save()
# Sync cluster global details
if "provisioner/%s" % NS.tendrl_context.integration_id \
in NS.node_context.tags:
all_volumes = NS.tendrl.objects.GlusterVolume(
NS.tendrl_context.integration_id
).load_all() or []
volumes = []
for volume in all_volumes:
if not str(volume.deleted).lower() == "true" and \
volume.current_job.get('status', '') \
in ['', 'finished', 'failed'] and \
volume.vol_id not in [None, ''] and \
volume.name not in [None, '']:
# only for first sync refresh volume TTL
# It will increase TTL based on no.of volumes
if _cnc.first_sync_done in [None, "no", ""]:
etcd_utils.refresh(
volume.value,
SYNC_TTL + VOLUME_TTL
)
volumes.append(volume)
cluster_status.sync_cluster_status(
volumes, SYNC_TTL + VOLUME_TTL
)
utilization.sync_utilization_details(volumes)
client_connections.sync_volume_connections(volumes)
georep_details.aggregate_session_status()
try:
evt.process_events()
except etcd.EtcdKeyNotFound:
pass
rebalance_status.sync_volume_rebalance_status(volumes)
rebalance_status.sync_volume_rebalance_estimated_time(
volumes
)
snapshots.sync_volume_snapshots(
raw_data['Volumes'],
int(NS.config.data.get(
"sync_interval", 10
)) + len(volumes) * 4
)
# update alert count
update_cluster_alert_count()
# check and enable volume profiling
if "provisioner/%s" % NS.tendrl_context.integration_id in \
NS.node_context.tags:
self._enable_disable_volume_profiling()
_cluster = NS.tendrl.objects.Cluster(
integration_id=NS.tendrl_context.integration_id
).load()
if _cluster.exists():
_cluster = _cluster.load()
_cluster.last_sync = str(tendrl_now())
# Mark the first sync done flag
_cnc = NS.tendrl.objects.ClusterNodeContext(
node_id=NS.node_context.node_id
).load()
if _cnc.first_sync_done in [None, "no"]:
_cnc.first_sync_done = "yes"
_cnc.save()
if _cluster.current_job.get(
'status', ''
) in ['', 'finished', 'failed'] and \
_cluster.status in [None, ""]:
_cluster.save()
except Exception as ex:
Event(
ExceptionMessage(
priority="error",
publisher=NS.publisher_id,
payload={"message": "gluster sds state sync error",
"exception": ex
}
)
)
try:
etcd_utils.read(
'/clusters/%s/_sync_now' %
NS.tendrl_context.integration_id
)
continue
except etcd.EtcdKeyNotFound:
pass
time.sleep(_sleep)
logger.log(
"debug",
NS.publisher_id,
{"message": "%s complete" % self.__class__.__name__}
)
def _run(self):
LOG.info("%s running" % self.__class__.__name__)
while not self._complete.is_set():
try:
gevent.sleep(3)
subprocess.call(
[
'gluster',
'get-state',
'glusterd',
'odir',
'/var/run',
'file',
'glusterd-state'
]
)
raw_data = ini2json.ini_to_dict('/var/run/glusterd-state')
subprocess.call(['rm', '-rf', '/var/run/glusterd-state'])
tendrl_ns.sync_object = tendrl_ns.gluster_integration.objects.\
SyncObject(data=json.dumps(raw_data))
tendrl_ns.sync_object.save()
if "Peers" in raw_data:
index = 1
peers = raw_data["Peers"]
while True:
try:
tendrl_ns.peer = tendrl_ns.gluster_integration.\
objects.Peer(
peer_uuid=peers['peer%s.uuid' % index],
hostname=peers['peer%s.primary_hostname' % index],
state=peers['peer%s.state' % index]
)
tendrl_ns.peer.save()
index += 1
except KeyError:
break
if "Volumes" in raw_data:
index = 1
volumes = raw_data['Volumes']
while True:
try:
tendrl_ns.volume = tendrl_ns.gluster_integration.objects.Volume(
vol_id=volumes[
'volume%s.id' % index
],
vol_type=volumes[
'volume%s.type' % index
],
name=volumes[
'volume%s.name' % index
],
transport_type=volumes[
'volume%s.transport_type' % index
],
status=volumes[
'volume%s.status' % index
],
brick_count=volumes[
'volume%s.brickcount' % index
],
snap_count=volumes[
'volume%s.snap_count' % index
],
stripe_count=volumes[
'volume%s.stripe_count' % index
],
replica_count=volumes[
'volume%s.replica_count' % index
],
subvol_count=volumes[
'volume%s.subvol_count' % index
],
arbiter_count=volumes[
'volume%s.arbiter_count' % index
],
disperse_count=volumes[
'volume%s.disperse_count' % index
],
redundancy_count=volumes[
'volume%s.redundancy_count' % index
],
quorum_status=volumes[
'volume%s.quorum_status' % index
],
snapd_status=volumes[
'volume%s.snapd_svc.online_status' % index
],
snapd_inited=volumes[
'volume%s.snapd_svc.inited' % index
],
rebal_id=volumes[
'volume%s.rebalance.id' % index
],
rebal_status=volumes[
'volume%s.rebalance.status' % index
],
rebal_failures=volumes[
'volume%s.rebalance.failures' % index
],
rebal_skipped=volumes[
'volume%s.rebalance.skipped' % index
],
rebal_lookedup=volumes[
'volume%s.rebalance.lookedup' % index
],
rebal_files=volumes[
'volume%s.rebalance.files' % index
],
rebal_data=volumes[
'volume%s.rebalance.data' % index
],
)
tendrl_ns.volume.save()
b_index = 1
while True:
try:
tendrl_ns.brick = tendrl_ns.gluster_integration\
.objects.Brick(
vol_id=volumes['volume%s.id' % index],
path=volumes[
'volume%s.brick%s.path' % (
index, b_index)],
hostname=volumes.get(
'volume%s.brick%s.hostname' % (
index, b_index)),
port=volumes.get(
'volume%s.brick%s.port' % (
index, b_index)),
status=volumes.get(
'volume%s.brick%s.status' % (
index, b_index)),
filesystem_type=volumes.get(
'volume%s.brick%s.filesystem_type' % (
index, b_index)),
mount_opts=volumes.get(
'volume%s.brick%s.mount_options' % (
index, b_index))
)
tendrl_ns.brick.save()
b_index += 1
except KeyError:
break
index += 1
except KeyError:
break
# poplate the volume options
reg_ex = re.compile("^volume[0-9]+.options+")
options = {}
for key in volumes.keys():
if reg_ex.match(key):
options[key] = volumes[key]
for key in options.keys():
volname = key.split('.')[0]
vol_id = volumes['%s.id' % volname]
dict1 = {}
for k, v in options.items():
if k.startswith('%s.options' % volname):
dict1['.'.join(k.split(".")[2:])] = v
options.pop(k, None)
tendrl_ns.vol_options = tendrl_ns.gluster_integration.objects.\
VolumeOptions(
vol_id=vol_id,
options=dict1
)
tendrl_ns.vol_options.save()
# Sync the cluster status details
args = ["gstatus", "-o", "json"]
p = subprocess.Popen(
args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=open(os.devnull, "r")
)
stdout, stderr = p.communicate()
if stderr == "" and p.returncode == 0:
# The format of the output from gstatus is as below
# "2017-02-23 18:13:01.944183 {"brick_count": 2,
# bricks_active": 2, "glfs_version": "3.9.0",
# node_count": 2, "nodes_active": 2, "over_commit": "No",
# product_name": "Community", "raw_capacity": 52181536768,
# sh_active": 0, "sh_enabled": 0, "snapshot_count": 0,
# status": "healthy", "usable_capacity": 52181536768,
# used_capacity": 2117836800, "volume_count": 1,
# volume_summary": [{"snapshot_count": 0, "state": "up",
# usable_capacity": 52181536768, "used_capacity": 2117836800,
# volume_name": "vol1"}]}\n"
out_dict = json.loads(stdout[stdout.index('{'): -1])
tendrl_ns.gluster_integration.objects.GlobalDetails(
status=out_dict['status']
).save()
tendrl_ns.gluster_integration.objects.Utilization(
raw_capacity=out_dict['raw_capacity'],
usable_capacity=out_dict['usable_capacity'],
used_capacity=out_dict['used_capacity'],
pcnt_used=(out_dict['used_capacity'] * 100 / out_dict['usable_capacity'])
).save()
for item in out_dict['volume_summary']:
tendrl_ns.gluster_integration.objects.Volume(
name=item['volume_name'],
usable_capacity=item['usable_capacity'],
used_capacity=item['used_capacity'],
pcnt_used=(item['used_capacity'] * 100 / item['usable_capacity'])
).save()
except Exception as ex:
LOG.error(ex)
LOG.info("%s complete" % self.__class__.__name__)