def test_sync_volumes(refresh, emit_event, compare, save, load, blivet): init() refresh.return_value = True compare.return_value = True save.return_value = True emit_event.return_value = True NS.node_context = node_context.NodeContext obj = node_context.NodeContext( node_id="eeebe9f5-6e43-4cf8-b321-37d648eb0510", fqdn="dhcp12-12.lab.abc.com", tags=["provisioner/%s" % NS.tendrl_context.integration_id], ipv4_addr="127.0.0.1") NS.node_context = obj load.return_value = obj sds_sync = importlib.import_module('tendrl.gluster_integration.sds_sync') raw_data = ini2json.ini_to_dict( os.path.join(os.path.dirname(__file__), "gluster-state.yaml")) raw_data_options = ini2json.ini_to_dict( os.path.join(os.path.dirname(__file__), "gluster-volume-option.yaml")) sds_sync.Event = MagicMock() sds_sync.Message = MagicMock() with patch.object(NS._int.client, "read", read): with patch.object(brick_device_details, "update_brick_device_details") as update: update.return_value = True with patch.object(brick_utilization, "brick_utilization") as utilization: utilization.return_value = 10 sds_sync.sync_volumes(raw_data['Volumes'], 1, raw_data_options.get('Volume Options'), 10)
def test_parse_continuation_line(self): body = """[foo]\nbar:1 2 jammy:jam\n foo bar\n boo""" filename = self._makeFile('pytest', body) sections = ini2json.ini_to_dict(filename) assert sections == ( {'foo': {'bar': ['1', '2', 'jammy:jam', 'foo', 'bar', 'boo']}} )
def test_snapshot_restored(read, call, sleep, load): load.return_value = maps.NamedDict( short_name="77deef29-b8e5-4dc5-8247-21e2a409a66a") sleep.return_value = True call.return_value = True keys = maps.NamedDict(key="sub_vol1/dhcp123-12.lab.abc.com:|gluster|b1") read.return_value = maps.NamedDict(leaves=[keys], value="/gluster/b1") event = { "message": { "volume_name": "GlusterVolume" }, "event": "SNAPSHOT_RESTORED", "ts": 1486634392, "nodeid": "0f5b4b99-fa4a-4b8f-be52-770b42879d67" } raw_data = ini2json.ini_to_dict( os.path.join(os.path.dirname(__file__), "gluster_state.yaml")) with patch.object(ini2json, "ini_to_dict") as ini_to_dict: ini_to_dict.return_value = raw_data obj = Callback() with patch.object(obj, "volume_remove_brick_force") as rm_brick: obj.snapshot_restored(event) rm_brick.assert_called_with({ 'event': 'SNAPSHOT_RESTORED', 'ts': 1486634392, 'message': { 'volume': 'GlusterVolume', 'bricks': '/gluster/b1' }, 'nodeid': '0f5b4b99-fa4a-4b8f-be52-770b42879d67' })
def test_snapshot_restored(read, call, sleep, load): load.return_value = maps.NamedDict( short_name="77deef29-b8e5-4dc5-8247-21e2a409a66a" ) sleep.return_value = True call.return_value = True keys = maps.NamedDict( key="sub_vol1/dhcp123-12.lab.abc.com:|gluster|b1" ) read.return_value = maps.NamedDict(leaves=[keys], value="/gluster/b1") event = {"message": {"volume_name": "GlusterVolume"}, "event": "SNAPSHOT_RESTORED", "ts": 1486634392, "nodeid": "0f5b4b99-fa4a-4b8f-be52-770b42879d67" } raw_data = ini2json.ini_to_dict( os.path.join(os.path.dirname(__file__), "gluster_state.yaml") ) with patch.object(ini2json, "ini_to_dict") as ini_to_dict: ini_to_dict.return_value = raw_data obj = Callback() with patch.object(obj, "volume_remove_brick_force") as rm_brick: obj.snapshot_restored(event) rm_brick.assert_called_with( {'event': 'SNAPSHOT_RESTORED', 'ts': 1486634392, 'message': {'volume': 'GlusterVolume', 'bricks': '/gluster/b1' }, 'nodeid': '0f5b4b99-fa4a-4b8f-be52-770b42879d67' } )
def test_parse_with_bad_format(self): body = """[foo]\nbar: 1 ;\n 2 3 4 jammy:jam\n""" filename = self._makeFile('pytest', body) sections = ini2json.ini_to_dict(filename) assert sections == ( {'foo': {'bar': ['1', '2', '3', '4', 'jammy:jam']}} )
def test_parse_without_space(self): body = """[foo]\nbar:""\n""" filename = self._makeFile('pytest', body) sections = ini2json.ini_to_dict(filename) assert sections == ( {'foo': {'bar': ''}} )
def test_parse_body_default(self): body = """[foo]\nbar=1\nbaz=2"\n[DEFAULT]\na=1""" filename = self._makeFile('pytest', body) sections = ini2json.ini_to_dict(filename) assert sections == ( {'foo': {'a': '1', 'bar': '1', 'baz': '2"'}} )
def _run(self): Event( Message( priority="info", publisher=NS.publisher_id, payload={"message": "%s running" % self.__class__.__name__})) gluster_brick_dir = NS.gluster.objects.\ GlusterBrickDir() gluster_brick_dir.save() while not self._complete.is_set(): try: # Acquire lock before updating the volume details in etcd # We are blocking till we acquire the lock. # the lock will live for 60 sec after which it will be released. lock = etcd.Lock(NS._int.wclient, 'volume') try: lock.acquire(blocking=True, lock_ttl=60) if lock.is_acquired: # renewing the lock lock.acquire(lock_ttl=60) except etcd.EtcdLockExpired: continue gevent.sleep(10) subprocess.call([ 'gluster', 'get-state', 'glusterd', 'odir', '/var/run', 'file', 'glusterd-state' ]) raw_data = ini2json.ini_to_dict('/var/run/glusterd-state') subprocess.call(['rm', '-rf', '/var/run/glusterd-state']) sync_object = NS.gluster.objects.\ SyncObject(data=json.dumps(raw_data)) sync_object.save() if "Peers" in raw_data: index = 1 peers = raw_data["Peers"] while True: try: peer = NS.gluster.\ objects.Peer( peer_uuid=peers['peer%s.uuid' % index], hostname=peers['peer%s.primary_hostname' % index], state=peers['peer%s.state' % index] ) peer.save() index += 1 except KeyError: break if "Volumes" in raw_data: index = 1 volumes = raw_data['Volumes'] node_context = NS.node_context.load() tag_list = list(node_context.tags) while True: try: # Raise alerts for volume state change. cluster_provisioner = "provisioner/%s" % NS.tendrl_context.integration_id if cluster_provisioner in tag_list: try: stored_volume_status = NS._int.client.read( "clusters/%s/Volumes/%s/status" % (NS.tendrl_context.integration_id, volumes['volume%s.id' % index])).value current_status = volumes['volume%s.status' % index] if current_status != stored_volume_status: msg = "Status of volume: %s changed from %s to %s" % ( volumes['volume%s.name' % index], stored_volume_status, current_status) instance = "volume_%s" % volumes[ 'volume%s.name' % index] self._emit_event( "volume_status", current_status, msg, instance) except etcd.EtcdKeyNotFound: pass volume = NS.gluster.objects.Volume( vol_id=volumes['volume%s.id' % index], vol_type=volumes['volume%s.type' % index], name=volumes['volume%s.name' % index], transport_type=volumes[ 'volume%s.transport_type' % index], status=volumes['volume%s.status' % index], brick_count=volumes['volume%s.brickcount' % index], snap_count=volumes['volume%s.snap_count' % index], stripe_count=volumes['volume%s.stripe_count' % index], replica_count=volumes['volume%s.replica_count' % index], subvol_count=volumes['volume%s.subvol_count' % index], arbiter_count=volumes['volume%s.arbiter_count' % index], disperse_count=volumes[ 'volume%s.disperse_count' % index], redundancy_count=volumes[ 'volume%s.redundancy_count' % index], quorum_status=volumes['volume%s.quorum_status' % index], snapd_status=volumes[ 'volume%s.snapd_svc.online_status' % index], snapd_inited=volumes[ 'volume%s.snapd_svc.inited' % index], rebal_id=volumes['volume%s.rebalance.id' % index], rebal_status=volumes[ 'volume%s.rebalance.status' % index], rebal_failures=volumes[ 'volume%s.rebalance.failures' % index], rebal_skipped=volumes[ 'volume%s.rebalance.skipped' % index], rebal_lookedup=volumes[ 'volume%s.rebalance.lookedup' % index], rebal_files=volumes['volume%s.rebalance.files' % index], rebal_data=volumes['volume%s.rebalance.data' % index], ) volume.save() b_index = 1 # ipv4 address of current node try: network_ip = [] networks = NS._int.client.read( "nodes/%s/Networks" % NS.node_context.node_id) for interface in networks.leaves: key = interface.key.split("/")[-1] network = NS.tendrl.objects.NodeNetwork( interface=key).load() network_ip.extend(network.ipv4) except etcd.EtcdKeyNotFound as ex: Event( ExceptionMessage( priority="debug", publisher=NS.publisher_id, payload={ "message": "Could not find any ipv4 networks for node %s" % NS.node_context.node_id, "exception": ex })) while True: try: # Update brick node wise hostname = volumes[ 'volume%s.brick%s.hostname' % (index, b_index)] if (NS.node_context.fqdn != hostname) and ( hostname not in network_ip): b_index += 1 continue # Raise alerts if the brick path changes try: stored_brick_status = NS._int.client.read( "clusters/%s/Volumes/%s/Bricks/%s/status" % (NS.tendrl_context.integration_id, volumes['volume%s.id' % index], volumes['volume%s.brick%s.path' % (index, b_index)].replace( "/", "_"))).value current_status = volumes.get( 'volume%s.brick%s.status' % (index, b_index)) if current_status != stored_brick_status: msg = "Status of brick: %s under volume %s changed from %s to %s" % ( volumes['volume%s.brick%s.path' % (index, b_index)], volumes['volume%s.name' % index], stored_brick_status, current_status) instance = "volume_%s|brick_%s" % ( volumes['volume%s.name' % index], volumes['volume%s.brick%s.path' % (index, b_index)]) self._emit_event( "brick_status", current_status, msg, instance) except etcd.EtcdKeyNotFound: pass brick = NS.gluster\ .objects.Brick( vol_id=volumes['volume%s.id' % index], sequence_number=b_index, path=volumes[ 'volume%s.brick%s.path' % ( index, b_index)], hostname=volumes.get( 'volume%s.brick%s.hostname' % ( index, b_index)), port=volumes.get( 'volume%s.brick%s.port' % ( index, b_index)), status=volumes.get( 'volume%s.brick%s.status' % ( index, b_index)), filesystem_type=volumes.get( 'volume%s.brick%s.filesystem_type' % ( index, b_index)), mount_opts=volumes.get( 'volume%s.brick%s.mount_options' % ( index, b_index)), utilization=brick_utilization\ .brick_utilization( volumes['volume%s.brick%s.path' % ( index, b_index)]) ) brick.save() b_index += 1 except KeyError: break index += 1 except KeyError: break # poplate the volume options reg_ex = re.compile("^volume[0-9]+.options+") options = {} for key in volumes.keys(): if reg_ex.match(key): options[key] = volumes[key] for key in options.keys(): volname = key.split('.')[0] vol_id = volumes['%s.id' % volname] dict1 = {} for k, v in options.items(): if k.startswith('%s.options' % volname): dict1['.'.join(k.split(".")[2:])] = v options.pop(k, None) vol_options = NS.gluster.objects.\ VolumeOptions( vol_id=vol_id, options=dict1 ) vol_options.save() # Sync the cluster status details args = ["gstatus", "-o", "json"] p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=open(os.devnull, "r")) stdout, stderr = p.communicate() if stderr == "" and p.returncode == 0: # The format of the output from gstatus is as below # "2017-02-23 18:13:01.944183 {"brick_count": 2, # bricks_active": 2, "glfs_version": "3.9.0", # node_count": 2, "nodes_active": 2, "over_commit": "No", # product_name": "Community", "raw_capacity": 52181536768, # sh_active": 0, "sh_enabled": 0, "snapshot_count": 0, # status": "healthy", "usable_capacity": 52181536768, # used_capacity": 2117836800, "volume_count": 1, # volume_summary": [{"snapshot_count": 0, "state": "up", # usable_capacity": 52181536768, "used_capacity": 2117836800, # volume_name": "vol1"}]}\n" out_dict = json.loads(stdout[stdout.index('{'):-1]) NS.gluster.objects.Utilization( raw_capacity=out_dict['raw_capacity'], usable_capacity=out_dict['usable_capacity'], used_capacity=out_dict['used_capacity'], pcnt_used=(out_dict['used_capacity'] * 100 / out_dict['usable_capacity'])).save() volume_up_degraded = 0 for item in out_dict['volume_summary']: if "up(degraded)" in item['state']: volume_up_degraded = volume_up_degraded + 1 volumes = NS._int.client.read( "clusters/%s/Volumes" % NS.tendrl_context.integration_id) for child in volumes._children: volume = NS.gluster.objects.Volume( vol_id=child['key'].split('/')[-1]).load() if volume.name == item['volume_name']: NS.gluster.objects.Volume( vol_id=child['key'].split('/')[-1], usable_capacity=str( item['usable_capacity']), used_capacity=str(item['used_capacity']), pcnt_used=str(item['used_capacity'] * 100 / item['usable_capacity']), vol_type=volume.vol_type, name=volume.name, transport_type=volume.transport_type, status=volume.status, brick_count=volume.brick_count, snap_count=volume.snap_count, stripe_count=volume.stripe_count, replica_count=volume.replica_count, subvol_count=volume.subvol_count, arbiter_count=volume.arbiter_count, disperse_count=volume.disperse_count, redundancy_count=volume.redundancy_count, quorum_status=volume.quorum_status, snapd_status=volume.snapd_status, snapd_inited=volume.snapd_inited, rebal_id=volume.rebal_id, rebal_status=volume.rebal_status, rebal_failures=volume.rebal_failures, rebal_skipped=volume.rebal_skipped, rebal_lookedup=volume.rebal_lookedup, rebal_files=volume.rebal_files, rebal_data=volume.rebal_data).save() connection_count = None connection_active = None # gstatus result: # Product: Community Capacity: 25.00 GiB(raw bricks) # Status: HEALTHY 3.00 GiB(raw used) # Glusterfs: 3.9.0 87.00 GiB(usable from volumes) # OverCommit: Yes Snapshots: 0 # Nodes : 2/ 2 Volumes: 4 Up # Self Heal : 0/ 0 0 Up(Degraded) # Bricks : 7/ 7 0 Up(Partial) # Connections : 1/ 2 0 Down # grep result: # Connections : 0/ 0 1 Down cmd = cmd_utils.Command('gstatus -s | grep Connections', True) out, err, rc = cmd.run() if not err: if "Connection" in out: out = (re.split(r'\s{5,}', out)[0]) out = out.split(":")[1].replace(" ", "").split("/") connection_active = int(out[0]) connection_count = int(out[1]) NS.gluster.objects.GlobalDetails( status=out_dict['status'], volume_up_degraded=volume_up_degraded, connection_active=connection_active, connection_count=connection_count).save() else: # If gstatus does not return any status details in absence # of volumes, default set the status and utilization details # A sample output from gstatus if no volumes in cluster is as below # # >gstatus -o json # This cluster doesn't have any volumes/daemons running. # The output below shows the current nodes attached to this host. # # UUID Hostname State # 6a48d2f7-3859-4542-86e0-0a8146588f31 {FQDN-1} Connected # 7380e303-83b6-4728-918f-e99029bc1bce {FQDN-2} Connected # 751ecb42-da85-4c3d-834d-9824d1ce7fd3 {FQDN-3} Connected # 388b708c-a86c-4a16-9a6f-f0d53ea79a51 localhost Connected out_lines = stdout.split('\n') connected = True for index in range(4, len(out_lines) - 2): node_status_det = out_lines[index].split('\t') if len(node_status_det) > 2: if node_status_det[2].strip() != 'Connected': connected = connected and False if connected: NS.gluster.objects.GlobalDetails( status='healthy').save() else: NS.gluster.objects.GlobalDetails( status='unhealthy').save() NS.gluster.objects.Utilization(raw_capacity=0, usable_capacity=0, used_capacity=0, pcnt_used=0).save() except Exception as ex: Event( ExceptionMessage(priority="error", publisher=NS.publisher_id, payload={ "message": "gluster sds state sync error", "exception": ex })) finally: lock.release() Event( Message( priority="debug", publisher=NS.publisher_id, payload={"message": "%s complete" % self.__class__.__name__}))
def run(self): logger.log( "info", NS.publisher_id, {"message": "%s running" % self.__class__.__name__} ) gluster_brick_dir = NS.gluster.objects.GlusterBrickDir() gluster_brick_dir.save() cluster = NS.tendrl.objects.Cluster( integration_id=NS.tendrl_context.integration_id ).load() if cluster.cluster_network in [None, ""]: try: node_networks = NS.tendrl.objects.NodeNetwork().load_all() cluster.cluster_network = node_networks[0].subnet cluster.save() except etcd.EtcdKeyNotFound as ex: logger.log( "error", NS.publisher_id, {"message": "Failed to sync cluster network details"} ) _sleep = 0 while not self._complete.is_set(): # To detect out of band deletes # refresh gluster object inventory at config['sync_interval'] SYNC_TTL = int(NS.config.data.get("sync_interval", 10)) + 100 NS.node_context = NS.node_context.load() NS.tendrl_context = NS.tendrl_context.load() if _sleep > 5: _sleep = int(NS.config.data.get("sync_interval", 10)) else: _sleep += 1 try: _cluster = NS.tendrl.objects.Cluster( integration_id=NS.tendrl_context.integration_id ).load() if (_cluster.status == "importing" and ( _cluster.current_job['status'] == 'failed')) or \ _cluster.status == "unmanaging" or \ _cluster.status == "set_volume_profiling": time.sleep(_sleep) continue _cnc = NS.tendrl.objects.ClusterNodeContext( node_id=NS.node_context.node_id ).load() _cnc.is_managed = "yes" _cnc.save() subprocess.call( [ 'gluster', 'get-state', 'glusterd', 'odir', '/var/run', 'file', 'glusterd-state', 'detail' ] ) raw_data = ini2json.ini_to_dict( '/var/run/glusterd-state' ) subprocess.call(['rm', '-rf', '/var/run/glusterd-state']) subprocess.call( [ 'gluster', 'get-state', 'glusterd', 'odir', '/var/run', 'file', 'glusterd-state-vol-opts', 'volumeoptions' ] ) raw_data_options = ini2json.ini_to_dict( '/var/run/glusterd-state-vol-opts' ) subprocess.call( [ 'rm', '-rf', '/var/run/glusterd-state-vol-opts' ] ) sync_object = NS.gluster.objects.\ SyncObject(data=json.dumps(raw_data)) sync_object.save() if "Peers" in raw_data: index = 1 peers = raw_data["Peers"] disconnected_hosts = [] while True: try: peer = NS.tendrl.\ objects.GlusterPeer( peer_uuid=peers['peer%s.uuid' % index], hostname=peers[ 'peer%s.primary_hostname' % index ], state=peers['peer%s.state' % index], connected=peers['peer%s.connected' % index] ) try: stored_peer_status = None # find peer detail using hostname ip = socket.gethostbyname( peers['peer%s.primary_hostname' % index] ) node_id = etcd_utils.read( "/indexes/ip/%s" % ip ).value stored_peer = NS.tendrl.objects.GlusterPeer( peer_uuid=peers['peer%s.uuid' % index], node_id=node_id ).load() stored_peer_status = stored_peer.connected current_status = peers[ 'peer%s.connected' % index ] if stored_peer_status and \ current_status != stored_peer_status: msg = ( "Peer %s in cluster %s " "is %s" ) % ( peers[ 'peer%s.primary_hostname' % index ], _cluster.short_name, current_status ) instance = "peer_%s" % peers[ 'peer%s.primary_hostname' % index ] event_utils.emit_event( "peer_status", current_status, msg, instance, 'WARNING' if current_status != 'Connected' else 'INFO' ) # save current status in actual peer # directory also stored_peer.connected = current_status stored_peer.save() # Disconnected host name to # raise brick alert if current_status.lower() == \ "disconnected": disconnected_hosts.append( peers[ 'peer%s.primary_hostname' % index ] ) except etcd.EtcdKeyNotFound: pass SYNC_TTL += 5 peer.save(ttl=SYNC_TTL) index += 1 except KeyError: break # Raise an alert for bricks when peer disconnected # or node goes down for disconnected_host in disconnected_hosts: brick_status_alert( disconnected_host ) if "Volumes" in raw_data: # create devicetree using lsblk devicetree = get_device_tree() # find lvs lvs = brick_utilization.get_lvs() index = 1 volumes = raw_data['Volumes'] total_brick_count = 0 while True: try: b_count = sync_volumes( volumes, index, raw_data_options.get('Volume Options'), SYNC_TTL + VOLUME_TTL, _cluster.short_name, devicetree, lvs ) index += 1 SYNC_TTL += 1 total_brick_count += b_count - 1 except KeyError: global VOLUME_TTL # from second sync volume ttl is # SYNC_TTL + (no.volumes) * 20 + # (no.of.bricks) * 10 + 160 if index > 1: volume_count = index - 1 # When all nodes are down we are updating all # volumes are down, node status TTL is 160, # So make sure volumes are present in etcd # while raising volume down alert VOLUME_TTL = (volume_count * 20) + ( total_brick_count * 10) + 160 break # populate the volume specific options reg_ex = re.compile("^volume[0-9]+.options+") options = {} for key in volumes.keys(): if reg_ex.match(key): options[key] = volumes[key] for key in options.keys(): volname = key.split('.')[0] vol_id = volumes['%s.id' % volname] dict1 = {} for k, v in options.items(): if k.startswith('%s.options' % volname): dict1['.'.join(k.split(".")[2:])] = v options.pop(k, None) volume = NS.tendrl.objects.GlusterVolume( NS.tendrl_context.integration_id, vol_id=vol_id ).load() if volume.options is not None: dest = dict(volume.options) dest.update(dict1) volume.options = dest volume.save() # Sync cluster global details if "provisioner/%s" % NS.tendrl_context.integration_id \ in NS.node_context.tags: all_volumes = NS.tendrl.objects.GlusterVolume( NS.tendrl_context.integration_id ).load_all() or [] volumes = [] for volume in all_volumes: if not str(volume.deleted).lower() == "true" and \ volume.current_job.get('status', '') \ in ['', 'finished', 'failed'] and \ volume.vol_id not in [None, ''] and \ volume.name not in [None, '']: # only for first sync refresh volume TTL # It will increase TTL based on no.of volumes if _cnc.first_sync_done in [None, "no", ""]: etcd_utils.refresh( volume.value, SYNC_TTL + VOLUME_TTL ) volumes.append(volume) cluster_status.sync_cluster_status( volumes, SYNC_TTL + VOLUME_TTL ) utilization.sync_utilization_details(volumes) client_connections.sync_volume_connections(volumes) georep_details.aggregate_session_status() try: evt.process_events() except etcd.EtcdKeyNotFound: pass rebalance_status.sync_volume_rebalance_status(volumes) rebalance_status.sync_volume_rebalance_estimated_time( volumes ) snapshots.sync_volume_snapshots( raw_data['Volumes'], int(NS.config.data.get( "sync_interval", 10 )) + len(volumes) * 4 ) # update alert count update_cluster_alert_count() # check and enable volume profiling if "provisioner/%s" % NS.tendrl_context.integration_id in \ NS.node_context.tags: self._update_volume_profiling() _cluster = NS.tendrl.objects.Cluster( integration_id=NS.tendrl_context.integration_id ).load() if _cluster.exists(): _cluster = _cluster.load() _cluster.last_sync = str(tendrl_now()) # Mark the first sync done flag _cnc = NS.tendrl.objects.ClusterNodeContext( node_id=NS.node_context.node_id ).load() if _cnc.first_sync_done in [None, "no"]: _cnc.first_sync_done = "yes" _cnc.save() if _cluster.current_job.get( 'status', '' ) in ['', 'finished', 'failed'] and \ _cluster.status in [None, ""]: _cluster.save() except Exception as ex: Event( ExceptionMessage( priority="error", publisher=NS.publisher_id, payload={"message": "gluster sds state sync error", "exception": ex } ) ) try: etcd_utils.read( '/clusters/%s/_sync_now' % NS.tendrl_context.integration_id ) continue except etcd.EtcdKeyNotFound: pass time.sleep(_sleep) logger.log( "debug", NS.publisher_id, {"message": "%s complete" % self.__class__.__name__} )
def test_parse_missing_section(self): body = """\nbar=1\nbaz=2"\n[DEFAULT]\na=1i""" filename = self._makeFile('pytest', body) with pytest.raises(ini2json.MissingSectionHeaderError): ini2json.ini_to_dict(filename)
def snapshot_restored(self, event): time.sleep(self.sync_interval) message = event["message"] volume = message['volume_name'] volume_id = "" bricks_to_remove = [] # get the list of current bricks by running get-state output_dir = '/var/run/' output_file = 'glusterd-state-snapshot-%s' % str(uuid.uuid4()) subprocess.call([ 'gluster', 'get-state', 'glusterd', 'odir', output_dir, 'file', output_file, 'detail' ]) raw_data = ini2json.ini_to_dict(output_dir + output_file) subprocess.call(['rm', '-rf', output_dir + output_file]) index = 1 while True: try: current_vol = 'volume%s.name' % index if raw_data['Volumes'][current_vol] == volume: current_vol_id = 'volume%s.id' % index volume_id = raw_data['Volumes'][current_vol_id] break except KeyError: return index += 1 latest_bricks = [] b_index = 1 while True: try: curr_brick = 'volume%s.brick%s.path' % (index, b_index) brick = raw_data['Volumes'][curr_brick] b_index += 1 except KeyError: break latest_bricks.append(brick) # get the list of bricks in etcd for this volume sub_volumes = etcd_utils.read( "/clusters/{0}/Volumes/{1}/Bricks".format( NS.tendrl_context.integration_id, volume_id)) for sub_volume in sub_volumes.leaves: bricks = etcd_utils.read(sub_volume.key) for brick in bricks.leaves: fqdn = brick.key.split('/')[-1].split(':')[0] path = brick.key.split('/')[-1].split(':')[-1][1:] brick_path = "clusters/{0}/Bricks/"\ "all/{1}/{2}".format( NS.tendrl_context.integration_id, fqdn, path ) brick_full_path = etcd_utils.read("%s/brick_path" % brick_path).value if brick_full_path not in latest_bricks: bricks_to_remove.append(brick_full_path) brick_details = {} brick_details["volume"] = volume brick_details["bricks"] = " ".join(bricks_to_remove) event["message"] = brick_details self.volume_remove_brick_force(event)
def Initialize(self, body): self.Manager.subprocess = self self.Manager.ini2json = self filename = self._makeFile('manager', body) self.sections = ini2json.ini_to_dict(filename)
def run(self): # To detect out of band deletes # refresh gluster object inventory at config['sync_interval'] # Default is 260 seconds SYNC_TTL = int(NS.config.data.get("sync_interval", 10)) + 250 Event( Message( priority="info", publisher=NS.publisher_id, payload={"message": "%s running" % self.__class__.__name__})) gluster_brick_dir = NS.gluster.objects.GlusterBrickDir() gluster_brick_dir.save() try: etcd_utils.read("clusters/%s/" "cluster_network" % NS.tendrl_context.integration_id) except etcd.EtcdKeyNotFound: try: node_networks = etcd_utils.read("nodes/%s/Networks" % NS.node_context.node_id) # TODO(team) this logic needs to change later # multiple networks supported for gluster use case node_network = NS.tendrl.objects.NodeNetwork( interface=node_networks.leaves.next().key.split( '/')[-1]).load() cluster = NS.tendrl.objects.Cluster( integration_id=NS.tendrl_context.integration_id).load() cluster.cluster_network = node_network.subnet cluster.save() except etcd.EtcdKeyNotFound as ex: Event( Message(priority="error", publisher=NS.publisher_id, payload={ "message": "Failed to sync cluster network details" })) _sleep = 0 while not self._complete.is_set(): if _sleep > 5: _sleep = int(NS.config.data.get("sync_interval", 10)) else: _sleep += 1 try: _cluster = NS.tendrl.objects.Cluster( integration_id=NS.tendrl_context.integration_id).load() if _cluster.import_status == "failed": continue try: NS._int.wclient.write("clusters/%s/" "sync_status" % NS.tendrl_context.integration_id, "in_progress", prevExist=False) except (etcd.EtcdAlreadyExist, etcd.EtcdCompareFailed) as ex: pass subprocess.call([ 'gluster', 'get-state', 'glusterd', 'odir', '/var/run', 'file', 'glusterd-state', 'detail' ]) raw_data = ini2json.ini_to_dict('/var/run/glusterd-state') subprocess.call(['rm', '-rf', '/var/run/glusterd-state']) subprocess.call([ 'gluster', 'get-state', 'glusterd', 'odir', '/var/run', 'file', 'glusterd-state-vol-opts', 'volumeoptions' ]) raw_data_options = ini2json.ini_to_dict( '/var/run/glusterd-state-vol-opts') subprocess.call( ['rm', '-rf', '/var/run/glusterd-state-vol-opts']) sync_object = NS.gluster.objects.\ SyncObject(data=json.dumps(raw_data)) sync_object.save() if "Peers" in raw_data: index = 1 peers = raw_data["Peers"] while True: try: peer = NS.gluster.\ objects.Peer( peer_uuid=peers['peer%s.uuid' % index], hostname=peers[ 'peer%s.primary_hostname' % index ], state=peers['peer%s.state' % index], connected=peers['peer%s.connected' % index] ) try: stored_peer_status = NS._int.client.read( "clusters/%s/Peers/%s/connected" % (NS.tendrl_context.integration_id, peers['peer%s.uuid' % index])).value current_status = peers['peer%s.connected' % index] if stored_peer_status != "" and \ current_status != stored_peer_status: msg = ( "Status of peer: %s in cluster %s " "changed from %s to %s") % ( peers['peer%s.primary_hostname' % index], NS.tendrl_context.integration_id, stored_peer_status, current_status) instance = "peer_%s" % peers[ 'peer%s.primary_hostname' % index] event_utils.emit_event( "peer_status", current_status, msg, instance, 'WARNING' if current_status != 'Connected' else 'INFO') except etcd.EtcdKeyNotFound: pass peer.save(ttl=SYNC_TTL) index += 1 except KeyError: break if "Volumes" in raw_data: index = 1 volumes = raw_data['Volumes'] while True: try: sync_volumes( volumes, index, raw_data_options.get('Volume Options')) index += 1 except KeyError: break # populate the volume specific options reg_ex = re.compile("^volume[0-9]+.options+") options = {} for key in volumes.keys(): if reg_ex.match(key): options[key] = volumes[key] for key in options.keys(): volname = key.split('.')[0] vol_id = volumes['%s.id' % volname] dict1 = {} for k, v in options.items(): if k.startswith('%s.options' % volname): dict1['.'.join(k.split(".")[2:])] = v options.pop(k, None) NS.gluster.objects.VolumeOptions(vol_id=vol_id, options=dict1).save() # Sync cluster global details if "provisioner/%s" % NS.tendrl_context.integration_id \ in NS.node_context.tags: all_volumes = NS.gluster.objects.Volume().load_all() or [] volumes = [] for volume in all_volumes: if not str(volume.deleted).lower() == "true": volumes.append(volume) cluster_status.sync_cluster_status(volumes) utilization.sync_utilization_details(volumes) client_connections.sync_volume_connections(volumes) georep_details.aggregate_session_status() evt.process_events() rebalance_status.sync_volume_rebalance_status(volumes) rebalance_status.sync_volume_rebalance_estimated_time( volumes) snapshots.sync_volume_snapshots( raw_data['Volumes'], int(NS.config.data.get("sync_interval", 10)) + len(volumes) * 10) _cluster = NS.tendrl.objects.Cluster( integration_id=NS.tendrl_context.integration_id) if _cluster.exists(): _cluster = _cluster.load() _cluster.sync_status = "done" _cluster.last_sync = str(tendrl_now()) _cluster.is_managed = "yes" _cluster.save() # Initialize alert count try: alerts_count_key = '/clusters/%s/alert_counters' % ( NS.tendrl_context.integration_id) etcd_utils.read(alerts_count_key) except (etcd.EtcdException) as ex: if type(ex) == etcd.EtcdKeyNotFound: ClusterAlertCounters( integration_id=NS.tendrl_context.integration_id ).save() # check and enable volume profiling if "provisioner/%s" % NS.tendrl_context.integration_id in \ NS.node_context.tags: self._enable_disable_volume_profiling() except Exception as ex: Event( ExceptionMessage(priority="error", publisher=NS.publisher_id, payload={ "message": "gluster sds state sync error", "exception": ex })) try: etcd_utils.read('/clusters/%s/_sync_now' % NS.tendrl_context.integration_id) continue except etcd.EtcdKeyNotFound: pass time.sleep(_sleep) Event( Message( priority="debug", publisher=NS.publisher_id, payload={"message": "%s complete" % self.__class__.__name__}))
def test_parse_duplicate_section(self): body = """[foo]\na=1\n[foo]\nbar=1\nbaz=2""" filename = self._makeFile('pytest', body) with pytest.raises(ValueError): ini2json.ini_to_dict(filename)
def test_parse_empty_no_leading_whitespace(self): body = """rem ember_last = True""" filename = self._makeFile('pytest', body) sections = ini2json.ini_to_dict(filename) assert sections == {}
def test_parse_empty_with_comment_line(self): body = """# empty with comment line""" filename = self._makeFile('pytest', body) sections = ini2json.ini_to_dict(filename) assert sections == {}
def test_parse_empty_nodefaults(self): filename = self._makeFile('empty', '') sections = ini2json.ini_to_dict(filename) assert sections == {}
def Initialize(self, body): self.Manager.subprocess = self self.Manager.ini2json = self filename = self._makeFile('manager', body) self.sections = ini2json.ini_to_dict(filename)
def _run(self): Event( Message( priority="info", publisher=NS.publisher_id, payload={"message": "%s running" % self.__class__.__name__} ) ) while not self._complete.is_set(): try: # Acquire lock before updating the volume details in etcd # We are blocking till we acquire the lock lock = etcd.Lock(NS.etcd_orm.client, 'volume') lock.acquire(blocking=True,lock_ttl=None) gevent.sleep(3) subprocess.call( [ 'gluster', 'get-state', 'glusterd', 'odir', '/var/run', 'file', 'glusterd-state' ] ) raw_data = ini2json.ini_to_dict('/var/run/glusterd-state') subprocess.call(['rm', '-rf', '/var/run/glusterd-state']) NS.sync_object = NS.gluster.objects.\ SyncObject(data=json.dumps(raw_data)) NS.sync_object.save() if "Peers" in raw_data: index = 1 peers = raw_data["Peers"] while True: try: NS.peer = NS.gluster.\ objects.Peer( peer_uuid=peers['peer%s.uuid' % index], hostname=peers['peer%s.primary_hostname' % index], state=peers['peer%s.state' % index] ) NS.peer.save() index += 1 except KeyError: break if "Volumes" in raw_data: index = 1 volumes = raw_data['Volumes'] while True: try: NS.volume = NS.gluster.objects.Volume( vol_id=volumes[ 'volume%s.id' % index ], vol_type=volumes[ 'volume%s.type' % index ], name=volumes[ 'volume%s.name' % index ], transport_type=volumes[ 'volume%s.transport_type' % index ], status=volumes[ 'volume%s.status' % index ], brick_count=volumes[ 'volume%s.brickcount' % index ], snap_count=volumes[ 'volume%s.snap_count' % index ], stripe_count=volumes[ 'volume%s.stripe_count' % index ], replica_count=volumes[ 'volume%s.replica_count' % index ], subvol_count=volumes[ 'volume%s.subvol_count' % index ], arbiter_count=volumes[ 'volume%s.arbiter_count' % index ], disperse_count=volumes[ 'volume%s.disperse_count' % index ], redundancy_count=volumes[ 'volume%s.redundancy_count' % index ], quorum_status=volumes[ 'volume%s.quorum_status' % index ], snapd_status=volumes[ 'volume%s.snapd_svc.online_status' % index ], snapd_inited=volumes[ 'volume%s.snapd_svc.inited' % index ], rebal_id=volumes[ 'volume%s.rebalance.id' % index ], rebal_status=volumes[ 'volume%s.rebalance.status' % index ], rebal_failures=volumes[ 'volume%s.rebalance.failures' % index ], rebal_skipped=volumes[ 'volume%s.rebalance.skipped' % index ], rebal_lookedup=volumes[ 'volume%s.rebalance.lookedup' % index ], rebal_files=volumes[ 'volume%s.rebalance.files' % index ], rebal_data=volumes[ 'volume%s.rebalance.data' % index ], ) NS.volume.save() b_index = 1 while True: try: NS.brick = NS.gluster\ .objects.Brick( vol_id=volumes['volume%s.id' % index], path=volumes[ 'volume%s.brick%s.path' % ( index, b_index)], hostname=volumes.get( 'volume%s.brick%s.hostname' % ( index, b_index)), port=volumes.get( 'volume%s.brick%s.port' % ( index, b_index)), status=volumes.get( 'volume%s.brick%s.status' % ( index, b_index)), filesystem_type=volumes.get( 'volume%s.brick%s.filesystem_type' % ( index, b_index)), mount_opts=volumes.get( 'volume%s.brick%s.mount_options' % ( index, b_index)) ) NS.brick.save() b_index += 1 except KeyError: break index += 1 except KeyError: break # poplate the volume options reg_ex = re.compile("^volume[0-9]+.options+") options = {} for key in volumes.keys(): if reg_ex.match(key): options[key] = volumes[key] for key in options.keys(): volname = key.split('.')[0] vol_id = volumes['%s.id' % volname] dict1 = {} for k, v in options.items(): if k.startswith('%s.options' % volname): dict1['.'.join(k.split(".")[2:])] = v options.pop(k, None) NS.vol_options = NS.gluster.objects.\ VolumeOptions( vol_id=vol_id, options=dict1 ) NS.vol_options.save() # Sync the cluster status details args = ["gstatus", "-o", "json"] p = subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=open(os.devnull, "r") ) stdout, stderr = p.communicate() if stderr == "" and p.returncode == 0: # The format of the output from gstatus is as below # "2017-02-23 18:13:01.944183 {"brick_count": 2, # bricks_active": 2, "glfs_version": "3.9.0", # node_count": 2, "nodes_active": 2, "over_commit": "No", # product_name": "Community", "raw_capacity": 52181536768, # sh_active": 0, "sh_enabled": 0, "snapshot_count": 0, # status": "healthy", "usable_capacity": 52181536768, # used_capacity": 2117836800, "volume_count": 1, # volume_summary": [{"snapshot_count": 0, "state": "up", # usable_capacity": 52181536768, "used_capacity": 2117836800, # volume_name": "vol1"}]}\n" out_dict = json.loads(stdout[stdout.index('{'): -1]) NS.gluster.objects.GlobalDetails( status=out_dict['status'] ).save() NS.gluster.objects.Utilization( raw_capacity=out_dict['raw_capacity'], usable_capacity=out_dict['usable_capacity'], used_capacity=out_dict['used_capacity'], pcnt_used=(out_dict['used_capacity'] * 100 / out_dict['usable_capacity']) ).save() for item in out_dict['volume_summary']: volumes = NS.etcd_orm.client.read( "clusters/%s/Volumes" % NS.tendrl_context.integration_id ) for child in volumes._children: volume = NS.gluster.objects.Volume( vol_id=child['key'].split('/')[-1] ).load() if volume.name == item['volume_name']: NS.gluster.objects.Volume( vol_id=child['key'].split('/')[-1], usable_capacity=str(item['usable_capacity']), used_capacity=str(item['used_capacity']), pcnt_used=str(item['used_capacity'] * 100 / item['usable_capacity']), vol_type=volume.vol_type, name=volume.name, transport_type=volume.transport_type, status=volume.status, brick_count=volume.brick_count, snap_count=volume.snap_count, stripe_count=volume.stripe_count, replica_count=volume.replica_count, subvol_count=volume.subvol_count, arbiter_count=volume.arbiter_count, disperse_count=volume.disperse_count, redundancy_count=volume.redundancy_count, quorum_status=volume.quorum_status, snapd_status=volume.snapd_status, snapd_inited=volume.snapd_inited, rebal_id=volume.rebal_id, rebal_status=volume.rebal_status, rebal_failures=volume.rebal_failures, rebal_skipped=volume.rebal_skipped, rebal_lookedup=volume.rebal_lookedup, rebal_files=volume.rebal_files, rebal_data=volume.rebal_data ).save() except Exception as ex: Event( ExceptionMessage( priority="error", publisher=NS.publisher_id, payload={"message": "error", "exception": ex } ) ) finally: lock.release() Event( Message( priority="info", publisher=NS.publisher_id, payload={"message": "%s complete" % self.__class__.__name__} ) )
def test_parse_(self): body = """[foo]\na=1\n[memo]\n bar=1\nbaz=2""" filename = self._makeFile('pytest', body) with pytest.raises(ini2json.ParsingError): ini2json.ini_to_dict(filename)
def snapshot_restored(self, event): time.sleep(self.sync_interval) message = event["message"] volume = message['volume_name'] volume_id = "" bricks_to_remove = [] # get the list of current bricks by running get-state output_dir = '/var/run/' output_file = 'glusterd-state-snapshot-%s' % str(uuid.uuid4()) subprocess.call( [ 'gluster', 'get-state', 'glusterd', 'odir', output_dir, 'file', output_file, 'detail' ] ) raw_data = ini2json.ini_to_dict( output_dir + output_file ) subprocess.call(['rm', '-rf', output_dir + output_file]) index = 1 while True: try: current_vol = 'volume%s.name' % index if raw_data['Volumes'][current_vol] == volume: current_vol_id = 'volume%s.id' % index volume_id = raw_data['Volumes'][current_vol_id] break except KeyError: return index += 1 latest_bricks = [] b_index = 1 while True: try: curr_brick = 'volume%s.brick%s.path' % ( index, b_index ) brick = raw_data['Volumes'][curr_brick] b_index += 1 except KeyError: break latest_bricks.append(brick) # get the list of bricks in etcd for this volume sub_volumes = etcd_utils.read( "/clusters/{0}/Volumes/{1}/Bricks".format( NS.tendrl_context.integration_id, volume_id ) ) for sub_volume in sub_volumes.leaves: bricks = etcd_utils.read( sub_volume.key ) for brick in bricks.leaves: fqdn = brick.key.split('/')[-1].split(':')[0] path = brick.key.split('/')[-1].split(':')[-1][1:] brick_path = "clusters/{0}/Bricks/"\ "all/{1}/{2}".format( NS.tendrl_context.integration_id, fqdn, path ) brick_full_path = etcd_utils.read( "%s/brick_path" % brick_path ).value if brick_full_path not in latest_bricks: bricks_to_remove.append(brick_full_path) brick_details = {} brick_details["volume"] = volume brick_details["bricks"] = " ".join(bricks_to_remove) event["message"] = brick_details self.volume_remove_brick_force(event)
def run(self): logger.log( "info", NS.publisher_id, {"message": "%s running" % self.__class__.__name__} ) gluster_brick_dir = NS.gluster.objects.GlusterBrickDir() gluster_brick_dir.save() cluster = NS.tendrl.objects.Cluster( integration_id=NS.tendrl_context.integration_id ).load() if cluster.cluster_network in [None, ""]: try: node_networks = NS.tendrl.objects.NodeNetwork().load_all() cluster.cluster_network = node_networks[0].subnet cluster.save() except etcd.EtcdKeyNotFound as ex: logger.log( "error", NS.publisher_id, {"message": "Failed to sync cluster network details"} ) _sleep = 0 while not self._complete.is_set(): # To detect out of band deletes # refresh gluster object inventory at config['sync_interval'] SYNC_TTL = int(NS.config.data.get("sync_interval", 10)) + 100 NS.node_context = NS.node_context.load() NS.tendrl_context = NS.tendrl_context.load() if _sleep > 5: _sleep = int(NS.config.data.get("sync_interval", 10)) else: _sleep += 1 try: _cluster = NS.tendrl.objects.Cluster( integration_id=NS.tendrl_context.integration_id ).load() if (_cluster.status == "importing" and _cluster.current_job['status'] == 'failed') or \ _cluster.status == "unmanaging" or \ _cluster.status == "set_volume_profiling": continue _cnc = NS.tendrl.objects.ClusterNodeContext( node_id=NS.node_context.node_id ).load() _cnc.is_managed = "yes" _cnc.save() subprocess.call( [ 'gluster', 'get-state', 'glusterd', 'odir', '/var/run', 'file', 'glusterd-state', 'detail' ] ) raw_data = ini2json.ini_to_dict( '/var/run/glusterd-state' ) subprocess.call(['rm', '-rf', '/var/run/glusterd-state']) subprocess.call( [ 'gluster', 'get-state', 'glusterd', 'odir', '/var/run', 'file', 'glusterd-state-vol-opts', 'volumeoptions' ] ) raw_data_options = ini2json.ini_to_dict( '/var/run/glusterd-state-vol-opts' ) subprocess.call( [ 'rm', '-rf', '/var/run/glusterd-state-vol-opts' ] ) sync_object = NS.gluster.objects.\ SyncObject(data=json.dumps(raw_data)) sync_object.save() if "Peers" in raw_data: index = 1 peers = raw_data["Peers"] disconnected_hosts = [] while True: try: peer = NS.tendrl.\ objects.GlusterPeer( peer_uuid=peers['peer%s.uuid' % index], hostname=peers[ 'peer%s.primary_hostname' % index ], state=peers['peer%s.state' % index], connected=peers['peer%s.connected' % index] ) try: stored_peer_status = None # find peer detail using hostname ip = socket.gethostbyname( peers['peer%s.primary_hostname' % index] ) node_id = etcd_utils.read( "/indexes/ip/%s" % ip ).value stored_peer = NS.tendrl.objects.GlusterPeer( peer_uuid=peers['peer%s.uuid' % index], node_id=node_id ).load() stored_peer_status = stored_peer.connected current_status = peers[ 'peer%s.connected' % index ] if stored_peer_status and \ current_status != stored_peer_status: msg = ( "Peer %s in cluster %s " "is %s" ) % ( peers[ 'peer%s.primary_hostname' % index ], _cluster.short_name, current_status ) instance = "peer_%s" % peers[ 'peer%s.primary_hostname' % index ] event_utils.emit_event( "peer_status", current_status, msg, instance, 'WARNING' if current_status != 'Connected' else 'INFO' ) # save current status in actual peer # directory also stored_peer.connected = current_status stored_peer.save() # Disconnected host name to # raise brick alert if current_status.lower() == \ "disconnected": disconnected_hosts.append( peers[ 'peer%s.primary_hostname' % index ] ) except etcd.EtcdKeyNotFound: pass SYNC_TTL += 5 peer.save(ttl=SYNC_TTL) index += 1 except KeyError: break # Raise an alert for bricks when peer disconnected # or node goes down for disconnected_host in disconnected_hosts: brick_status_alert( disconnected_host ) if "Volumes" in raw_data: index = 1 volumes = raw_data['Volumes'] # instantiating blivet class, this will be used for # getting brick_device_details b = blivet.Blivet() # reset blivet during every sync to get latest information # about storage devices in the machine b.reset() devicetree = b.devicetree total_brick_count = 0 while True: try: b_count = sync_volumes( volumes, index, raw_data_options.get('Volume Options'), SYNC_TTL + VOLUME_TTL, _cluster.short_name, devicetree ) index += 1 SYNC_TTL += 1 total_brick_count += b_count - 1 except KeyError: global VOLUME_TTL # from second sync volume ttl is # SYNC_TTL + (no.volumes) * 20 + # (no.of.bricks) * 10 + 160 if index > 1: volume_count = index - 1 # When all nodes are down we are updating all # volumes are down, node status TTL is 160, # So make sure volumes are present in etcd # while raising volume down alert VOLUME_TTL = (volume_count * 20) + ( total_brick_count * 10) + 160 break # populate the volume specific options reg_ex = re.compile("^volume[0-9]+.options+") options = {} for key in volumes.keys(): if reg_ex.match(key): options[key] = volumes[key] for key in options.keys(): volname = key.split('.')[0] vol_id = volumes['%s.id' % volname] dict1 = {} for k, v in options.items(): if k.startswith('%s.options' % volname): dict1['.'.join(k.split(".")[2:])] = v options.pop(k, None) volume = NS.tendrl.objects.GlusterVolume( NS.tendrl_context.integration_id, vol_id=vol_id ).load() if volume.options is not None: dest = dict(volume.options) dest.update(dict1) volume.options = dest volume.save() # Sync cluster global details if "provisioner/%s" % NS.tendrl_context.integration_id \ in NS.node_context.tags: all_volumes = NS.tendrl.objects.GlusterVolume( NS.tendrl_context.integration_id ).load_all() or [] volumes = [] for volume in all_volumes: if not str(volume.deleted).lower() == "true" and \ volume.current_job.get('status', '') \ in ['', 'finished', 'failed'] and \ volume.vol_id not in [None, ''] and \ volume.name not in [None, '']: # only for first sync refresh volume TTL # It will increase TTL based on no.of volumes if _cnc.first_sync_done in [None, "no", ""]: etcd_utils.refresh( volume.value, SYNC_TTL + VOLUME_TTL ) volumes.append(volume) cluster_status.sync_cluster_status( volumes, SYNC_TTL + VOLUME_TTL ) utilization.sync_utilization_details(volumes) client_connections.sync_volume_connections(volumes) georep_details.aggregate_session_status() try: evt.process_events() except etcd.EtcdKeyNotFound: pass rebalance_status.sync_volume_rebalance_status(volumes) rebalance_status.sync_volume_rebalance_estimated_time( volumes ) snapshots.sync_volume_snapshots( raw_data['Volumes'], int(NS.config.data.get( "sync_interval", 10 )) + len(volumes) * 4 ) # update alert count update_cluster_alert_count() # check and enable volume profiling if "provisioner/%s" % NS.tendrl_context.integration_id in \ NS.node_context.tags: self._enable_disable_volume_profiling() _cluster = NS.tendrl.objects.Cluster( integration_id=NS.tendrl_context.integration_id ).load() if _cluster.exists(): _cluster = _cluster.load() _cluster.last_sync = str(tendrl_now()) # Mark the first sync done flag _cnc = NS.tendrl.objects.ClusterNodeContext( node_id=NS.node_context.node_id ).load() if _cnc.first_sync_done in [None, "no"]: _cnc.first_sync_done = "yes" _cnc.save() if _cluster.current_job.get( 'status', '' ) in ['', 'finished', 'failed'] and \ _cluster.status in [None, ""]: _cluster.save() except Exception as ex: Event( ExceptionMessage( priority="error", publisher=NS.publisher_id, payload={"message": "gluster sds state sync error", "exception": ex } ) ) try: etcd_utils.read( '/clusters/%s/_sync_now' % NS.tendrl_context.integration_id ) continue except etcd.EtcdKeyNotFound: pass time.sleep(_sleep) logger.log( "debug", NS.publisher_id, {"message": "%s complete" % self.__class__.__name__} )
def _run(self): LOG.info("%s running" % self.__class__.__name__) while not self._complete.is_set(): try: gevent.sleep(3) subprocess.call( [ 'gluster', 'get-state', 'glusterd', 'odir', '/var/run', 'file', 'glusterd-state' ] ) raw_data = ini2json.ini_to_dict('/var/run/glusterd-state') subprocess.call(['rm', '-rf', '/var/run/glusterd-state']) tendrl_ns.sync_object = tendrl_ns.gluster_integration.objects.\ SyncObject(data=json.dumps(raw_data)) tendrl_ns.sync_object.save() if "Peers" in raw_data: index = 1 peers = raw_data["Peers"] while True: try: tendrl_ns.peer = tendrl_ns.gluster_integration.\ objects.Peer( peer_uuid=peers['peer%s.uuid' % index], hostname=peers['peer%s.primary_hostname' % index], state=peers['peer%s.state' % index] ) tendrl_ns.peer.save() index += 1 except KeyError: break if "Volumes" in raw_data: index = 1 volumes = raw_data['Volumes'] while True: try: tendrl_ns.volume = tendrl_ns.gluster_integration.objects.Volume( vol_id=volumes[ 'volume%s.id' % index ], vol_type=volumes[ 'volume%s.type' % index ], name=volumes[ 'volume%s.name' % index ], transport_type=volumes[ 'volume%s.transport_type' % index ], status=volumes[ 'volume%s.status' % index ], brick_count=volumes[ 'volume%s.brickcount' % index ], snap_count=volumes[ 'volume%s.snap_count' % index ], stripe_count=volumes[ 'volume%s.stripe_count' % index ], replica_count=volumes[ 'volume%s.replica_count' % index ], subvol_count=volumes[ 'volume%s.subvol_count' % index ], arbiter_count=volumes[ 'volume%s.arbiter_count' % index ], disperse_count=volumes[ 'volume%s.disperse_count' % index ], redundancy_count=volumes[ 'volume%s.redundancy_count' % index ], quorum_status=volumes[ 'volume%s.quorum_status' % index ], snapd_status=volumes[ 'volume%s.snapd_svc.online_status' % index ], snapd_inited=volumes[ 'volume%s.snapd_svc.inited' % index ], rebal_id=volumes[ 'volume%s.rebalance.id' % index ], rebal_status=volumes[ 'volume%s.rebalance.status' % index ], rebal_failures=volumes[ 'volume%s.rebalance.failures' % index ], rebal_skipped=volumes[ 'volume%s.rebalance.skipped' % index ], rebal_lookedup=volumes[ 'volume%s.rebalance.lookedup' % index ], rebal_files=volumes[ 'volume%s.rebalance.files' % index ], rebal_data=volumes[ 'volume%s.rebalance.data' % index ], ) tendrl_ns.volume.save() b_index = 1 while True: try: tendrl_ns.brick = tendrl_ns.gluster_integration\ .objects.Brick( vol_id=volumes['volume%s.id' % index], path=volumes[ 'volume%s.brick%s.path' % ( index, b_index)], hostname=volumes.get( 'volume%s.brick%s.hostname' % ( index, b_index)), port=volumes.get( 'volume%s.brick%s.port' % ( index, b_index)), status=volumes.get( 'volume%s.brick%s.status' % ( index, b_index)), filesystem_type=volumes.get( 'volume%s.brick%s.filesystem_type' % ( index, b_index)), mount_opts=volumes.get( 'volume%s.brick%s.mount_options' % ( index, b_index)) ) tendrl_ns.brick.save() b_index += 1 except KeyError: break index += 1 except KeyError: break # poplate the volume options reg_ex = re.compile("^volume[0-9]+.options+") options = {} for key in volumes.keys(): if reg_ex.match(key): options[key] = volumes[key] for key in options.keys(): volname = key.split('.')[0] vol_id = volumes['%s.id' % volname] dict1 = {} for k, v in options.items(): if k.startswith('%s.options' % volname): dict1['.'.join(k.split(".")[2:])] = v options.pop(k, None) tendrl_ns.vol_options = tendrl_ns.gluster_integration.objects.\ VolumeOptions( vol_id=vol_id, options=dict1 ) tendrl_ns.vol_options.save() # Sync the cluster status details args = ["gstatus", "-o", "json"] p = subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=open(os.devnull, "r") ) stdout, stderr = p.communicate() if stderr == "" and p.returncode == 0: # The format of the output from gstatus is as below # "2017-02-23 18:13:01.944183 {"brick_count": 2, # bricks_active": 2, "glfs_version": "3.9.0", # node_count": 2, "nodes_active": 2, "over_commit": "No", # product_name": "Community", "raw_capacity": 52181536768, # sh_active": 0, "sh_enabled": 0, "snapshot_count": 0, # status": "healthy", "usable_capacity": 52181536768, # used_capacity": 2117836800, "volume_count": 1, # volume_summary": [{"snapshot_count": 0, "state": "up", # usable_capacity": 52181536768, "used_capacity": 2117836800, # volume_name": "vol1"}]}\n" out_dict = json.loads(stdout[stdout.index('{'): -1]) tendrl_ns.gluster_integration.objects.GlobalDetails( status=out_dict['status'] ).save() tendrl_ns.gluster_integration.objects.Utilization( raw_capacity=out_dict['raw_capacity'], usable_capacity=out_dict['usable_capacity'], used_capacity=out_dict['used_capacity'], pcnt_used=(out_dict['used_capacity'] * 100 / out_dict['usable_capacity']) ).save() for item in out_dict['volume_summary']: tendrl_ns.gluster_integration.objects.Volume( name=item['volume_name'], usable_capacity=item['usable_capacity'], used_capacity=item['used_capacity'], pcnt_used=(item['used_capacity'] * 100 / item['usable_capacity']) ).save() except Exception as ex: LOG.error(ex) LOG.info("%s complete" % self.__class__.__name__)