def update_symm_groups_async(self, groups=None): _queue = set() def _get_data_groups(group): return group.meta['couple'] def _get_lrc_groups(group): return group.meta['lrc']['groups'] def _create_groupset_if_needed(groups, group_type, ns_id): for gid in groups: if gid not in storage.groups: logger.info( 'Group {group} is not found, adding fake group ' 'for groupset {groups}'.format( group=gid, groups=groups, )) storage.groups.add(gid) groupset_str = ':'.join((str(gid) for gid in sorted(groups))) if groupset_str not in storage.groupsets: # TODO: somehow check that couple type matches group.type # for all groups in couple (not very easy when metakey read # fails) logger.info( 'Creating groupset {groups}, group type "{group_type}"'. format( groups=groupset_str, group_type=group_type, )) c = storage.groupsets.add( groups=(storage.groups[gid] for gid in groups), group_type=group_type, ) for gid in groups: infrastructure.update_group_history(storage.groups[gid]) if ns_id not in storage.namespaces: logger.info('Creating storage namespace {}'.format(ns_id)) ns = storage.namespaces.add(ns_id) else: ns = storage.namespaces[ns_id] ns.add_couple(c) return storage.groupsets[groupset_str] def _process_group_metadata(response, group, elapsed_time=None, end_time=None): logger.debug( 'Cluster updating: group {0} meta key read time: {1}.{2}'. format(group.group_id, elapsed_time.tsec, elapsed_time.tnsec)) if response.error.code: if response.error.code == errors.ELLIPTICS_NOT_FOUND: # This group is some kind of uncoupled group, not an error group.parse_meta(None) logger.info( 'Group {group} has no metakey'.format(group=group)) elif response.error.code in ( # Route list did not contain the group, expected error errors.ELLIPTICS_GROUP_NOT_IN_ROUTE_LIST, # Timeout in reading metakey from the group, expected error errors.ELLIPTICS_TIMEOUT, ): group.reset_meta() logger.error( 'Error on updating metakey from group {group}: {error}' .format( group=group, error=response.error.message, )) else: raise RuntimeError(response.error.mssage) return meta = response.data group.parse_meta(meta) if group.type == storage.Group.TYPE_UNCOUPLED_LRC_8_2_2_V1: return ns_id = group.meta.get('namespace') if ns_id is None: logger.error( 'Inconsistent meta read from group {group}, missing namespace: {meta}' .format( group=group, meta=group.meta, )) return if group.type == storage.Group.TYPE_DATA: groups = _get_data_groups(group) elif group.type == storage.Group.TYPE_LRC_8_2_2_V1: groups = _get_lrc_groups(group) elif group.type == storage.Group.TYPE_CACHE: groups = _get_data_groups(group) else: raise RuntimeError( 'Group {group_id}, unexpected type to process: {type}'. format( group_id=group.group_id, type=group.type, )) logger.info('Read symmetric groups from group {}: {}'.format( group.group_id, groups)) for gid in groups: if gid != group.group_id: logger.info('Scheduling update for group {}'.format(gid)) _queue.add(gid) groupset = _create_groupset_if_needed(groups, group.type, ns_id) if group.type == storage.Group.TYPE_LRC_8_2_2_V1: # TODO: this will become unnecessary when new "Couple" instance # is introduced data_groups = _get_data_groups(group) data_groupset = _create_groupset_if_needed( data_groups, storage.Group.TYPE_DATA, ns_id) data_groupset.lrc822v1_groupset = groupset # TODO: this should point to a new "Couple" object groupset.couple = data_groupset return try: check_groups = groups or storage.groups.keys() results = {} for group in check_groups: session = self.__session.clone() session.set_exceptions_policy( elliptics.exceptions_policy.no_exceptions) session.set_filter(elliptics.filters.all_with_ack) session.add_groups([group.group_id]) logger.debug('Request to read {0} for group {1}'.format( keys.SYMMETRIC_GROUPS_KEY.replace('\0', '\\0'), group.group_id)) results[group.group_id] = session.read_data( keys.SYMMETRIC_GROUPS_KEY) jobs = {} if self.job_finder: try: params = {'statuses': Job.ACTIVE_STATUSES} if groups: params['groups'] = [g.group_id for g in groups] for job in self.job_finder.jobs(**params): # TODO: this should definitely be done some other way if hasattr(job, 'group'): jobs[job.group] = job except Exception as e: logger.exception( 'Failed to fetch pending jobs: {0}'.format(e)) pass while results: # TODO: Think on queue, it does not work well with lrc couples if _queue: group_id = _queue.pop() if group_id not in results: continue result = results.pop(group_id) else: group_id, result = results.popitem() group = storage.groups[group_id] try: h.process_elliptics_async_result( result, _process_group_metadata, group, raise_on_error=False, ) except Exception as e: logger.exception( 'Critical error on updating metakey from group {}'. format(group_id)) group.parse_meta(None) finally: try: group.set_active_job(jobs.get(group.group_id)) except Exception as e: logger.exception( 'Failed to set group active job: {}'.format(e)) pass try: group.update_status_recursive() except Exception as e: logger.exception( 'Failed to update group {0} status: {1}'.format( group, e)) pass if groups is None: self.update_couple_settings() load_manager.update(storage) weight_manager.update(storage) infrastructure.schedule_history_update() except Exception as e: logger.exception('Critical error during symmetric group update')
def update_symm_groups_async(self, groups=None): _queue = set() def _process_group_metadata(response, group, elapsed_time=None, end_time=None): logger.debug('Cluster updating: group {0} meta key read time: {1}.{2}'.format( group.group_id, elapsed_time.tsec, elapsed_time.tnsec)) meta = response.data group.parse_meta(meta) couple = group.meta.get('couple') if couple is None: logger.error('Read symmetric groups from group {} (no couple data): {}'.format( group.group_id, group.meta)) return logger.info('Read symmetric groups from group {}: {}'.format(group.group_id, couple)) for gid in couple: if gid != group.group_id: logger.info('Scheduling update for group {}'.format(gid)) _queue.add(gid) couple_str = ':'.join((str(gid) for gid in sorted(couple))) logger.debug('{0} in storage.couples: {1}'.format( couple_str, couple_str in storage.couples)) if couple_str not in storage.couples and couple_str not in storage.cache_couples: ns_id = group.meta.get('namespace') if ns_id is None: logger.error('Inconsistent meta read from group {}, ' 'missing namespace: {}'.format(group, group.meta)) return if group.type == storage.Group.TYPE_DATA: logger.info('Creating couple {0}'.format(couple_str)) for gid in couple: if gid not in storage.groups: logger.info('Group {} is not found adding fake group for ' 'couple {}'.format(gid, couple)) storage.groups.add(gid) c = storage.couples.add(storage.groups[gid] for gid in couple) logger.info('Created couple {0} {1}'.format(c, repr(c))) elif group.type == storage.Group.TYPE_CACHE: logger.info('Creating cache couple {0}'.format(couple_str)) c = storage.cache_couples.add(storage.groups[gid] for gid in couple) logger.info('Created cache couple {0} {1}'.format(c, repr(c))) else: raise ValueError('Unknown group type for group {}: {}'.format( group, group.type)) for gid in couple: infrastructure.update_group_history(storage.groups[gid]) if ns_id not in storage.namespaces: logger.info('Creating storage namespace {}'.format(ns_id)) ns = storage.namespaces.add(ns_id) else: ns = storage.namespaces[ns_id] ns.add_couple(c) return try: check_groups = groups or storage.groups.keys() results = {} for group in check_groups: session = self.__session.clone() session.add_groups([group.group_id]) logger.debug('Request to read {0} for group {1}'.format( keys.SYMMETRIC_GROUPS_KEY.replace('\0', '\\0'), group.group_id)) results[group.group_id] = session.read_data(keys.SYMMETRIC_GROUPS_KEY) jobs = {} if self.job_finder: try: params = {'statuses': Job.ACTIVE_STATUSES} if groups: params['groups'] = [g.group_id for g in groups] for job in self.job_finder.jobs(**params): jobs[job.group] = job except Exception as e: logger.exception('Failed to fetch pending jobs: {0}'.format(e)) pass while results: if _queue: group_id = _queue.pop() if group_id not in results: continue result = results.pop(group_id) else: group_id, result = results.popitem() group = storage.groups[group_id] try: h.process_elliptics_async_result(result, _process_group_metadata, group) except elliptics.NotFoundError as e: logger.warn('Failed to read symmetric_groups from group {0}: {1}'.format( group_id, e)) group.parse_meta(None) except Exception as e: logger.exception('Failed to read symmetric_groups from group {0}: {1}'.format( group_id, e)) group.parse_meta(None) finally: try: group.set_active_job(jobs.get(group.group_id)) except Exception as e: logger.exception('Failed to set group active job: {}'.format(e)) pass try: group.update_status_recursive() except Exception as e: logger.exception('Failed to update group {0} status: {1}'.format(group, e)) pass if groups is None: load_manager.update(storage) weight_manager.update(storage) infrastructure.schedule_history_update() except Exception as e: logger.exception('Critical error during symmetric group update')
def update_symm_groups_async(self, groups=None): _queue = set() def _process_group_metadata(response, group, elapsed_time=None, end_time=None): logger.debug( 'Cluster updating: group {0} meta key read time: {1}.{2}'. format(group.group_id, elapsed_time.tsec, elapsed_time.tnsec)) meta = response.data group.parse_meta(meta) couple = group.meta.get('couple') if couple is None: logger.error( 'Read symmetric groups from group {} (no couple data): {}'. format(group.group_id, meta)) return logger.info('Read symmetric groups from group {}: {}'.format( group.group_id, couple)) for gid in couple: if gid != group.group_id: logger.info('Scheduling update for group {}'.format(gid)) _queue.add(gid) couple_str = ':'.join((str(gid) for gid in sorted(couple))) logger.debug('{0} in storage.couples: {1}'.format( couple_str, couple_str in storage.couples)) if couple_str not in storage.couples and couple_str not in storage.cache_couples: ns_id = group.meta.get('namespace') if ns_id is None: logger.error('Inconsistent meta read from group {}, ' 'missing namespace: {}'.format( group, group.meta)) return if group.type == storage.Group.TYPE_DATA: logger.info('Creating couple {0}'.format(couple_str)) for gid in couple: if gid not in storage.groups: logger.info( 'Group {} is not found adding fake group for ' 'couple {}'.format(gid, couple)) storage.groups.add(gid) c = storage.couples.add( [storage.groups[gid] for gid in couple]) logger.info('Created couple {0} {1}'.format(c, repr(c))) elif group.type == storage.Group.TYPE_CACHE: logger.info('Creating cache couple {0}'.format(couple_str)) c = storage.cache_couples.add( [storage.groups[gid] for gid in couple]) logger.info('Created cache couple {0} {1}'.format( c, repr(c))) else: raise ValueError( 'Unknown group type for group {}: {}'.format( group, group.type)) for gid in couple: infrastructure.update_group_history(storage.groups[gid]) if ns_id not in storage.namespaces: logger.info('Creating storage namespace {}'.format(ns_id)) ns = storage.namespaces.add(ns_id) else: ns = storage.namespaces[ns_id] ns.add_couple(c) return try: check_groups = groups or storage.groups.keys() results = {} for group in check_groups: session = self.__session.clone() session.add_groups([group.group_id]) logger.debug('Request to read {0} for group {1}'.format( keys.SYMMETRIC_GROUPS_KEY.replace('\0', '\\0'), group.group_id)) results[group.group_id] = session.read_data( keys.SYMMETRIC_GROUPS_KEY) jobs = {} if self.job_finder: try: params = {'statuses': Job.ACTIVE_STATUSES} if groups: params['groups'] = [g.group_id for g in groups] for job in self.job_finder.jobs(**params): jobs[job.group] = job except Exception as e: logger.exception( 'Failed to fetch pending jobs: {0}'.format(e)) pass while results: if _queue: group_id = _queue.pop() if group_id not in results: continue result = results.pop(group_id) else: group_id, result = results.popitem() group = storage.groups[group_id] try: h.process_elliptics_async_result(result, _process_group_metadata, group) except elliptics.NotFoundError as e: logger.warn( 'Failed to read symmetric_groups from group {0}: {1}'. format(group_id, e)) group.parse_meta(None) except Exception as e: logger.exception( 'Failed to read symmetric_groups from group {0}: {1}'. format(group_id, e)) group.parse_meta(None) finally: try: group.set_active_job(jobs.get(group.group_id)) except Exception as e: logger.exception( 'Failed to set group active job: {}'.format(e)) pass try: group.update_status_recursive() except Exception as e: logger.exception( 'Failed to update group {0} status: {1}'.format( group, e)) pass load_manager.update(storage) weight_manager.update(storage) infrastructure.schedule_history_update() except Exception as e: logger.exception('Critical error during symmetric group update')
def update_symm_groups_async(self, groups=None): _queue = set() def _get_data_groups(group): return group.meta['couple'] def _get_lrc_groups(group): return group.meta['lrc']['groups'] def _create_groupset_if_needed(groups, group_type, ns_id): for gid in groups: if gid not in storage.groups: logger.info( 'Group {group} is not found, adding fake group ' 'for groupset {groups}'.format( group=gid, groups=groups, ) ) storage.groups.add(gid) groupset_str = ':'.join((str(gid) for gid in sorted(groups))) if groupset_str not in storage.groupsets: # TODO: somehow check that couple type matches group.type # for all groups in couple (not very easy when metakey read # fails) logger.info('Creating groupset {groups}, group type "{group_type}"'.format( groups=groupset_str, group_type=group_type, )) c = storage.groupsets.add( groups=(storage.groups[gid] for gid in groups), group_type=group_type, ) for gid in groups: infrastructure.update_group_history(storage.groups[gid]) if ns_id not in storage.namespaces: logger.info('Creating storage namespace {}'.format(ns_id)) ns = storage.namespaces.add(ns_id) else: ns = storage.namespaces[ns_id] ns.add_couple(c) return storage.groupsets[groupset_str] def _process_group_metadata(response, group, elapsed_time=None, end_time=None): logger.debug('Cluster updating: group {0} meta key read time: {1}.{2}'.format( group.group_id, elapsed_time.tsec, elapsed_time.tnsec)) if response.error.code: if response.error.code == errors.ELLIPTICS_NOT_FOUND: # This group is some kind of uncoupled group, not an error group.parse_meta(None) logger.info( 'Group {group} has no metakey'.format(group=group) ) elif response.error.code in ( # Route list did not contain the group, expected error errors.ELLIPTICS_GROUP_NOT_IN_ROUTE_LIST, # Timeout in reading metakey from the group, expected error errors.ELLIPTICS_TIMEOUT, ): group.reset_meta() logger.error( 'Error on updating metakey from group {group}: {error}'.format( group=group, error=response.error.message, ) ) else: raise RuntimeError(response.error.mssage) return meta = response.data group.parse_meta(meta) if group.type == storage.Group.TYPE_UNCOUPLED_LRC_8_2_2_V1: return ns_id = group.meta.get('namespace') if ns_id is None: logger.error( 'Inconsistent meta read from group {group}, missing namespace: {meta}'.format( group=group, meta=group.meta, ) ) return if group.type == storage.Group.TYPE_DATA: groups = _get_data_groups(group) elif group.type == storage.Group.TYPE_LRC_8_2_2_V1: groups = _get_lrc_groups(group) elif group.type == storage.Group.TYPE_CACHE: groups = _get_data_groups(group) else: raise RuntimeError( 'Group {group_id}, unexpected type to process: {type}'.format( group_id=group.group_id, type=group.type, ) ) logger.info('Read symmetric groups from group {}: {}'.format(group.group_id, groups)) for gid in groups: if gid != group.group_id: logger.info('Scheduling update for group {}'.format(gid)) _queue.add(gid) groupset = _create_groupset_if_needed(groups, group.type, ns_id) if group.type == storage.Group.TYPE_LRC_8_2_2_V1: # TODO: this will become unnecessary when new "Couple" instance # is introduced data_groups = _get_data_groups(group) data_groupset = _create_groupset_if_needed( data_groups, storage.Group.TYPE_DATA, ns_id ) data_groupset.lrc822v1_groupset = groupset # TODO: this should point to a new "Couple" object groupset.couple = data_groupset return try: check_groups = groups or storage.groups.keys() results = {} for group in check_groups: session = self.__session.clone() session.set_exceptions_policy(elliptics.exceptions_policy.no_exceptions) session.set_filter(elliptics.filters.all_with_ack) session.add_groups([group.group_id]) logger.debug('Request to read {0} for group {1}'.format( keys.SYMMETRIC_GROUPS_KEY.replace('\0', '\\0'), group.group_id)) results[group.group_id] = session.read_data(keys.SYMMETRIC_GROUPS_KEY) jobs = {} if self.job_finder: try: params = {'statuses': Job.ACTIVE_STATUSES} if groups: params['groups'] = [g.group_id for g in groups] for job in self.job_finder.jobs(**params): # TODO: this should definitely be done some other way if hasattr(job, 'group'): jobs[job.group] = job except Exception as e: logger.exception('Failed to fetch pending jobs: {0}'.format(e)) pass while results: # TODO: Think on queue, it does not work well with lrc couples if _queue: group_id = _queue.pop() if group_id not in results: continue result = results.pop(group_id) else: group_id, result = results.popitem() group = storage.groups[group_id] try: h.process_elliptics_async_result( result, _process_group_metadata, group, raise_on_error=False, ) except Exception as e: logger.exception( 'Critical error on updating metakey from group {}'.format(group_id) ) group.parse_meta(None) finally: try: group.set_active_job(jobs.get(group.group_id)) except Exception as e: logger.exception('Failed to set group active job: {}'.format(e)) pass try: group.update_status_recursive() except Exception as e: logger.exception('Failed to update group {0} status: {1}'.format(group, e)) pass if groups is None: self.update_couple_settings() load_manager.update(storage) weight_manager.update(storage) infrastructure.schedule_history_update() except Exception as e: logger.exception('Critical error during symmetric group update')