def get_blocks_at_pnn(pnn, pcli, multi_das_calls=True): """ Get the list of completed replicas of closed blocks at a site :pnn: the phedex node name :pcli: phedex client instance returns a dictionnary with <block name>: <number of files> """ # This is not optimal in terms of calls and time but # prevents dasgoclient to explode memory footprint if multi_das_calls: blocks_at_pnn = {} logging.notice('Getting blocks with multiple das calls. %s', list(string.letters + string.digits)) for item in list(string.letters + string.digits): for block in pcli.list_data_items(pnn=pnn, pditem='/' + item + '*/*/*'): if block['block'][0]['is_open'] == 'n' and\ block['block'][0]['replica'][0]['complete'] == 'y': blocks_at_pnn[block['block'][0] ['name']] = block['block'][0]['files'] logging.notice('Got blocks for %s', item) return blocks_at_pnn else: # list(string.letters + string.digits) return { item['block'][0]['name']: item['block'][0]['files'] for item in pcli.list_data_items(pnn=pnn) if item['block'][0]['is_open'] == 'n' and\ item['block'][0]['replica'][0]['complete'] == 'y' }
def dataset_replica_update(dataset, pnn, rse, pcli, account, dry): """ Just wrapping the update method. """ try: rcli = Client(account=account) except CannotAuthenticate: logging.warning("cannot authenticate with account %s, skipping pnn %s", account, pnn) return None logging.my_fmt(label='update:rse=%s:rds=%s' % (pnn, dataset)) logging.notice('Starting.') try: ret = _replica_update(dataset, pnn, rse, pcli, rcli, dry) #pylint: disable=broad-except except Exception as exc: logging.error('Exception %s raised: %s', type(exc).__name__, traceback.format_exc().replace('\n', '~~')) return None logging.notice('Finished %s.', ret)
def get_blocks_at_pnn(pnn, pcli, multi_das_calls=True, prefix=None): """ Get the list of completed replicas of closed blocks at a site :pnn: the phedex node name :pcli: phedex client instance returns a dictionnary with <block name>: <number of files> """ # This is not optimal in terms of calls and time but reduces the memory footprint blocks_at_pnn = {} if prefix: logging.summary('Getting subset of blocks at %s beginning with %s' % (pnn, prefix)) with monitor.record_timer_block('cms_sync.pnn_blocks_split'): logging.summary('Getting blocks at %s starting with %s' % (pnn, prefix)) some_blocks_at_pnn = pcli.blocks_at_site(pnn=pnn, prefix=prefix) blocks_at_pnn.update(some_blocks_at_pnn) logging.summary('Got blocks at %s starting with %s' % (pnn, prefix)) elif multi_das_calls: logging.summary('Getting all blocks at %s. Multiple %s' % (pnn, multi_das_calls)) logging.notice('Getting blocks with multiple das calls. %s', list(string.letters + string.digits)) for item in list(string.letters + string.digits): with monitor.record_timer_block('cms_sync.pnn_blocks_split'): logging.summary('Getting blocks at %s starting with %s' % (pnn, item)) some_blocks_at_pnn = pcli.blocks_at_site(pnn=pnn, prefix=item) blocks_at_pnn.update(some_blocks_at_pnn) logging.summary('Got blocks at %s starting with %s' % (pnn, item)) else: logging.summary('Getting all blocks at %s in one call' % pnn) with monitor.record_timer_block('cms_sync.pnn_blocks_all'): blocks_at_pnn = pcli.blocks_at_site(pnn=pnn) logging.summary('Got blocks at %s.' % pnn) return blocks_at_pnn
def sync(config, logs): """ Main Sync process """ logging.my_logfile(logs=logs) logging.my_fmt(label='main_sync') starttime = datetime.now() modify = {} workers = {} # this is the array of running pnns pnns = None # this is the array of pnn to be launched pool = None pcli = PhEDEx() install_mp_handler() conf = _load_config(config, modify, starttime) pnns = [] size = conf['main']['pool'] logging.summary('Starting') while conf['main']['run']: if pool is None: logging.notice('Started pool of size %d', size) pool = multiprocessing.NDPool(size) add = [ pnn for pnn, sec in conf.items() if pnn != 'main' if sec['run'] if pnn not in workers if pnn not in pnns ] pnns += add random.shuffle(pnns) if not _ping(): logging.warning('Cannot ping, not launching workers') else: _launch_workers(pool, workers, pnns, pcli) pnns = [] _poll_workers(workers, pnns) conf = _load_config(config, modify, starttime) if not conf['main']['run'] or\ conf['main']['pool'] != size: # trigger draining of all workers, close the pool and wait # for the task to be over conf = _load_config(config, {'default': {'run': False}}, starttime) _drain_up(workers, pnns) workers = {} pool.close() pool = None size = conf['main']['pool'] else: time.sleep(conf['main']['sleep']) logging.summary('Exiting.') return config
def pnn_sync(pnn, pcli): """ Synchronize one rucio dataset at one rse :pnn: phedex node name. :pcli: phedex client. """ summary = copy.deepcopy(DEFAULT_PNN_SUMMARY) conf = _get_config(pnn) summary['conf'] = conf if 'verbosity' in conf: logging.my_lvl(conf['verbosity']) rcli = Client(account=SYNC_ACCOUNT_FMT % pnn.lower()) if _pnn_abort(pnn, summary, rcli): return summary diff = get_node_diff(pnn, pcli, rcli, conf) summary['timing'].update(diff['timing']) diff = diff['return'] summary['diff'] = diff['summary'] if (diff['summary']['tot'] == diff['summary']['to_remove']) and \ not conf['allow_clean']: logging.warning('All datasets to be removed. Aborting.') summary['status'] = 'aborted' return summary logging.notice("Got diff=%s, timing=%s", summary['diff'], summary['timing']) if _pnn_abort(pnn, summary, rcli): return summary workers = get_timing(_launch_pnn_workers(conf, diff, pnn, pcli, rcli), summary['timing']) summary['workers'] = len(workers) logging.notice("Launched %d workers, pool size %d, timing %s", summary['workers'], int(conf['pool']), summary['timing']['_launch_pnn_workers']) left = int(conf['chunck']) - summary['workers'] + int( conf['min_deletions']) if left > 0: workers_st = get_timing( _launch_pnn_workers_st(left, diff, pnn, pcli, rcli), summary['timing']) summary['workers_st'] = len(workers_st) logging.notice("Launched %d single thread workers, timing %s", summary['workers_st'], summary['timing']['_launch_pnn_workers_st']) workers = dict(workers, **workers_st) _get_pnn_workers(workers, summary) summary['status'] = 'finished' return summary
def update_replicas(self, dry=False): """ Add or removes replicas for the dataset at rse. :dry: Drydrun. default false """ logging.notice('Updating replicas for %s:%s at %s' % (self.scope, self.dataset, self.rse)) replicas = self.rcli.list_replicas([{ 'scope': self.scope, 'name': self.dataset }], rse_expression='rse=%s' % self.rse) rrepl = [repl['name'] for repl in replicas] prepl = [repl for repl in self.replicas.keys()] missing = list(set(prepl) - set(rrepl)) to_remove = list(set(rrepl) - set(prepl)) if missing and dry: logging.dry('Adding replicas %s to rse %s.', str(missing), self.rse) elif missing: logging.verbose('Adding replicas %s to rse %s.', str(missing), self.rse) self.rcli.add_replicas(rse=self.rse, files=[{ 'scope': self.scope, 'name': self.replicas[lfn]['name'], 'adler32': self.replicas[lfn]['checksum'], 'bytes': self.replicas[lfn]['size'], } for lfn in missing]) # missing files that are not in the list of dataset files # are to be attached. lfns = [ item['name'] for item in self.rcli.list_files(scope=self.scope, name=self.dataset) ] missing_lfns = list(set(missing) - set(lfns)) if missing_lfns: logging.verbose('Attaching lfns %s to dataset %s.', str(missing_lfns), self.dataset) try: self.rcli.attach_dids( scope=self.scope, name=self.dataset, dids=[{ 'scope': self.scope, 'name': lfn } for lfn in list(set(missing) - set(lfns))]) except FileAlreadyExists: logging.warning('Trying to attach already existing files.') if to_remove and dry: logging.dry('Removing replicas %s from rse %s.', str(to_remove), self.rse) elif to_remove: logging.verbose('Removing replicas %s from rse %s.', str(to_remove), self.rse) for to_remove_chunk in chunks(to_remove, REMOVE_CHUNK_SIZE): attempt = 0 while True: attempt += 1 try: self.rcli.delete_replicas(rse=self.rse, files=[{ 'scope': self.scope, 'name': lfn, } for lfn in to_remove_chunk ]) break except DatabaseException: logging.warning( 'DatabaseException raised, retrying...') if attempt > 3: raise time.sleep(randint(1, 5)) return {'added': missing, 'removed': to_remove}
def pnn_sync(pnn, pcli): """ Synchronize one rucio dataset at one rse :pnn: phedex node name. :pcli: phedex client. """ monitor.record_counter('cms_sync.site_started') summary = copy.deepcopy(DEFAULT_PNN_SUMMARY) conf = _get_config(pnn) summary['conf'] = conf if 'verbosity' in conf: logging.my_lvl(conf['verbosity']) rcli = Client(account=SYNC_ACCOUNT_FMT % pnn.lower()) if _pnn_abort(pnn, summary, rcli): return summary # Do the loop here? with conf['multi_das'] if conf['multi_das_calls']: prefixes = list(string.letters + string.digits) random.shuffle(prefixes) else: prefixes = [None] for prefix in prefixes: diff = get_node_diff(pnn, pcli, rcli, conf, prefix=prefix) summary['timing'].update(diff['timing']) diff = diff['return'] summary['diff'] = diff['summary'] if (diff['summary']['tot'] == diff['summary']['to_remove']) and not conf['allow_clean']: logging.warning('All datasets to be removed. Aborting.') summary['status'] = 'aborted' continue # return summary logging.notice("Got diff=%s, timing=%s", summary['diff'], summary['timing']) if _pnn_abort(pnn, summary, rcli): return summary workers = get_timing(_launch_pnn_workers(conf, diff, pnn, pcli, rcli), summary['timing']) summary['workers'] = len(workers) logging.notice("Launched %d workers, pool size %d, timing %s", summary['workers'], int(conf['pool']), summary['timing']['_launch_pnn_workers']) left = int(conf['chunck']) - summary['workers'] + int( conf['min_deletions']) if left > 0: workers_st = get_timing( _launch_pnn_workers_st(left, diff, pnn, pcli, rcli), summary['timing']) summary['workers_st'] = len(workers_st) logging.notice("Launched %d single thread workers, timing %s", summary['workers_st'], summary['timing']['_launch_pnn_workers_st']) workers = dict(workers, **workers_st) _get_pnn_workers(workers, summary) monitor.record_counter('cms_sync.site_completed') summary['status'] = 'finished' return summary