def register_dataset(self, dry=False): """ Register the dataset (if there is a replica at the pnn) :dry: Dry run. Default false. """ try: self.rcli.get_did(scope=self.scope, name=self.dataset) return 'exists' except DataIdentifierNotFound: pass if self.is_at_pnn and dry: logging.dry('Create dataset %s in scope %s.', self.dataset, self.scope) return 'created' elif self.is_at_pnn: logging.verbose('Create dataset %s in scope %s.', self.dataset, self.scope) self.rcli.add_dataset(scope=self.scope, name=self.dataset, lifetime=self.lifetime) self.rcli.attach_dids(scope=self.scope, name=self.container, dids=[{ 'scope': self.scope, 'name': self.dataset }]) return 'created' return 'skipped'
def register_container(self, dry=False): """ Register container of the dataset (only if there is a dataset replica on the pnn) :dry: Dry run. Default false. """ try: self.rcli.get_did(scope=self.scope, name=self.container) return 'exists' except DataIdentifierNotFound: pass if self.is_at_pnn and dry: logging.dry('Create container %s in scope %s.', self.container, self.scope) return 'created' elif self.is_at_pnn: logging.verbose('Create container %s in scope %s.', self.container, self.scope) try: self.rcli.add_container(scope=self.scope, name=self.container, lifetime=self.lifetime) except DataIdentifierAlreadyExists: logging.warning('Container was created in the meanwhile') return 'exists' return 'created' return 'skipped'
def _get_pnn_workers(workers, summary): """ Gets the finished workers for the pnn worker """ for dataset, work in workers.items(): ret = work.get() logging.verbose("Recovered worker %s, ret %s", dataset, ret) _get_pnn_worker_ret(ret, summary) return summary
def _ping(rcli=None): """ pings the Rucio server """ if rcli is None: rcli = Client() try: rcli.ping() except RucioException as exc: logging.verbose('Ping Exception %s, %s', type(exc).__name__, traceback.format_exc().replace('\n', '~~')) return False return True
def _get_dset_list(pcli, datasets): logging.verbose("Getting datasets list for: %s", datasets) ret = [] wildcard = re.compile(r'\S*[*]\S*') for dset in datasets: ret.extend([ item for item in pcli.list_data_items( pditem=dset, metadata=False, locality=False) if not wildcard.match(item) ]) ret = list(set(ret)) logging.verbose("Got %d datasets", len(ret)) return ret
def update_replicas(self, dry=False): """ Add or removes replicas for the dataset at rse. :dry: Drydrun. default false """ logging.notice('Updating replicas for %s:%s at %s' % (self.scope, self.dataset, self.rse)) replicas = self.rcli.list_replicas([{ 'scope': self.scope, 'name': self.dataset }], rse_expression='rse=%s' % self.rse) rrepl = [repl['name'] for repl in replicas] prepl = [repl for repl in self.replicas.keys()] missing = list(set(prepl) - set(rrepl)) to_remove = list(set(rrepl) - set(prepl)) if missing and dry: logging.dry('Adding replicas %s to rse %s.', str(missing), self.rse) elif missing: logging.verbose('Adding replicas %s to rse %s.', str(missing), self.rse) self.rcli.add_replicas(rse=self.rse, files=[{ 'scope': self.scope, 'name': self.replicas[lfn]['name'], 'adler32': self.replicas[lfn]['checksum'], 'bytes': self.replicas[lfn]['size'], } for lfn in missing]) # missing files that are not in the list of dataset files # are to be attached. lfns = [ item['name'] for item in self.rcli.list_files(scope=self.scope, name=self.dataset) ] missing_lfns = list(set(missing) - set(lfns)) if missing_lfns: logging.verbose('Attaching lfns %s to dataset %s.', str(missing_lfns), self.dataset) try: self.rcli.attach_dids( scope=self.scope, name=self.dataset, dids=[{ 'scope': self.scope, 'name': lfn } for lfn in list(set(missing) - set(lfns))]) except FileAlreadyExists: logging.warning('Trying to attach already existing files.') if to_remove and dry: logging.dry('Removing replicas %s from rse %s.', str(to_remove), self.rse) elif to_remove: logging.verbose('Removing replicas %s from rse %s.', str(to_remove), self.rse) for to_remove_chunk in chunks(to_remove, REMOVE_CHUNK_SIZE): attempt = 0 while True: attempt += 1 try: self.rcli.delete_replicas(rse=self.rse, files=[{ 'scope': self.scope, 'name': lfn, } for lfn in to_remove_chunk ]) break except DatabaseException: logging.warning( 'DatabaseException raised, retrying...') if attempt > 3: raise time.sleep(randint(1, 5)) return {'added': missing, 'removed': to_remove}