def get(self, scope, name): """ List all replicas of a data identifier. .. :quickref: Files; List replicas of DID. :resheader Content-Type: application/x-json-stream :param scope: The scope of the data identifier. :param name: The name of the data identifier. :query long: Flag to trigger long output :status 200: DID found :status 401: Invalid Auth Token :status 404: DID not found :status 500: Database Exception :returns: A dictionary containing all replicas information. """ long = False if "long" in request.args: long = True try: data = "" for file in list_files(scope=scope, name=name, long=long): data += dumps(file) + "\n" return Response(data, content_type="application/x-json-stream") except DataIdentifierNotFound, error: return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0][0])
def GET(self, scope, name): """ List all replicas of a data identifier. HTTP Success: 200 OK HTTP Error: 401 Unauthorized 406 Not Acceptable 500 InternalError :returns: A dictionary containing all replicas information. """ header('Content-Type', 'application/x-json-stream') long = False if ctx.query: params = parse_qs(ctx.query[1:]) if 'long' in params: long = True try: for file in list_files(scope=scope, name=name, long=long): yield dumps(file) + "\n" except DataIdentifierNotFound as error: raise generate_http_error(404, 'DataIdentifierNotFound', error.args[0]) except RucioException as error: raise generate_http_error(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) raise InternalError(error)
def add_missing_replicas(self, missing): """ :param missing: possible missing lfns :return: """ with monitor.record_timer_block('cms_sync.time_add_replica'): if missing and self.dry_run: logging.info('Dry run: Adding replicas %s to rse %s.', str(missing), self.rse) elif missing: logging.info('Adding %s replicas to rse %s.', len(missing), self.rse) replicas_to_add = [self.replicas[lfn] for lfn in missing] files = replica_file_list(replicas=replicas_to_add, scope=self.scope) for rucio_file in files: try: update_file = copy.deepcopy(rucio_file) update_file.update({'scope': InternalScope(self.scope), "rse_id": self.rse_id, "state": "A"}) update_replicas_states(replicas=[update_file], add_tombstone=False) except ReplicaNotFound: resurrect_file = copy.deepcopy(rucio_file) resurrect_file.update({'scope': 'cms', 'type': 'FILE'}) try: add_replicas(rse=self.rse, files=[resurrect_file], issuer=self.account, ignore_availability=True) except RucioException: logging.critical('Could not add %s to %s. Constraint violated?', resurrect_file, self.rse) resurrect_file.update({'scope': 'cms', 'type': 'FILE'}) # Reset to Internal scope by call resurrect([resurrect_file], issuer=self.account) resurrect_file.update({'scope': 'cms', 'type': 'FILE'}) # Reset to Internal scope by call add_replicas(rse=self.rse, files=[resurrect_file], issuer=self.account, ignore_availability=True) logging.critical('Resurrected %s at %s', resurrect_file, self.rse) # add_replicas(rse=self.rse, files=files, issuer=self.account) lfns = [item['name'] for item in list_files(scope=self.scope, name=self.block_name, long=False)] missing_lfns = list(set(missing) - set(lfns)) if missing_lfns: logging.debug('Attaching %s lfns to %s at %s', len(missing_lfns), self.block_name, self.rse) dids = [{'scope': self.scope, 'name': lfn} for lfn in missing_lfns] try: attach_dids(scope=self.scope, name=self.block_name, attachment={'dids': dids}, issuer=self.account) except FileAlreadyExists: logging.warning('Trying to attach already existing files to %s', self.block_name) except DataIdentifierNotFound: logging.critical('Could not attach to %s at %s. Constraint violated?', self.block_name, self.rse) return len(missing_lfns)
def add_missing_replicas(self, missing): """ :param missing: possible missing lfns :return: """ with monitor.record_timer_block('cms_sync.time_add_replica'): if missing and self.dry_run: logging.info('Dry run: Adding replicas %s to rse %s.', str(missing), self.rse) elif missing: logging.debug('Adding %s replicas to rse %s.', len(missing), self.rse) replicas_to_add = [self.replicas[lfn] for lfn in missing] files = replica_file_list(replicas=replicas_to_add, scope=self.scope) add_replicas(rse=self.rse, files=files, issuer=self.account) lfns = [ item['name'] for item in list_files( scope=self.scope, name=self.block_name, long=False) ] missing_lfns = list(set(missing) - set(lfns)) if missing_lfns: logging.debug('Attaching %s lfns to %s at %s', len(missing_lfns), self.block_name, self.rse) dids = [{ 'scope': self.scope, 'name': lfn } for lfn in missing_lfns] try: attach_dids(scope=self.scope, name=self.block_name, attachment={'dids': dids}, issuer=self.account) except FileAlreadyExists: logging.warning( 'Trying to attach already existing files to %s', self.block_name) return len(missing_lfns)
def generate(vo): for file in list_files(scope=scope, name=name, long=long, vo=vo): yield dumps(file) + '\n'