def post(self): """ Create file replicas at a given RSE. .. :quickref: Replicas; create replicas at RSE :<json string rse: The RSE name. :<json list files: list of dicts with 'scope', 'name', 'bytes', 'meta' and 'adler32'. :<json bool ignore_availability: Flag to ignore the RSE blacklisting. :status 201: Replica Successfully created. :status 400: Invalid Path. :status 401: Invalid auth token. :status 404: RSE not found. :status 409: Replica already exists. :status 409: DID already exists. :status 503: Resource Temporary Unavailable. """ json_data = request.data try: parameters = parse_response(json_data) except ValueError: return generate_http_error_flask( 400, 'ValueError', 'Cannot decode json parameter list') try: add_replicas(rse=parameters['rse'], files=parameters['files'], issuer=request.environ.get('issuer'), ignore_availability=parameters.get( 'ignore_availability', False)) except InvalidPath, e: return generate_http_error_flask(400, 'InvalidPath', e.args[0][0])
def cache_add_replicas(rse, files, account, lifetime): """ Rucio Cache add replicas """ return_code = 0 for file in files: # check metadata try: metadata = get_metadata(file["scope"], file["name"]) except exception.DataIdentifierNotFound: logging.error("%s:%s not found. Skip to add it to replicas" % (file["scope"], file["name"])) logging.error(str(format_exc())) return_code = DID_NOT_FOUND continue if int(metadata["bytes"]) != int(file["bytes"]) or metadata["adler32"] != file["adler32"]: logging.error("%s:%s(bytes:%s, adler32:%s) has different size or checksum with metadata(bytes:%s, adler32:%s). Skip to add it to replicas" % (file["scope"], file["name"], file["bytes"], file["adler32"], metadata["bytes"], metadata["adler32"])) return_code = META_MISMATCH continue # add replica try: add_replicas(rse, [file], issuer=account) except exception.Duplicate: logging.warn("%s:%s already exists in %s with error details: %s" % (file["scope"], file["name"], rse, str(format_exc()))) return_code = ADD_REPLICA_ERROR return return_code
def POST(self): """ Create file replicas at a given RSE. HTTP Success: 201 Created HTTP Error: 401 Unauthorized 409 Conflict 500 Internal Error """ json_data = data() try: parameters = parse_response(json_data) except ValueError: raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list') try: add_replicas(rse=parameters['rse'], files=parameters['files'], issuer=ctx.env.get('issuer'), ignore_availability=parameters.get( 'ignore_availability', False)) except InvalidPath, e: raise generate_http_error(400, 'InvalidPath', e.args[0][0])
def post(self): """ Create file replicas at a given RSE. .. :quickref: Replicas; create replicas at RSE :<json string rse: The RSE name. :<json list files: list of dicts with 'scope', 'name', 'bytes', 'meta' and 'adler32'. :<json bool ignore_availability: Flag to ignore the RSE blacklisting. :status 201: Replica Successfully created. :status 400: Invalid Path. :status 401: Invalid auth token. :status 404: RSE not found. :status 409: Replica already exists. :status 409: DID already exists. :status 503: Resource Temporary Unavailable. """ json_data = request.data try: parameters = parse_response(json_data) except ValueError: return generate_http_error_flask( 400, 'ValueError', 'Cannot decode json parameter list') try: add_replicas(rse=parameters['rse'], files=parameters['files'], issuer=request.environ.get('issuer'), vo=request.environ.get('vo'), ignore_availability=parameters.get( 'ignore_availability', False)) except InvalidPath as error: return generate_http_error_flask(400, 'InvalidPath', error.args[0]) except AccessDenied as error: return generate_http_error_flask(401, 'AccessDenied', error.args[0]) except Duplicate as error: return generate_http_error_flask(409, 'Duplicate', error.args[0]) except DataIdentifierAlreadyExists as error: return generate_http_error_flask(409, 'DataIdentifierAlreadyExists', error.args[0]) except RSENotFound as error: return generate_http_error_flask(404, 'RSENotFound', error.args[0]) except ResourceTemporaryUnavailable as error: return generate_http_error_flask(503, 'ResourceTemporaryUnavailable', error.args[0]) except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) return error, 500 return 'Created', 201
def POST(self): """ Create file replicas at a given RSE. HTTP Success: 201 Created HTTP Error: 401 Unauthorized 409 Conflict 500 Internal Error """ json_data = data() try: parameters = parse_response(json_data) except ValueError: raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list') try: add_replicas(rse=parameters['rse'], files=parameters['files'], issuer=ctx.env.get('issuer'), vo=ctx.env.get('vo'), ignore_availability=parameters.get( 'ignore_availability', False)) except InvalidPath as error: raise generate_http_error(400, 'InvalidPath', error.args[0]) except AccessDenied as error: raise generate_http_error(401, 'AccessDenied', error.args[0]) except Duplicate as error: raise generate_http_error(409, 'Duplicate', error.args[0]) except DataIdentifierAlreadyExists as error: raise generate_http_error(409, 'DataIdentifierAlreadyExists', error.args[0]) except RSENotFound as error: raise generate_http_error(404, 'RSENotFound', error.args[0]) except ScopeNotFound as error: raise generate_http_error(404, 'ScopeNotFound', error.args[0]) except ResourceTemporaryUnavailable as error: raise generate_http_error(503, 'ResourceTemporaryUnavailable', error.args[0]) except RucioException as error: raise generate_http_error(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) raise InternalError(error) raise Created()
def add_missing_replicas(self, missing): """ :param missing: possible missing lfns :return: """ with monitor.record_timer_block('cms_sync.time_add_replica'): if missing and self.dry_run: logging.info('Dry run: Adding replicas %s to rse %s.', str(missing), self.rse) elif missing: logging.info('Adding %s replicas to rse %s.', len(missing), self.rse) replicas_to_add = [self.replicas[lfn] for lfn in missing] files = replica_file_list(replicas=replicas_to_add, scope=self.scope) for rucio_file in files: try: update_file = copy.deepcopy(rucio_file) update_file.update({'scope': InternalScope(self.scope), "rse_id": self.rse_id, "state": "A"}) update_replicas_states(replicas=[update_file], add_tombstone=False) except ReplicaNotFound: resurrect_file = copy.deepcopy(rucio_file) resurrect_file.update({'scope': 'cms', 'type': 'FILE'}) try: add_replicas(rse=self.rse, files=[resurrect_file], issuer=self.account, ignore_availability=True) except RucioException: logging.critical('Could not add %s to %s. Constraint violated?', resurrect_file, self.rse) resurrect_file.update({'scope': 'cms', 'type': 'FILE'}) # Reset to Internal scope by call resurrect([resurrect_file], issuer=self.account) resurrect_file.update({'scope': 'cms', 'type': 'FILE'}) # Reset to Internal scope by call add_replicas(rse=self.rse, files=[resurrect_file], issuer=self.account, ignore_availability=True) logging.critical('Resurrected %s at %s', resurrect_file, self.rse) # add_replicas(rse=self.rse, files=files, issuer=self.account) lfns = [item['name'] for item in list_files(scope=self.scope, name=self.block_name, long=False)] missing_lfns = list(set(missing) - set(lfns)) if missing_lfns: logging.debug('Attaching %s lfns to %s at %s', len(missing_lfns), self.block_name, self.rse) dids = [{'scope': self.scope, 'name': lfn} for lfn in missing_lfns] try: attach_dids(scope=self.scope, name=self.block_name, attachment={'dids': dids}, issuer=self.account) except FileAlreadyExists: logging.warning('Trying to attach already existing files to %s', self.block_name) except DataIdentifierNotFound: logging.critical('Could not attach to %s at %s. Constraint violated?', self.block_name, self.rse) return len(missing_lfns)
def post(self): """ Create file replicas at a given RSE. .. :quickref: Replicas; create replicas at RSE :<json string rse: The RSE name. :<json list files: list of dicts with 'scope', 'name', 'bytes', 'meta' and 'adler32'. :<json bool ignore_availability: Flag to ignore the RSE blacklisting. :status 201: Replica Successfully created. :status 400: Invalid Path. :status 401: Invalid auth token. :status 404: RSE not found. :status 404: Scope not found. :status 409: Replica already exists. :status 409: DID already exists. :status 503: Resource Temporary Unavailable. """ parameters = json_parameters(parse_response) rse = param_get(parameters, 'rse') files = param_get(parameters, 'files') try: add_replicas( rse=rse, files=files, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'), ignore_availability=param_get(parameters, 'ignore_availability', default=False), ) except InvalidPath as error: return generate_http_error_flask(400, error) except AccessDenied as error: return generate_http_error_flask(401, error) except (Duplicate, DataIdentifierAlreadyExists) as error: return generate_http_error_flask(409, error) except (RSENotFound, ScopeNotFound) as error: return generate_http_error_flask(404, error) except ResourceTemporaryUnavailable as error: return generate_http_error_flask(503, error) return 'Created', 201
def add_missing_replicas(self, missing): """ :param missing: possible missing lfns :return: """ with monitor.record_timer_block('cms_sync.time_add_replica'): if missing and self.dry_run: logging.info('Dry run: Adding replicas %s to rse %s.', str(missing), self.rse) elif missing: logging.debug('Adding %s replicas to rse %s.', len(missing), self.rse) replicas_to_add = [self.replicas[lfn] for lfn in missing] files = replica_file_list(replicas=replicas_to_add, scope=self.scope) add_replicas(rse=self.rse, files=files, issuer=self.account) lfns = [ item['name'] for item in list_files( scope=self.scope, name=self.block_name, long=False) ] missing_lfns = list(set(missing) - set(lfns)) if missing_lfns: logging.debug('Attaching %s lfns to %s at %s', len(missing_lfns), self.block_name, self.rse) dids = [{ 'scope': self.scope, 'name': lfn } for lfn in missing_lfns] try: attach_dids(scope=self.scope, name=self.block_name, attachment={'dids': dids}, issuer=self.account) except FileAlreadyExists: logging.warning( 'Trying to attach already existing files to %s', self.block_name) return len(missing_lfns)
def POST(self): """ Create file replicas at a given RSE. HTTP Success: 201 Created HTTP Error: 401 Unauthorized 409 Conflict 500 Internal Error """ json_data = data() try: parameters = parse_response(json_data) except ValueError: raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list') try: add_replicas(rse=parameters['rse'], files=parameters['files'], issuer=ctx.env.get('issuer'), ignore_availability=parameters.get('ignore_availability', False)) except AccessDenied, e: raise generate_http_error(401, 'AccessDenied', e.args[0][0])
def test_api_replica(self): """ REPLICA (API): Test external representation of replicas """ did = 'ext_' + str(generate_uuid()) pfn = 'srm://mock2.com:8443/srm/managerv2?SFN=/rucio/tmpdisk/rucio_tests/%s/%s' % (self.scope_name, generate_uuid()) add_replicas(self.rse2_name, files=[{'scope': self.scope_name, 'name': did, 'bytes': 100, 'pfn': pfn}], issuer='root', **self.vo) add_did(self.scope_name, 'ext_parent_2', 'dataset', issuer='root', account=self.account_name, **self.vo) attachment = {'scope': self.scope_name, 'name': 'ext_parent_2', 'dids': [{'scope': self.scope_name, 'name': did}]} attach_dids_to_dids([attachment], issuer='root', **self.vo) out = get_did_from_pfns([pfn], self.rse2_name, **self.vo) out = list(out) assert_not_equal(0, len(out)) did_found = False for p in out: for key in p: if p[key]['name'] == did: did_found = True assert_equal(self.scope_name, p[key]['scope']) assert_true(did_found) out = list_replicas(dids=[{'scope': self.scope_name, 'name': did}], resolve_parents=True, **self.vo) out = list(out) assert_not_equal(0, len(out)) parents_found = False for rep in out: assert_equal(rep['scope'], self.scope_name) if 'parents' in rep: parents_found = True for parent in rep['parents']: assert_in(self.scope_name, parent) if self.multi_vo: assert_not_in(self.scope.internal, parent) assert_true(parents_found)
def test_api_replica(self): """ REPLICA (API): Test external representation of replicas """ did = did_name_generator('file') did_parent = did_name_generator('dataset') pfn = 'srm://mock2.com:8443/srm/managerv2?SFN=/rucio/tmpdisk/rucio_tests/%s/%s' % ( self.scope_name, generate_uuid()) add_replicas(self.rse2_name, files=[{ 'scope': self.scope_name, 'name': did, 'bytes': 100, 'pfn': pfn }], issuer='root', **self.vo) add_did(self.scope_name, did_parent, 'dataset', issuer='root', account=self.account_name, **self.vo) attachment = { 'scope': self.scope_name, 'name': did_parent, 'dids': [{ 'scope': self.scope_name, 'name': did }] } attach_dids_to_dids([attachment], issuer='root', **self.vo) out = get_did_from_pfns([pfn], self.rse2_name, **self.vo) out = list(out) assert 0 != len(out) did_found = False for p in out: for key in p: if p[key]['name'] == did: did_found = True assert self.scope_name == p[key]['scope'] assert did_found out = list_replicas(dids=[{ 'scope': self.scope_name, 'name': did }], resolve_parents=True, **self.vo) out = list(out) assert 0 != len(out) parents_found = False for rep in out: assert rep['scope'] == self.scope_name if 'parents' in rep: parents_found = True for parent in rep['parents']: assert self.scope_name in parent if self.multi_vo: assert self.scope.internal not in parent assert parents_found