def post(self):
"""
Create file replicas at a given RSE.
.. :quickref: Replicas; create replicas at RSE
:<json string rse: The RSE name.
:<json list files: list of dicts with 'scope', 'name', 'bytes', 'meta' and 'adler32'.
:<json bool ignore_availability: Flag to ignore the RSE blacklisting.
:status 201: Replica Successfully created.
:status 400: Invalid Path.
:status 401: Invalid auth token.
:status 404: RSE not found.
:status 409: Replica already exists.
:status 409: DID already exists.
:status 503: Resource Temporary Unavailable.
"""
json_data = request.data
try:
parameters = parse_response(json_data)
except ValueError:
return generate_http_error_flask(
400, 'ValueError', 'Cannot decode json parameter list')
try:
add_replicas(rse=parameters['rse'],
files=parameters['files'],
issuer=request.environ.get('issuer'),
ignore_availability=parameters.get(
'ignore_availability', False))
except InvalidPath, e:
return generate_http_error_flask(400, 'InvalidPath', e.args[0][0])
def cache_add_replicas(rse, files, account, lifetime):
""" Rucio Cache add replicas """
return_code = 0
for file in files:
# check metadata
try:
metadata = get_metadata(file["scope"], file["name"])
except exception.DataIdentifierNotFound:
logging.error("%s:%s not found. Skip to add it to replicas" % (file["scope"], file["name"]))
logging.error(str(format_exc()))
return_code = DID_NOT_FOUND
continue
if int(metadata["bytes"]) != int(file["bytes"]) or metadata["adler32"] != file["adler32"]:
logging.error("%s:%s(bytes:%s, adler32:%s) has different size or checksum with metadata(bytes:%s, adler32:%s). Skip to add it to replicas" % (file["scope"], file["name"], file["bytes"], file["adler32"], metadata["bytes"], metadata["adler32"]))
return_code = META_MISMATCH
continue
# add replica
try:
add_replicas(rse, [file], issuer=account)
except exception.Duplicate:
logging.warn("%s:%s already exists in %s with error details: %s" % (file["scope"], file["name"], rse, str(format_exc())))
return_code = ADD_REPLICA_ERROR
return return_code
def POST(self):
"""
Create file replicas at a given RSE.
HTTP Success:
201 Created
HTTP Error:
401 Unauthorized
409 Conflict
500 Internal Error
"""
json_data = data()
try:
parameters = parse_response(json_data)
except ValueError:
raise generate_http_error(400, 'ValueError',
'Cannot decode json parameter list')
try:
add_replicas(rse=parameters['rse'],
files=parameters['files'],
issuer=ctx.env.get('issuer'),
ignore_availability=parameters.get(
'ignore_availability', False))
except InvalidPath, e:
raise generate_http_error(400, 'InvalidPath', e.args[0][0])
def post(self):
"""
Create file replicas at a given RSE.
.. :quickref: Replicas; create replicas at RSE
:<json string rse: The RSE name.
:<json list files: list of dicts with 'scope', 'name', 'bytes', 'meta' and 'adler32'.
:<json bool ignore_availability: Flag to ignore the RSE blacklisting.
:status 201: Replica Successfully created.
:status 400: Invalid Path.
:status 401: Invalid auth token.
:status 404: RSE not found.
:status 409: Replica already exists.
:status 409: DID already exists.
:status 503: Resource Temporary Unavailable.
"""
json_data = request.data
try:
parameters = parse_response(json_data)
except ValueError:
return generate_http_error_flask(
400, 'ValueError', 'Cannot decode json parameter list')
try:
add_replicas(rse=parameters['rse'],
files=parameters['files'],
issuer=request.environ.get('issuer'),
vo=request.environ.get('vo'),
ignore_availability=parameters.get(
'ignore_availability', False))
except InvalidPath as error:
return generate_http_error_flask(400, 'InvalidPath', error.args[0])
except AccessDenied as error:
return generate_http_error_flask(401, 'AccessDenied',
error.args[0])
except Duplicate as error:
return generate_http_error_flask(409, 'Duplicate', error.args[0])
except DataIdentifierAlreadyExists as error:
return generate_http_error_flask(409,
'DataIdentifierAlreadyExists',
error.args[0])
except RSENotFound as error:
return generate_http_error_flask(404, 'RSENotFound', error.args[0])
except ResourceTemporaryUnavailable as error:
return generate_http_error_flask(503,
'ResourceTemporaryUnavailable',
error.args[0])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
return error, 500
return 'Created', 201
def POST(self):
"""
Create file replicas at a given RSE.
HTTP Success:
201 Created
HTTP Error:
401 Unauthorized
409 Conflict
500 Internal Error
"""
json_data = data()
try:
parameters = parse_response(json_data)
except ValueError:
raise generate_http_error(400, 'ValueError',
'Cannot decode json parameter list')
try:
add_replicas(rse=parameters['rse'],
files=parameters['files'],
issuer=ctx.env.get('issuer'),
vo=ctx.env.get('vo'),
ignore_availability=parameters.get(
'ignore_availability', False))
except InvalidPath as error:
raise generate_http_error(400, 'InvalidPath', error.args[0])
except AccessDenied as error:
raise generate_http_error(401, 'AccessDenied', error.args[0])
except Duplicate as error:
raise generate_http_error(409, 'Duplicate', error.args[0])
except DataIdentifierAlreadyExists as error:
raise generate_http_error(409, 'DataIdentifierAlreadyExists',
error.args[0])
except RSENotFound as error:
raise generate_http_error(404, 'RSENotFound', error.args[0])
except ScopeNotFound as error:
raise generate_http_error(404, 'ScopeNotFound', error.args[0])
except ResourceTemporaryUnavailable as error:
raise generate_http_error(503, 'ResourceTemporaryUnavailable',
error.args[0])
except RucioException as error:
raise generate_http_error(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
raise InternalError(error)
raise Created()
def add_missing_replicas(self, missing):
"""
:param missing: possible missing lfns
:return:
"""
with monitor.record_timer_block('cms_sync.time_add_replica'):
if missing and self.dry_run:
logging.info('Dry run: Adding replicas %s to rse %s.', str(missing), self.rse)
elif missing:
logging.info('Adding %s replicas to rse %s.', len(missing), self.rse)
replicas_to_add = [self.replicas[lfn] for lfn in missing]
files = replica_file_list(replicas=replicas_to_add, scope=self.scope)
for rucio_file in files:
try:
update_file = copy.deepcopy(rucio_file)
update_file.update({'scope': InternalScope(self.scope), "rse_id": self.rse_id, "state": "A"})
update_replicas_states(replicas=[update_file], add_tombstone=False)
except ReplicaNotFound:
resurrect_file = copy.deepcopy(rucio_file)
resurrect_file.update({'scope': 'cms', 'type': 'FILE'})
try:
add_replicas(rse=self.rse, files=[resurrect_file], issuer=self.account,
ignore_availability=True)
except RucioException:
logging.critical('Could not add %s to %s. Constraint violated?', resurrect_file, self.rse)
resurrect_file.update({'scope': 'cms', 'type': 'FILE'}) # Reset to Internal scope by call
resurrect([resurrect_file], issuer=self.account)
resurrect_file.update({'scope': 'cms', 'type': 'FILE'}) # Reset to Internal scope by call
add_replicas(rse=self.rse, files=[resurrect_file], issuer=self.account,
ignore_availability=True)
logging.critical('Resurrected %s at %s', resurrect_file, self.rse)
# add_replicas(rse=self.rse, files=files, issuer=self.account)
lfns = [item['name'] for item in list_files(scope=self.scope, name=self.block_name, long=False)]
missing_lfns = list(set(missing) - set(lfns))
if missing_lfns:
logging.debug('Attaching %s lfns to %s at %s', len(missing_lfns), self.block_name, self.rse)
dids = [{'scope': self.scope, 'name': lfn} for lfn in missing_lfns]
try:
attach_dids(scope=self.scope, name=self.block_name, attachment={'dids': dids},
issuer=self.account)
except FileAlreadyExists:
logging.warning('Trying to attach already existing files to %s', self.block_name)
except DataIdentifierNotFound:
logging.critical('Could not attach to %s at %s. Constraint violated?', self.block_name, self.rse)
return len(missing_lfns)
def post(self):
"""
Create file replicas at a given RSE.
.. :quickref: Replicas; create replicas at RSE
:<json string rse: The RSE name.
:<json list files: list of dicts with 'scope', 'name', 'bytes', 'meta' and 'adler32'.
:<json bool ignore_availability: Flag to ignore the RSE blacklisting.
:status 201: Replica Successfully created.
:status 400: Invalid Path.
:status 401: Invalid auth token.
:status 404: RSE not found.
:status 404: Scope not found.
:status 409: Replica already exists.
:status 409: DID already exists.
:status 503: Resource Temporary Unavailable.
"""
parameters = json_parameters(parse_response)
rse = param_get(parameters, 'rse')
files = param_get(parameters, 'files')
try:
add_replicas(
rse=rse,
files=files,
issuer=request.environ.get('issuer'),
vo=request.environ.get('vo'),
ignore_availability=param_get(parameters,
'ignore_availability',
default=False),
)
except InvalidPath as error:
return generate_http_error_flask(400, error)
except AccessDenied as error:
return generate_http_error_flask(401, error)
except (Duplicate, DataIdentifierAlreadyExists) as error:
return generate_http_error_flask(409, error)
except (RSENotFound, ScopeNotFound) as error:
return generate_http_error_flask(404, error)
except ResourceTemporaryUnavailable as error:
return generate_http_error_flask(503, error)
return 'Created', 201
def add_missing_replicas(self, missing):
"""
:param missing: possible missing lfns
:return:
"""
with monitor.record_timer_block('cms_sync.time_add_replica'):
if missing and self.dry_run:
logging.info('Dry run: Adding replicas %s to rse %s.',
str(missing), self.rse)
elif missing:
logging.debug('Adding %s replicas to rse %s.', len(missing),
self.rse)
replicas_to_add = [self.replicas[lfn] for lfn in missing]
files = replica_file_list(replicas=replicas_to_add,
scope=self.scope)
add_replicas(rse=self.rse, files=files, issuer=self.account)
lfns = [
item['name'] for item in list_files(
scope=self.scope, name=self.block_name, long=False)
]
missing_lfns = list(set(missing) - set(lfns))
if missing_lfns:
logging.debug('Attaching %s lfns to %s at %s',
len(missing_lfns), self.block_name, self.rse)
dids = [{
'scope': self.scope,
'name': lfn
} for lfn in missing_lfns]
try:
attach_dids(scope=self.scope,
name=self.block_name,
attachment={'dids': dids},
issuer=self.account)
except FileAlreadyExists:
logging.warning(
'Trying to attach already existing files to %s',
self.block_name)
return len(missing_lfns)
def POST(self):
"""
Create file replicas at a given RSE.
HTTP Success:
201 Created
HTTP Error:
401 Unauthorized
409 Conflict
500 Internal Error
"""
json_data = data()
try:
parameters = parse_response(json_data)
except ValueError:
raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list')
try:
add_replicas(rse=parameters['rse'], files=parameters['files'], issuer=ctx.env.get('issuer'), ignore_availability=parameters.get('ignore_availability', False))
except AccessDenied, e:
raise generate_http_error(401, 'AccessDenied', e.args[0][0])
def test_api_replica(self):
""" REPLICA (API): Test external representation of replicas """
did = 'ext_' + str(generate_uuid())
pfn = 'srm://mock2.com:8443/srm/managerv2?SFN=/rucio/tmpdisk/rucio_tests/%s/%s' % (self.scope_name, generate_uuid())
add_replicas(self.rse2_name, files=[{'scope': self.scope_name, 'name': did, 'bytes': 100, 'pfn': pfn}], issuer='root', **self.vo)
add_did(self.scope_name, 'ext_parent_2', 'dataset', issuer='root', account=self.account_name, **self.vo)
attachment = {'scope': self.scope_name, 'name': 'ext_parent_2',
'dids': [{'scope': self.scope_name, 'name': did}]}
attach_dids_to_dids([attachment], issuer='root', **self.vo)
out = get_did_from_pfns([pfn], self.rse2_name, **self.vo)
out = list(out)
assert_not_equal(0, len(out))
did_found = False
for p in out:
for key in p:
if p[key]['name'] == did:
did_found = True
assert_equal(self.scope_name, p[key]['scope'])
assert_true(did_found)
out = list_replicas(dids=[{'scope': self.scope_name, 'name': did}], resolve_parents=True, **self.vo)
out = list(out)
assert_not_equal(0, len(out))
parents_found = False
for rep in out:
assert_equal(rep['scope'], self.scope_name)
if 'parents' in rep:
parents_found = True
for parent in rep['parents']:
assert_in(self.scope_name, parent)
if self.multi_vo:
assert_not_in(self.scope.internal, parent)
assert_true(parents_found)
def test_api_replica(self):
""" REPLICA (API): Test external representation of replicas """
did = did_name_generator('file')
did_parent = did_name_generator('dataset')
pfn = 'srm://mock2.com:8443/srm/managerv2?SFN=/rucio/tmpdisk/rucio_tests/%s/%s' % (
self.scope_name, generate_uuid())
add_replicas(self.rse2_name,
files=[{
'scope': self.scope_name,
'name': did,
'bytes': 100,
'pfn': pfn
}],
issuer='root',
**self.vo)
add_did(self.scope_name,
did_parent,
'dataset',
issuer='root',
account=self.account_name,
**self.vo)
attachment = {
'scope': self.scope_name,
'name': did_parent,
'dids': [{
'scope': self.scope_name,
'name': did
}]
}
attach_dids_to_dids([attachment], issuer='root', **self.vo)
out = get_did_from_pfns([pfn], self.rse2_name, **self.vo)
out = list(out)
assert 0 != len(out)
did_found = False
for p in out:
for key in p:
if p[key]['name'] == did:
did_found = True
assert self.scope_name == p[key]['scope']
assert did_found
out = list_replicas(dids=[{
'scope': self.scope_name,
'name': did
}],
resolve_parents=True,
**self.vo)
out = list(out)
assert 0 != len(out)
parents_found = False
for rep in out:
assert rep['scope'] == self.scope_name
if 'parents' in rep:
parents_found = True
for parent in rep['parents']:
assert self.scope_name in parent
if self.multi_vo:
assert self.scope.internal not in parent
assert parents_found
