def get(self, scope, name):
""" List all replicas of a data identifier.
.. :quickref: Files; List replicas of DID.
:resheader Content-Type: application/x-json-stream
:param scope: The scope of the data identifier.
:param name: The name of the data identifier.
:query long: Flag to trigger long output
:status 200: DID found
:status 401: Invalid Auth Token
:status 404: DID not found
:status 500: Database Exception
:returns: A dictionary containing all replicas information.
"""
long = False
if "long" in request.args:
long = True
try:
data = ""
for file in list_files(scope=scope, name=name, long=long):
data += dumps(file) + "\n"
return Response(data, content_type="application/x-json-stream")
except DataIdentifierNotFound, error:
return generate_http_error_flask(404, 'DataIdentifierNotFound',
error.args[0][0])
def GET(self, scope, name):
""" List all replicas of a data identifier.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
406 Not Acceptable
500 InternalError
:returns: A dictionary containing all replicas information.
"""
header('Content-Type', 'application/x-json-stream')
long = False
if ctx.query:
params = parse_qs(ctx.query[1:])
if 'long' in params:
long = True
try:
for file in list_files(scope=scope, name=name, long=long):
yield dumps(file) + "\n"
except DataIdentifierNotFound as error:
raise generate_http_error(404, 'DataIdentifierNotFound', error.args[0])
except RucioException as error:
raise generate_http_error(500, error.__class__.__name__, error.args[0])
except Exception as error:
print(format_exc())
raise InternalError(error)
def add_missing_replicas(self, missing):
"""
:param missing: possible missing lfns
:return:
"""
with monitor.record_timer_block('cms_sync.time_add_replica'):
if missing and self.dry_run:
logging.info('Dry run: Adding replicas %s to rse %s.', str(missing), self.rse)
elif missing:
logging.info('Adding %s replicas to rse %s.', len(missing), self.rse)
replicas_to_add = [self.replicas[lfn] for lfn in missing]
files = replica_file_list(replicas=replicas_to_add, scope=self.scope)
for rucio_file in files:
try:
update_file = copy.deepcopy(rucio_file)
update_file.update({'scope': InternalScope(self.scope), "rse_id": self.rse_id, "state": "A"})
update_replicas_states(replicas=[update_file], add_tombstone=False)
except ReplicaNotFound:
resurrect_file = copy.deepcopy(rucio_file)
resurrect_file.update({'scope': 'cms', 'type': 'FILE'})
try:
add_replicas(rse=self.rse, files=[resurrect_file], issuer=self.account,
ignore_availability=True)
except RucioException:
logging.critical('Could not add %s to %s. Constraint violated?', resurrect_file, self.rse)
resurrect_file.update({'scope': 'cms', 'type': 'FILE'}) # Reset to Internal scope by call
resurrect([resurrect_file], issuer=self.account)
resurrect_file.update({'scope': 'cms', 'type': 'FILE'}) # Reset to Internal scope by call
add_replicas(rse=self.rse, files=[resurrect_file], issuer=self.account,
ignore_availability=True)
logging.critical('Resurrected %s at %s', resurrect_file, self.rse)
# add_replicas(rse=self.rse, files=files, issuer=self.account)
lfns = [item['name'] for item in list_files(scope=self.scope, name=self.block_name, long=False)]
missing_lfns = list(set(missing) - set(lfns))
if missing_lfns:
logging.debug('Attaching %s lfns to %s at %s', len(missing_lfns), self.block_name, self.rse)
dids = [{'scope': self.scope, 'name': lfn} for lfn in missing_lfns]
try:
attach_dids(scope=self.scope, name=self.block_name, attachment={'dids': dids},
issuer=self.account)
except FileAlreadyExists:
logging.warning('Trying to attach already existing files to %s', self.block_name)
except DataIdentifierNotFound:
logging.critical('Could not attach to %s at %s. Constraint violated?', self.block_name, self.rse)
return len(missing_lfns)
def add_missing_replicas(self, missing):
"""
:param missing: possible missing lfns
:return:
"""
with monitor.record_timer_block('cms_sync.time_add_replica'):
if missing and self.dry_run:
logging.info('Dry run: Adding replicas %s to rse %s.',
str(missing), self.rse)
elif missing:
logging.debug('Adding %s replicas to rse %s.', len(missing),
self.rse)
replicas_to_add = [self.replicas[lfn] for lfn in missing]
files = replica_file_list(replicas=replicas_to_add,
scope=self.scope)
add_replicas(rse=self.rse, files=files, issuer=self.account)
lfns = [
item['name'] for item in list_files(
scope=self.scope, name=self.block_name, long=False)
]
missing_lfns = list(set(missing) - set(lfns))
if missing_lfns:
logging.debug('Attaching %s lfns to %s at %s',
len(missing_lfns), self.block_name, self.rse)
dids = [{
'scope': self.scope,
'name': lfn
} for lfn in missing_lfns]
try:
attach_dids(scope=self.scope,
name=self.block_name,
attachment={'dids': dids},
issuer=self.account)
except FileAlreadyExists:
logging.warning(
'Trying to attach already existing files to %s',
self.block_name)
return len(missing_lfns)
def generate(vo):
for file in list_files(scope=scope, name=name, long=long, vo=vo):
yield dumps(file) + '\n'
