def POST(self, scope, name):
"""
Append data identifiers to data identifiers.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
500 InternalError
:param scope: Create the data identifier within this scope.
:param name: Create the data identifier with this name.
"""
try:
json_data = loads(data())
except ValueError:
raise generate_http_error(400, 'ValueError',
'Cannot decode json parameter list')
try:
attach_dids(scope=scope,
name=name,
attachment=json_data,
issuer=ctx.env.get('issuer'))
except DataIdentifierNotFound, error:
raise generate_http_error(404, 'DataIdentifierNotFound',
error.args[0][0])
def register_block(self):
"""
Register the dataset (if there is a replica at the pnn) and attach to container
:dry: Dry run. Default false.
"""
# FIXME: The logic here could use some improvement as we try to create a block even if it exists already
try:
get_did(scope=self.scope, name=self.block_name)
self.block_exists = True
monitor.record_counter('cms_sync.dataset_exists')
except DataIdentifierNotFound:
self.block_exists = False
if self.is_at_pnn and self.dry_run:
logging.info('Dry Run: Create dataset %s in scope %s.',
self.block_name, self.scope)
self.block_exists = True
elif self.is_at_pnn:
logging.info('Create block %s in scope %s.', self.block_name,
self.scope)
try:
if not self.block_exists:
add_did(scope=self.scope,
name=self.block_name,
type='DATASET',
issuer=self.account,
lifetime=self.lifetime)
monitor.record_counter('cms_sync.dataset_created')
except DataIdentifierAlreadyExists:
logging.warning('Attempt to add %s:%s failed, already exists.',
self.scope, self.block_name)
monitor.record_counter('cms_sync.dataset_collision')
try:
attach_dids(scope=self.scope,
name=self.container,
attachment={
'dids': [{
'scope': self.scope,
'name': self.block_name
}]
},
issuer=self.account)
except DuplicateContent:
logging.warning(
'Attempt to add %s:%s to %s failed, already exists.',
self.scope, self.block_name, self.container)
except DataIdentifierNotFound:
logging.error(
'Attempt to add %s:%s to %s failed. Container does not exist.',
self.scope, self.block_name, self.container)
return False
self.block_exists = True
else:
logging.warning('Block %s was not at PNN', self.block_name)
return self.block_exists
def post(self, scope_name):
"""
Append data identifiers to data identifiers.
.. :quickref: Attachment; Append DID to DID.
**Example request**:
.. sourcecode:: http
POST /dids/scope1/datasets1/dids HTTP/1.1
Host: rucio.cern.ch
[{"scope": "scope1", "name": "file1"},
{"scope": "scope1", "name": "file2"},
{"scope": "scope1", "name": "file3"}]
**Example response**:
.. sourcecode:: http
HTTP/1.1 201 Created
Vary: Accept
:param scope_name: data identifier (scope)/(name).
:<json list attachments: List of dicts of DIDs to attach.
:status 201: DIDs successfully attached
:status 401: Invalid Auth Token
:status 404: DID not found
:status 409: DIDs already attached
"""
try:
scope, name = parse_scope_name(scope_name,
request.environ.get('vo'))
except ValueError as error:
return generate_http_error_flask(400, error)
attachments = json_parameters()
try:
attach_dids(scope=scope,
name=name,
attachment=attachments,
issuer=request.environ.get('issuer'),
vo=request.environ.get('vo'))
except InvalidPath as error:
return generate_http_error_flask(400, error)
except (DataIdentifierNotFound, RSENotFound) as error:
return generate_http_error_flask(404, error)
except (DuplicateContent, UnsupportedOperation,
FileAlreadyExists) as error:
return generate_http_error_flask(409, error)
except AccessDenied as error:
return generate_http_error_flask(401, error)
return 'Created', 201
def register_and_attach_did(self, scope='cms', name=None, did_type=None, parent_did=None):
existed = False
created = False
attached = False
already_attached = False
try:
get_did(scope=scope, name=name)
existed = True
logging.info('DID existed: %s %s:%s', did_type, scope, name)
except DataIdentifierNotFound:
try:
add_did(scope=scope, name=name, type=did_type,
issuer=self.account, lifetime=self.lifetime)
created = True
logging.info('Created DID: %s %s:%s', did_type, scope, name)
except:
logging.critical('Attempt to add %s:%s failed. Unknown', scope, name)
logging.critical('Reg and attach exiting. Existed %s, created %s, attached %s', existed, created,
attached)
except:
logging.critical('Attempt to get %s:%s failed. Unknown', scope, name)
logging.critical('Reg and attach exiting. Existed %s, created %s, attached %s', existed, created,
attached)
if parent_did:
try:
attach_dids(scope=scope, name=parent_did,
attachment={'dids': [{'scope': scope, 'name': name}]}, issuer=self.account)
logging.info('Attached %s to %s', name, parent_did)
attached = True
except DuplicateContent:
logging.warning('Attempt to add %s:%s to %s failed, already exists.',
self.scope, self.block_name, self.container)
attached = True
already_attached = True
except DataIdentifierNotFound:
logging.critical('Attempt to add %s to %s failed. Parent does not exist.',
self.block_name, self.container)
logging.critical('Reg and attach failed to attach. Existed %s, created %s, attached %s', existed,
created,
attached)
except:
logging.critical('Attempt to attach %s to %s failed. Unknown', name, parent_did)
logging.critical('Reg and attach failed to attach. Existed %s, created %s, attached %s', existed,
created,
attached)
self.block_exists = True
return existed, created, attached, already_attached
def add_missing_replicas(self, missing):
"""
:param missing: possible missing lfns
:return:
"""
with monitor.record_timer_block('cms_sync.time_add_replica'):
if missing and self.dry_run:
logging.info('Dry run: Adding replicas %s to rse %s.', str(missing), self.rse)
elif missing:
logging.info('Adding %s replicas to rse %s.', len(missing), self.rse)
replicas_to_add = [self.replicas[lfn] for lfn in missing]
files = replica_file_list(replicas=replicas_to_add, scope=self.scope)
for rucio_file in files:
try:
update_file = copy.deepcopy(rucio_file)
update_file.update({'scope': InternalScope(self.scope), "rse_id": self.rse_id, "state": "A"})
update_replicas_states(replicas=[update_file], add_tombstone=False)
except ReplicaNotFound:
resurrect_file = copy.deepcopy(rucio_file)
resurrect_file.update({'scope': 'cms', 'type': 'FILE'})
try:
add_replicas(rse=self.rse, files=[resurrect_file], issuer=self.account,
ignore_availability=True)
except RucioException:
logging.critical('Could not add %s to %s. Constraint violated?', resurrect_file, self.rse)
resurrect_file.update({'scope': 'cms', 'type': 'FILE'}) # Reset to Internal scope by call
resurrect([resurrect_file], issuer=self.account)
resurrect_file.update({'scope': 'cms', 'type': 'FILE'}) # Reset to Internal scope by call
add_replicas(rse=self.rse, files=[resurrect_file], issuer=self.account,
ignore_availability=True)
logging.critical('Resurrected %s at %s', resurrect_file, self.rse)
# add_replicas(rse=self.rse, files=files, issuer=self.account)
lfns = [item['name'] for item in list_files(scope=self.scope, name=self.block_name, long=False)]
missing_lfns = list(set(missing) - set(lfns))
if missing_lfns:
logging.debug('Attaching %s lfns to %s at %s', len(missing_lfns), self.block_name, self.rse)
dids = [{'scope': self.scope, 'name': lfn} for lfn in missing_lfns]
try:
attach_dids(scope=self.scope, name=self.block_name, attachment={'dids': dids},
issuer=self.account)
except FileAlreadyExists:
logging.warning('Trying to attach already existing files to %s', self.block_name)
except DataIdentifierNotFound:
logging.critical('Could not attach to %s at %s. Constraint violated?', self.block_name, self.rse)
return len(missing_lfns)
def post(self, scope, name):
"""
Append data identifiers to data identifiers.
.. :quickref: Attachment; Append DID to DID.
**Example request**:
.. sourcecode:: http
POST /dids/scope1/datasets1/dids HTTP/1.1
Host: rucio.cern.ch
[{"scope": "scope1", "name": "file1"},
{"scope": "scope1", "name": "file2"},
{"scope": "scope1", "name": "file3"}]
**Example response**:
.. sourcecode:: http
HTTP/1.1 201 Created
Vary: Accept
:param scope: The scope of the DID to attach to.
:param name: The name of the DID to attach to.
:<json list attachments: List of dicts of DIDs to attach.
:status 201: DIDs successfully attached
:status 401: Invalid Auth Token
:status 404: DID not found
:status 409: DIDs already attached
:status 500: Database Exception
"""
try:
json_data = loads(request.data)
except ValueError:
return generate_http_error_flask(
400, 'ValueError', 'Cannot decode json parameter list')
try:
attach_dids(scope=scope,
name=name,
attachment=json_data,
issuer=request.environ.get('issuer'))
except DataIdentifierNotFound, error:
return generate_http_error_flask(404, 'DataIdentifierNotFound',
error.args[0][0])
def POST(self, scope, name):
"""
Append data identifiers to data identifiers.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
500 InternalError
:param scope: Create the data identifier within this scope.
:param name: Create the data identifier with this name.
"""
try:
json_data = loads(data())
except ValueError:
raise generate_http_error(400, 'ValueError',
'Cannot decode json parameter list')
try:
attach_dids(scope=scope,
name=name,
attachment=json_data,
issuer=ctx.env.get('issuer'),
vo=ctx.env.get('vo'))
except DataIdentifierNotFound as error:
raise generate_http_error(404, 'DataIdentifierNotFound',
error.args[0])
except DuplicateContent as error:
raise generate_http_error(409, 'DuplicateContent', error.args[0])
except AccessDenied as error:
raise generate_http_error(401, 'AccessDenied', error.args[0])
except UnsupportedOperation as error:
raise generate_http_error(409, 'UnsupportedOperation',
error.args[0])
except RSENotFound as error:
raise generate_http_error(404, 'RSENotFound', error.args[0])
except RucioException as error:
raise generate_http_error(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
raise InternalError(error)
raise Created()
def add_missing_replicas(self, missing):
"""
:param missing: possible missing lfns
:return:
"""
with monitor.record_timer_block('cms_sync.time_add_replica'):
if missing and self.dry_run:
logging.info('Dry run: Adding replicas %s to rse %s.',
str(missing), self.rse)
elif missing:
logging.debug('Adding %s replicas to rse %s.', len(missing),
self.rse)
replicas_to_add = [self.replicas[lfn] for lfn in missing]
files = replica_file_list(replicas=replicas_to_add,
scope=self.scope)
add_replicas(rse=self.rse, files=files, issuer=self.account)
lfns = [
item['name'] for item in list_files(
scope=self.scope, name=self.block_name, long=False)
]
missing_lfns = list(set(missing) - set(lfns))
if missing_lfns:
logging.debug('Attaching %s lfns to %s at %s',
len(missing_lfns), self.block_name, self.rse)
dids = [{
'scope': self.scope,
'name': lfn
} for lfn in missing_lfns]
try:
attach_dids(scope=self.scope,
name=self.block_name,
attachment={'dids': dids},
issuer=self.account)
except FileAlreadyExists:
logging.warning(
'Trying to attach already existing files to %s',
self.block_name)
return len(missing_lfns)
