def register_container(self):
self.container_exists = False
if self.is_at_pnn and self.dry_run:
logging.info('Dry Run: Create container %s in scope %s.',
self.container, self.scope)
self.container_exists = True
return self.container_exists
try:
get_did(scope=self.scope, name=self.container)
monitor.record_counter('cms_sync.container_exists')
self.container_exists = True
logging.info('Found container %s', self.container)
except DataIdentifierNotFound:
if self.is_at_pnn:
try:
logging.info('Create container %s in scope %s.',
self.container, self.scope)
add_did(scope=self.scope,
name=self.container,
type='CONTAINER',
issuer=self.account,
lifetime=self.lifetime)
monitor.record_counter('cms_sync.container_created')
self.container_exists = True
logging.info('Created container %s in scope %s.',
self.container, self.scope)
except DataIdentifierAlreadyExists:
logging.warning('Container was created in the meanwhile')
monitor.record_counter('cms_sync.container_collision')
self.container_exists = True
else:
logging.warning('Container was not at PNN')
return self.container_exists
def register_block(self):
"""
Register the dataset (if there is a replica at the pnn) and attach to container
:dry: Dry run. Default false.
"""
# FIXME: The logic here could use some improvement as we try to create a block even if it exists already
try:
get_did(scope=self.scope, name=self.block_name)
self.block_exists = True
monitor.record_counter('cms_sync.dataset_exists')
except DataIdentifierNotFound:
self.block_exists = False
if self.is_at_pnn and self.dry_run:
logging.info('Dry Run: Create dataset %s in scope %s.',
self.block_name, self.scope)
self.block_exists = True
elif self.is_at_pnn:
logging.info('Create block %s in scope %s.', self.block_name,
self.scope)
try:
if not self.block_exists:
add_did(scope=self.scope,
name=self.block_name,
type='DATASET',
issuer=self.account,
lifetime=self.lifetime)
monitor.record_counter('cms_sync.dataset_created')
except DataIdentifierAlreadyExists:
logging.warning('Attempt to add %s:%s failed, already exists.',
self.scope, self.block_name)
monitor.record_counter('cms_sync.dataset_collision')
try:
attach_dids(scope=self.scope,
name=self.container,
attachment={
'dids': [{
'scope': self.scope,
'name': self.block_name
}]
},
issuer=self.account)
except DuplicateContent:
logging.warning(
'Attempt to add %s:%s to %s failed, already exists.',
self.scope, self.block_name, self.container)
except DataIdentifierNotFound:
logging.error(
'Attempt to add %s:%s to %s failed. Container does not exist.',
self.scope, self.block_name, self.container)
return False
self.block_exists = True
else:
logging.warning('Block %s was not at PNN', self.block_name)
return self.block_exists
def register_and_attach_did(self, scope='cms', name=None, did_type=None, parent_did=None):
existed = False
created = False
attached = False
already_attached = False
try:
get_did(scope=scope, name=name)
existed = True
logging.info('DID existed: %s %s:%s', did_type, scope, name)
except DataIdentifierNotFound:
try:
add_did(scope=scope, name=name, type=did_type,
issuer=self.account, lifetime=self.lifetime)
created = True
logging.info('Created DID: %s %s:%s', did_type, scope, name)
except:
logging.critical('Attempt to add %s:%s failed. Unknown', scope, name)
logging.critical('Reg and attach exiting. Existed %s, created %s, attached %s', existed, created,
attached)
except:
logging.critical('Attempt to get %s:%s failed. Unknown', scope, name)
logging.critical('Reg and attach exiting. Existed %s, created %s, attached %s', existed, created,
attached)
if parent_did:
try:
attach_dids(scope=scope, name=parent_did,
attachment={'dids': [{'scope': scope, 'name': name}]}, issuer=self.account)
logging.info('Attached %s to %s', name, parent_did)
attached = True
except DuplicateContent:
logging.warning('Attempt to add %s:%s to %s failed, already exists.',
self.scope, self.block_name, self.container)
attached = True
already_attached = True
except DataIdentifierNotFound:
logging.critical('Attempt to add %s to %s failed. Parent does not exist.',
self.block_name, self.container)
logging.critical('Reg and attach failed to attach. Existed %s, created %s, attached %s', existed,
created,
attached)
except:
logging.critical('Attempt to attach %s to %s failed. Unknown', name, parent_did)
logging.critical('Reg and attach failed to attach. Existed %s, created %s, attached %s', existed,
created,
attached)
self.block_exists = True
return existed, created, attached, already_attached
def GET(self, scope, name):
"""
Retrieve a single data identifier.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
406 Not Acceptable
404 Not Found
:param scope: The scope name.
:param name: The data identifier name.
"""
header('Content-Type', 'application/json')
try:
dynamic = False
if ctx.query:
params = parse_qs(ctx.query[1:])
if 'dynamic' in params:
dynamic = True
did = get_did(scope=scope, name=name, dynamic=dynamic)
return render_json(**did)
except ScopeNotFound as error:
raise generate_http_error(404, 'ScopeNotFound', error.args[0])
except DataIdentifierNotFound as error:
raise generate_http_error(404, 'DataIdentifierNotFound', error.args[0])
except RucioException as error:
raise generate_http_error(500, error.__class__.__name__, error.args[0])
except Exception as error:
print(format_exc())
raise InternalError(error)
def GET(self, scope, name):
"""
Retrieve a single data identifier.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
404 Not Found
:param scope: The scope name.
:param name: The data identifier name.
"""
header('Content-Type', 'application/json')
try:
dynamic = False
if ctx.query:
params = parse_qs(ctx.query[1:])
if 'dynamic' in params:
dynamic = True
did = get_did(scope=scope, name=name, dynamic=dynamic)
return render_json(**did)
except ScopeNotFound, error:
raise generate_http_error(404, 'ScopeNotFound', error.args[0][0])
def get(self, scope, name):
"""
Retrieve a single data identifier.
.. :quickref: DIDs; Retrieve a single DID.
**Example request**:
.. sourcecode:: http
GET /dids/scope1/dataset1?dynamic HTTP/1.1
Host: rucio.cern.ch
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: application/json
{"scope": "scope1", "did_type": "DATASET", "name": "dataset1",
"bytes": 234, "length": 3, "account": "jdoe", "open": True,
"monotonic": False, "expired_at": null}
:query dynamic: Flag to dynamically calculate size for open DIDs
:resheader Content-Type: application/json
:status 200: DID found
:status 401: Invalid Auth Token
:status 404: Scope not found
:status 406: Not Acceptable.
:returns: Dictionary with DID metadata
"""
try:
dynamic = False
if 'dynamic' in request.args:
dynamic = True
did = get_did(scope=scope, name=name, dynamic=dynamic)
return Response(render_json(**did),
content_type='application/json')
except ScopeNotFound as error:
return generate_http_error_flask(404, 'ScopeNotFound',
error.args[0])
except DataIdentifierNotFound as error:
return generate_http_error_flask(404, 'DataIdentifierNotFound',
error.args[0])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
return error, 500
def get(self, scope_name):
"""
Retrieve a single data identifier.
.. :quickref: DIDs; Retrieve a single DID.
**Example request**:
.. sourcecode:: http
GET /dids/scope1/dataset1?dynamic HTTP/1.1
Host: rucio.cern.ch
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: application/json
{"scope": "scope1", "did_type": "DATASET", "name": "dataset1",
"bytes": 234, "length": 3, "account": "jdoe", "open": True,
"monotonic": False, "expired_at": null}
:query dynamic: Flag to dynamically calculate size for open DIDs
:resheader Content-Type: application/json
:status 200: DID found
:status 401: Invalid Auth Token
:status 404: Scope not found
:status 406: Not Acceptable.
:returns: Dictionary with DID metadata
"""
try:
scope, name = parse_scope_name(scope_name,
request.environ.get('vo'))
dynamic = 'dynamic' in request.args
did = get_did(scope=scope,
name=name,
dynamic=dynamic,
vo=request.environ.get('vo'))
return Response(render_json(**did),
content_type='application/json')
except ValueError as error:
return generate_http_error_flask(400, error)
except (ScopeNotFound, DataIdentifierNotFound) as error:
return generate_http_error_flask(404, error)
