def register_container(self): self.container_exists = False if self.is_at_pnn and self.dry_run: logging.info('Dry Run: Create container %s in scope %s.', self.container, self.scope) self.container_exists = True return self.container_exists try: get_did(scope=self.scope, name=self.container) monitor.record_counter('cms_sync.container_exists') self.container_exists = True logging.info('Found container %s', self.container) except DataIdentifierNotFound: if self.is_at_pnn: try: logging.info('Create container %s in scope %s.', self.container, self.scope) add_did(scope=self.scope, name=self.container, type='CONTAINER', issuer=self.account, lifetime=self.lifetime) monitor.record_counter('cms_sync.container_created') self.container_exists = True logging.info('Created container %s in scope %s.', self.container, self.scope) except DataIdentifierAlreadyExists: logging.warning('Container was created in the meanwhile') monitor.record_counter('cms_sync.container_collision') self.container_exists = True else: logging.warning('Container was not at PNN') return self.container_exists
def register_block(self): """ Register the dataset (if there is a replica at the pnn) and attach to container :dry: Dry run. Default false. """ # FIXME: The logic here could use some improvement as we try to create a block even if it exists already try: get_did(scope=self.scope, name=self.block_name) self.block_exists = True monitor.record_counter('cms_sync.dataset_exists') except DataIdentifierNotFound: self.block_exists = False if self.is_at_pnn and self.dry_run: logging.info('Dry Run: Create dataset %s in scope %s.', self.block_name, self.scope) self.block_exists = True elif self.is_at_pnn: logging.info('Create block %s in scope %s.', self.block_name, self.scope) try: if not self.block_exists: add_did(scope=self.scope, name=self.block_name, type='DATASET', issuer=self.account, lifetime=self.lifetime) monitor.record_counter('cms_sync.dataset_created') except DataIdentifierAlreadyExists: logging.warning('Attempt to add %s:%s failed, already exists.', self.scope, self.block_name) monitor.record_counter('cms_sync.dataset_collision') try: attach_dids(scope=self.scope, name=self.container, attachment={ 'dids': [{ 'scope': self.scope, 'name': self.block_name }] }, issuer=self.account) except DuplicateContent: logging.warning( 'Attempt to add %s:%s to %s failed, already exists.', self.scope, self.block_name, self.container) except DataIdentifierNotFound: logging.error( 'Attempt to add %s:%s to %s failed. Container does not exist.', self.scope, self.block_name, self.container) return False self.block_exists = True else: logging.warning('Block %s was not at PNN', self.block_name) return self.block_exists
def register_and_attach_did(self, scope='cms', name=None, did_type=None, parent_did=None): existed = False created = False attached = False already_attached = False try: get_did(scope=scope, name=name) existed = True logging.info('DID existed: %s %s:%s', did_type, scope, name) except DataIdentifierNotFound: try: add_did(scope=scope, name=name, type=did_type, issuer=self.account, lifetime=self.lifetime) created = True logging.info('Created DID: %s %s:%s', did_type, scope, name) except: logging.critical('Attempt to add %s:%s failed. Unknown', scope, name) logging.critical('Reg and attach exiting. Existed %s, created %s, attached %s', existed, created, attached) except: logging.critical('Attempt to get %s:%s failed. Unknown', scope, name) logging.critical('Reg and attach exiting. Existed %s, created %s, attached %s', existed, created, attached) if parent_did: try: attach_dids(scope=scope, name=parent_did, attachment={'dids': [{'scope': scope, 'name': name}]}, issuer=self.account) logging.info('Attached %s to %s', name, parent_did) attached = True except DuplicateContent: logging.warning('Attempt to add %s:%s to %s failed, already exists.', self.scope, self.block_name, self.container) attached = True already_attached = True except DataIdentifierNotFound: logging.critical('Attempt to add %s to %s failed. Parent does not exist.', self.block_name, self.container) logging.critical('Reg and attach failed to attach. Existed %s, created %s, attached %s', existed, created, attached) except: logging.critical('Attempt to attach %s to %s failed. Unknown', name, parent_did) logging.critical('Reg and attach failed to attach. Existed %s, created %s, attached %s', existed, created, attached) self.block_exists = True return existed, created, attached, already_attached
def GET(self, scope, name): """ Retrieve a single data identifier. HTTP Success: 200 OK HTTP Error: 401 Unauthorized 406 Not Acceptable 404 Not Found :param scope: The scope name. :param name: The data identifier name. """ header('Content-Type', 'application/json') try: dynamic = False if ctx.query: params = parse_qs(ctx.query[1:]) if 'dynamic' in params: dynamic = True did = get_did(scope=scope, name=name, dynamic=dynamic) return render_json(**did) except ScopeNotFound as error: raise generate_http_error(404, 'ScopeNotFound', error.args[0]) except DataIdentifierNotFound as error: raise generate_http_error(404, 'DataIdentifierNotFound', error.args[0]) except RucioException as error: raise generate_http_error(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) raise InternalError(error)
def GET(self, scope, name): """ Retrieve a single data identifier. HTTP Success: 200 OK HTTP Error: 401 Unauthorized 404 Not Found :param scope: The scope name. :param name: The data identifier name. """ header('Content-Type', 'application/json') try: dynamic = False if ctx.query: params = parse_qs(ctx.query[1:]) if 'dynamic' in params: dynamic = True did = get_did(scope=scope, name=name, dynamic=dynamic) return render_json(**did) except ScopeNotFound, error: raise generate_http_error(404, 'ScopeNotFound', error.args[0][0])
def get(self, scope, name): """ Retrieve a single data identifier. .. :quickref: DIDs; Retrieve a single DID. **Example request**: .. sourcecode:: http GET /dids/scope1/dataset1?dynamic HTTP/1.1 Host: rucio.cern.ch **Example response**: .. sourcecode:: http HTTP/1.1 200 OK Vary: Accept Content-Type: application/json {"scope": "scope1", "did_type": "DATASET", "name": "dataset1", "bytes": 234, "length": 3, "account": "jdoe", "open": True, "monotonic": False, "expired_at": null} :query dynamic: Flag to dynamically calculate size for open DIDs :resheader Content-Type: application/json :status 200: DID found :status 401: Invalid Auth Token :status 404: Scope not found :status 406: Not Acceptable. :returns: Dictionary with DID metadata """ try: dynamic = False if 'dynamic' in request.args: dynamic = True did = get_did(scope=scope, name=name, dynamic=dynamic) return Response(render_json(**did), content_type='application/json') except ScopeNotFound as error: return generate_http_error_flask(404, 'ScopeNotFound', error.args[0]) except DataIdentifierNotFound as error: return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0]) except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) return error, 500
def get(self, scope_name): """ Retrieve a single data identifier. .. :quickref: DIDs; Retrieve a single DID. **Example request**: .. sourcecode:: http GET /dids/scope1/dataset1?dynamic HTTP/1.1 Host: rucio.cern.ch **Example response**: .. sourcecode:: http HTTP/1.1 200 OK Vary: Accept Content-Type: application/json {"scope": "scope1", "did_type": "DATASET", "name": "dataset1", "bytes": 234, "length": 3, "account": "jdoe", "open": True, "monotonic": False, "expired_at": null} :query dynamic: Flag to dynamically calculate size for open DIDs :resheader Content-Type: application/json :status 200: DID found :status 401: Invalid Auth Token :status 404: Scope not found :status 406: Not Acceptable. :returns: Dictionary with DID metadata """ try: scope, name = parse_scope_name(scope_name, request.environ.get('vo')) dynamic = 'dynamic' in request.args did = get_did(scope=scope, name=name, dynamic=dynamic, vo=request.environ.get('vo')) return Response(render_json(**did), content_type='application/json') except ValueError as error: return generate_http_error_flask(400, error) except (ScopeNotFound, DataIdentifierNotFound) as error: return generate_http_error_flask(404, error)