def get(self, scope): """ List all data identifiers in a scope which match a given metadata. .. :quickref: Search; Search DIDs in a scope with given metadata. **Example request**: .. sourcecode:: http GET /dids/scope1/dids/search?type=collection&long=True&length.lt=10 HTTP/1.1 Host: rucio.cern.ch **Example response**: .. sourcecode:: http HTTP/1.1 200 OK Vary: Accept Content-Type: application/x-json-stream {"scope": "scope1", "did_type": "CONTAINER", "name": "container1", "bytes": 1234, "length": 1} {"scope": "scope1", "did_type": "DATASET", "name": "dataset1", "bytes": 234, "length": 3} :query type: specify a DID type to search for :query long: set to True for long output, otherwise only name :query recursive: set to True to recursively list DIDs content :query created_before: Date string in RFC-1123 format where the creation date was earlier :query created_after: Date string in RFC-1123 format where the creation date was later :query length: Exact number of attached DIDs :query length.gt: Number of attached DIDs greater than :query length.lt: Number of attached DIDs less than :query length.gte: Number of attached DIDs greater than or equal to :query length.lte: Number of attached DIDs less than or equal to :query name: Name or pattern of a DID name :resheader Content-Type: application/x-json-stream :status 200: DIDs found :status 401: Invalid Auth Token :status 404: Invalid key in filters :status 409: Wrong DID type :returns: Line separated name of DIDs or dictionaries of DIDs for long option """ filters = {} long = False recursive = False type = 'collection' for k, v in request.args.items(): if k == 'type': type = v elif k == 'long': long = v == '1' elif k == 'recursive': recursive = v == 'True' else: filters[k] = v try: data = "" for did in list_dids(scope=scope, filters=filters, type=type, long=long, recursive=recursive): data += dumps(did) + '\n' return Response(data, content_type='application/x-json-stream') except UnsupportedOperation as error: return generate_http_error_flask(409, 'UnsupportedOperation', error.args[0]) except KeyNotFound as error: return generate_http_error_flask(404, 'KeyNotFound', error.args[0]) except Exception as error: print(format_exc()) return error, 500
def post(self): """ Create a new replication rule. .. :quickref: AllRule; create new rule :<json list dids: List of data identifiers. :<json string account: Account issuing the rule. :<json int copies: The number of replicas. :<json string rse_expression: RSE expression which gets resolved into a list of RSEs. :<json string grouping: ALL - All files will be replicated to the same RSE. DATASET - All files in the same dataset will be replicated to the same RSE. NONE - Files will be completely spread over all allowed RSEs without any grouping considerations at all :<json int weight: Weighting scheme to be used. :<json int lifetime: The lifetime of the replication rule in seconds. :<json string locked: If the is locked. :<json string subscription_id: The subscription_id, if the rule is created by a subscription. :<json string source_replica_expression: Only use replicas as source from these RSEs. :<json string activity: Activity to be passed to the conveyor. :<json string notify: Notification setting of the rule ('Y', 'N', 'C'; None = 'N'). :<json bool purge_replicas: Purge setting if a replica should be directly deleted after the rule is deleted. :<json bool ignore_availability: Option to ignore the availability of RSEs. :<json string comments: Comment about the rule. :<json bool ask_approval: Ask for approval for this rule. :<json bool asynchronous: Create replication rule asynchronously by the judge-injector. :<json int priority: Priority of the rule and the transfers which should be submitted. :<json bool split_container: Should a container rule be split into individual dataset rules. :<json string meta: Dictionary with metadata from the WFMS. :status 201: rule created :status 401: Invalid Auth Token :status 404: DID not found :status 409: Invalid Replication Rule :status 409: Duplicate Replication Rule :status 409: Insufficient Target RSEs :status 409: Insufficient Account Limit :status 409: Invalid RSE Expression :status 409: Replication Rule Creation Temporary Failed :status 409: Invalid Rule Weight :status 409: Staging Area Rule Requires Lifetime :status 409: Scratch Disk Lifetime Conflict :status 409: Manual Rule Approval Blocked :status 409: Invalid Object :returns: List of ids for created rules """ json_data = request.data try: grouping, weight, lifetime, locked, subscription_id, source_replica_expression, activity, notify,\ purge_replicas, ignore_availability, comment, ask_approval, asynchronous, priority,\ split_container, meta = 'DATASET', None, None, False, None, None, None, None, False, False, None,\ False, False, 3, False, None params = loads(json_data) dids = params['dids'] account = params['account'] copies = params['copies'] rse_expression = params['rse_expression'] if 'grouping' in params: grouping = params['grouping'] if 'weight' in params: weight = params['weight'] if 'lifetime' in params: lifetime = params['lifetime'] if 'locked' in params: locked = params['locked'] if 'subscription_id' in params: subscription_id = params['subscription_id'] if 'source_replica_expression' in params: source_replica_expression = params['source_replica_expression'] if 'activity' in params: activity = params['activity'] if 'notify' in params: notify = params['notify'] if 'purge_replicas' in params: purge_replicas = params['purge_replicas'] if 'ignore_availability' in params: ignore_availability = params['ignore_availability'] if 'comment' in params: comment = params['comment'] if 'ask_approval' in params: ask_approval = params['ask_approval'] if 'asynchronous' in params: asynchronous = params['asynchronous'] if 'priority' in params: priority = params['priority'] if 'split_container' in params: split_container = params['split_container'] if 'meta' in params: meta = params['meta'] except ValueError: return generate_http_error_flask(400, 'ValueError', 'Cannot decode json parameter list') try: rule_ids = add_replication_rule(dids=dids, copies=copies, rse_expression=rse_expression, weight=weight, lifetime=lifetime, grouping=grouping, account=account, locked=locked, subscription_id=subscription_id, source_replica_expression=source_replica_expression, activity=activity, notify=notify, purge_replicas=purge_replicas, ignore_availability=ignore_availability, comment=comment, ask_approval=ask_approval, asynchronous=asynchronous, priority=priority, split_container=split_container, meta=meta, issuer=request.environ.get('issuer'), vo=request.environ.get('vo')) # TODO: Add all other error cases here except InvalidReplicationRule as error: return generate_http_error_flask(409, 'InvalidReplicationRule', error.args[0]) except DuplicateRule as error: return generate_http_error_flask(409, 'DuplicateRule', error.args[0]) except InsufficientTargetRSEs as error: return generate_http_error_flask(409, 'InsufficientTargetRSEs', error.args[0]) except InsufficientAccountLimit as error: return generate_http_error_flask(409, 'InsufficientAccountLimit', error.args[0]) except InvalidRSEExpression as error: return generate_http_error_flask(409, 'InvalidRSEExpression', error.args[0]) except DataIdentifierNotFound as error: return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0]) except ReplicationRuleCreationTemporaryFailed as error: return generate_http_error_flask(409, 'ReplicationRuleCreationTemporaryFailed', error.args[0]) except InvalidRuleWeight as error: return generate_http_error_flask(409, 'InvalidRuleWeight', error.args[0]) except StagingAreaRuleRequiresLifetime as error: return generate_http_error_flask(409, 'StagingAreaRuleRequiresLifetime', error.args[0]) except ScratchDiskLifetimeConflict as error: return generate_http_error_flask(409, 'ScratchDiskLifetimeConflict', error.args[0]) except ManualRuleApprovalBlocked as error: return generate_http_error_flask(409, 'ManualRuleApprovalBlocked', error.args[0]) except InvalidObject as error: return generate_http_error_flask(409, 'InvalidObject', error.args[0]) except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: print(error) print(format_exc()) return error, 500 return Response(dumps(rule_ids), status=201)
def get(self, scope, name): """ Metalink redirect .. :quickref: MetaLinkRedirector; Metalink redirect. HTTP Success: 200 OK HTTP Error: 401 Unauthorized 500 InternalError 404 Notfound :param scope: The scope name of the file. :param name: The name of the file. :resheader Content-Type: application/metalink4+xml'. :status 200: OK. :status 401: Invalid Auth Token. :status 404: RSE Not Found. :status 404: DID Not Found. :status 406: Not Acceptable. :status 500: Internal Error. :returns: Metalink file """ dids = [{'scope': scope, 'name': name}] # set the correct client IP client_ip = request.environ.get('HTTP_X_FORWARDED_FOR') if client_ip is None: client_ip = request.remote_addr client_location = {'ip': client_ip, 'fqdn': None, 'site': None} schemes = request.args.get( 'schemes', ['http', 'https', 'root', 'gsiftp', 'srm', 'davs']) select = request.args.get('select', None) if 'sort' in request.args: select = request.args['sort'] client_location['ip'] = request.args.get('ip', None) client_location['fqdn'] = request.args.get('fqdn', None) client_location['site'] = request.args.get('site', None) try: tmp_replicas = [ rep for rep in list_replicas(dids=dids, schemes=schemes, client_location=client_location) ] if not tmp_replicas: return 'no redirection possible - cannot find the DID', 404 # first, set the appropriate content type, and stream the header data = '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n' # iteratively stream the XML per file for rfile in tmp_replicas: replicas = [] dictreplica = {} for rse in rfile['rses']: for replica in rfile['rses'][rse]: replicas.append(replica) dictreplica[replica] = rse # stream metadata data += ' <file name="' + rfile['name'] + '">\n' data += ' <identity>' + rfile['scope'] + ':' + rfile[ 'name'] + '</identity>\n' if rfile['adler32'] is not None: data += ' <hash type="adler32">' + rfile[ 'adler32'] + '</hash>\n' if rfile['md5'] is not None: data += ' <hash type="md5">' + rfile['md5'] + '</hash>\n' data += ' <size>' + str(rfile['bytes']) + '</size>\n' data += ' <glfn name="/atlas/rucio/%s:%s">' % (rfile['scope'], rfile['name']) data += '</glfn>\n' # sort the actual replicas if necessary if select == 'geoip': replicas = sort_geoip(dictreplica, client_location['ip'], ignore_error=True) elif select == 'closeness': replicas = sort_closeness(dictreplica, client_location) elif select == 'dynamic': replicas = sort_dynamic(dictreplica, client_location) elif select == 'ranking': replicas = sort_ranking(dictreplica, client_location) else: replicas = sort_random(dictreplica) # stream URLs idx = 1 for replica in replicas: data += ' <url location="' + str( dictreplica[replica]) + '" priority="' + str( idx) + '">' + replica + '</url>\n' idx += 1 data += ' </file>\n' # don't forget to send the metalink footer data += '</metalink>\n' return Response(data, content_type='application/metalink4+xml') except DataIdentifierNotFound as error: return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0]) except ReplicaNotFound as error: return generate_http_error_flask(404, 'ReplicaNotFound', error.args[0]) except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) return error, 500
def get(self, scope, name): """ List all replicas for data identifiers. .. :quickref: Replicas; List all replicas for did HTTP Success: 200 OK HTTP Error: 401 Unauthorized 500 InternalError :reqheader HTTP_ACCEPT: application/metalink4+xml :param scope: data identifier scope. :param name: data identifier name. :resheader Content-Type: application/x-json-stream :resheader Content-Type: application/metalink4+xml :status 200: OK. :status 401: Invalid auth token. :status 404: DID not found. :status 500: Internal Error. :returns: A dictionary containing all replicas information. :returns: A metalink description of replicas if metalink(4)+xml is specified in Accept: """ metalink = False if request.environ.get('HTTP_ACCEPT') is not None: tmp = request.environ.get('HTTP_ACCEPT').split(',') if 'application/metalink4+xml' in tmp: metalink = True dids, schemes, select, limit = [{ 'scope': scope, 'name': name }], None, None, None schemes = request.args.get('schemes', None) select = request.args.get('select', None) limit = request.args.get('limit', None) if limit: limit = int(limit) data = "" content_type = 'application/x-json-stream' try: # first, set the appropriate content type, and stream the header if metalink: content_type = 'application/metalink4+xml' data += '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n' # then, stream the replica information for rfile in list_replicas(dids=dids, schemes=schemes): client_ip = request.environ.get('HTTP_X_FORWARDED_FOR') if client_ip is None: client_ip = request.remote_addr replicas = [] dictreplica = {} for rse in rfile['rses']: for replica in rfile['rses'][rse]: replicas.append(replica) dictreplica[replica] = rse if select == 'geoip': try: replicas = sort_geoip(dictreplica, client_ip) except AddressNotFoundError: pass else: replicas = sort_random(dictreplica) if not metalink: data += dumps(rfile) + '\n' else: data += ' <file name="' + rfile['name'] + '">\n' data += ' <identity>' + rfile['scope'] + ':' + rfile[ 'name'] + '</identity>\n' if rfile['adler32'] is not None: data += ' <hash type="adler32">' + rfile[ 'adler32'] + '</hash>\n' if rfile['md5'] is not None: data += ' <hash type="md5">' + rfile[ 'md5'] + '</hash>\n' data += ' <size>' + str(rfile['bytes']) + '</size>\n' data += ' <glfn name="/atlas/rucio/%s:%s">' % ( rfile['scope'], rfile['name']) data += '</glfn>\n' idx = 0 for replica in replicas: data += ' <url location="' + str( dictreplica[replica]) + '" priority="' + str( idx + 1) + '">' + replica + '</url>\n' idx += 1 if limit and limit == idx: break data += ' </file>\n' # don't forget to send the metalink footer if metalink: data += '</metalink>\n' return Response(data, content_type=content_type) except DataIdentifierNotFound as error: return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0]) except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: print format_exc() return error, 500
def get(self): """ List requests for a given source and destination RSE or site. .. :quickref: RequestsGet; list requests :reqheader Content-Type: application/x-json-stream :status 200: Request found. :status 404: Request not found. :status 406: Not Acceptable. """ src_rse = f_request.get('src_rse') dst_rse = f_request.get('dst_rse') src_site = f_request.get('src_site') dst_site = f_request.get('dst_site') request_states = f_request.get('request_states') if not request_states: return generate_http_error_flask(400, 'MissingParameter', 'Request state is missing') if src_rse and not dst_rse: return generate_http_error_flask(400, 'MissingParameter', 'Destination RSE is missing') elif dst_rse and not src_rse: return generate_http_error_flask(400, 'MissingParameter', 'Source RSE is missing') elif src_site and not dst_site: return generate_http_error_flask(400, 'MissingParameter', 'Destination site is missing') elif dst_site and not src_site: return generate_http_error_flask(400, 'MissingParameter', 'Source site is missing') try: states = [ RequestState.from_string(state) for state in request_states.split(',') ] except ValueError: return generate_http_error_flask(400, 'Invalid', 'Request state value is invalid') src_rses = [] dst_rses = [] if src_site: src_rses = get_rses_with_attribute_value( key='site', value=src_site, lookup_key='site', vo=f_request.environ.get('vo')) if not src_rses: return generate_http_error_flask( 404, 'NotFound', 'Could not resolve site name %s to RSE' % src_site) src_rses = [get_rse_name(rse['rse_id']) for rse in src_rses] dst_rses = get_rses_with_attribute_value( key='site', value=dst_site, lookup_key='site', vo=f_request.environ.get('vo')) if not dst_rses: return generate_http_error_flask( 404, 'NotFound', 'Could not resolve site name %s to RSE' % dst_site) dst_rses = [get_rse_name(rse['rse_id']) for rse in dst_rses] else: dst_rses = [dst_rse] src_rses = [src_rse] results = [] for result in request.list_requests( src_rses, dst_rses, states, issuer=f_request.environ.get('issuer'), vo=f_request.environ.get('vo')): del result['_sa_instance_state'] results.append(result) return json.dumps(results, cls=APIEncoder)
def get(self): """ Sign a URL for a limited lifetime for a particular service. :reqheader X-Rucio-VO: VO name as a string (Multi-VO only). :reqheader X-Rucio-Account: Account identifier as a string. :reqheader X-Rucio-AppID: Application identifier as a string. :resheader Access-Control-Allow-Origin: :resheader Access-Control-Allow-Headers: :resheader Access-Control-Allow-Methods: :resheader Access-Control-Allow-Credentials: :resheader Access-Control-Expose-Headers: :resheader X-Rucio-Auth-Token: The authentication token :status 200: Successfully signed URL :status 400: Bad Request :status 401: Unauthorized :status 406: Not Acceptable :status 500: Internal Server Error """ response = Response() response.headers['Access-Control-Allow-Origin'] = request.environ.get( 'HTTP_ORIGIN') response.headers['Access-Control-Allow-Headers'] = request.environ.get( 'HTTP_ACCESS_CONTROL_REQUEST_HEADERS') response.headers['Access-Control-Allow-Methods'] = '*' response.headers['Access-Control-Allow-Credentials'] = 'true' response.headers[ 'Access-Control-Expose-Headers'] = 'X-Rucio-Auth-Token' response.headers['Content-Type'] = 'application/octet-stream' response.headers[ 'Cache-Control'] = 'no-cache, no-store, max-age=0, must-revalidate' response.headers['Cache-Control'] = 'post-check=0, pre-check=0' response.headers['Pragma'] = 'no-cache' vo = request.environ.get('HTTP_X_RUCIO_VO', 'def') account = request.environ.get('HTTP_X_RUCIO_ACCOUNT') appid = request.environ.get('HTTP_X_RUCIO_APPID') if appid is None: appid = 'unknown' ip = request.environ.get('HTTP_X_FORWARDED_FOR') if ip is None: ip = request.remote_addr try: validate_auth_token(request.environ.get('HTTP_X_RUCIO_AUTH_TOKEN')) except AccessDenied: return generate_http_error_flask( 401, 'CannotAuthenticate', 'Cannot authenticate to account %(account)s with given credentials' % locals()) except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) return error, 500 rse, svc, operation, url = None, None, None, None try: params = parse_qs(request.query[1:]) lifetime = params.get('lifetime', [600])[0] service = params.get('svc', ['gcs'])[0] operation = params.get('op', ['read'])[0] url = params.get('url', [None])[0] rse = params.get('rse', [None])[0] except ValueError: return generate_http_error_flask( 400, 'ValueError', 'Cannot decode json parameter list') if service not in ['gcs', 's3', 'swift']: return generate_http_error_flask( 400, 'ValueError', 'Parameter "svc" must be either empty(=gcs), gcs, s3 or swift') if url is None: return generate_http_error_flask(400, 'ValueError', 'Parameter "url" not found') if rse is None: return generate_http_error_flask(400, 'ValueError', 'Parameter "rse" not found') if operation not in ['read', 'write', 'delete']: return generate_http_error_flask( 400, 'ValueError', 'Parameter "op" must be either empty(=read), read, write, or delete.' ) try: result = get_signed_url(account, appid, ip, rse=rse, service=service, operation=operation, url=url, lifetime=lifetime, vo=vo) except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) return error, 500 if not result: return generate_http_error_flask( 401, 'CannotAuthenticate', 'Cannot generate signed URL for account %(account)s' % locals()) return response
def put(self, rse, scheme, hostname=None, port=None): """ Updates attributes of an existing protocol entry. Because protocol identifier, hostname, and port are used as unique identifier they are immutable. .. :quickref: Protocol; Update RSE protocol. :param rse: The RSE name. :param scheme: The protocol identifier. :param hostname: The hostname defined for the scheme, used if more than one scheme is registered with the same identifier. :param port: The port registered for the hostname, ued if more than one scheme is registered with the same identifier and hostname. :<json dict paramaters: parameter of the new protocol entry. :status 201: Created. :status 400: Cannot decode json parameter dictionary. :status 401: Invalid Auth Token. :status 404: RSE not found. :status 404: RSE Protocol Not Supported. :status 404: RSE Protocol Domain Not Supported. :status 409: RSE Protocol Priority Error. :status 500: Internal Error. """ json_data = request.data try: parameter = loads(json_data) except ValueError: return generate_http_error_flask( 400, 'ValueError', 'Cannot decode json parameter dictionary') try: update_protocols(rse, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'), scheme=scheme, hostname=hostname, port=port, data=parameter) except InvalidObject as error: return generate_http_error_flask(400, 'InvalidObject', error.args[0]) except RSEProtocolNotSupported as error: return generate_http_error_flask(404, 'RSEProtocolNotSupported', error.args[0]) except RSENotFound as error: return generate_http_error_flask(404, 'RSENotFound', error.args[0]) except RSEProtocolDomainNotSupported as error: return generate_http_error_flask(404, 'RSEProtocolDomainNotSupported', error.args[0]) except RSEProtocolPriorityError as error: return generate_http_error_flask(409, 'RSEProtocolPriorityError', error.args[0]) except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: print(error) print(format_exc()) return error, 500 return "OK", 200
def get(self): """ Authenticate a Rucio account temporarily via SSH key exchange. .. :quickref: SSH; Authenticate with SSH key exchange. :reqheader Rucio-Account: Account identifier as a string. :reqheader Rucio-SSH-Signature: Response to server challenge signed with SSH private key as a base64 encoded string. :reqheader Rucio-AppID: Application identifier as a string. :resheader Access-Control-Allow-Origin: :resheader Access-Control-Allow-Headers: :resheader Access-Control-Allow-Methods: :resheader Access-Control-Allow-Credentials: :resheader Access-Control-Expose-Headers: :resheader X-Rucio-Auth-Token: The authentication token :status 200: Successfully authenticated :status 404: Invalid credentials """ response = Response() response.headers['Access-Control-Allow-Origin'] = request.environ.get( 'HTTP_ORIGIN') response.headers['Access-Control-Allow-Headers'] = request.environ.get( 'HTTP_ACCESS_CONTROL_REQUEST_HEADERS') response.headers['Access-Control-Allow-Methods'] = '*' response.headers['Access-Control-Allow-Credentials'] = 'true' response.headers[ 'Access-Control-Expose-Headers'] = 'X-Rucio-Auth-Token' response.headers['Content-Type'] = 'application/octet-stream' response.headers[ 'Cache-Control'] = 'no-cache, no-store, max-age=0, must-revalidate' response.headers['Cache-Control'] = 'post-check=0, pre-check=0' response.headers['Pragma'] = 'no-cache' account = request.environ.get('HTTP_X_RUCIO_ACCOUNT') signature = request.environ.get('HTTP_X_RUCIO_SSH_SIGNATURE') appid = request.environ.get('HTTP_X_RUCIO_APPID') if appid is None: appid = 'unknown' ip = request.environ.get('HTTP_X_FORWARDED_FOR') if ip is None: ip = request.remote_addr # decode the signature which must come in base64 encoded try: signature = base64.b64decode(signature) except Exception as error: # noqa: F841 return generate_http_error_flask( 401, 'CannotAuthenticate', 'Cannot authenticate to account %(account)s with malformed signature' % locals()) try: result = get_auth_token_ssh(account, signature, appid, ip) except AccessDenied: return generate_http_error_flask( 401, 'CannotAuthenticate', 'Cannot authenticate to account %(account)s with given credentials' % locals()) except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) return error, 500 if not result: return generate_http_error_flask( 401, 'CannotAuthenticate', 'Cannot authenticate to account %(account)s with given credentials' % locals()) response.headers['X-Rucio-Auth-Token'] = result response.set_data(str()) return response
def post(self, scope, name): """ Append data identifiers to data identifiers. .. :quickref: Attachment; Append DID to DID. **Example request**: .. sourcecode:: http POST /dids/scope1/datasets1/dids HTTP/1.1 Host: rucio.cern.ch [{"scope": "scope1", "name": "file1"}, {"scope": "scope1", "name": "file2"}, {"scope": "scope1", "name": "file3"}] **Example response**: .. sourcecode:: http HTTP/1.1 201 Created Vary: Accept :param scope: The scope of the DID to attach to. :param name: The name of the DID to attach to. :<json list attachments: List of dicts of DIDs to attach. :status 201: DIDs successfully attached :status 401: Invalid Auth Token :status 404: DID not found :status 409: DIDs already attached :status 500: Database Exception """ try: json_data = loads(request.data) except ValueError: return generate_http_error_flask( 400, 'ValueError', 'Cannot decode json parameter list') try: attach_dids(scope=scope, name=name, attachment=json_data, issuer=request.environ.get('issuer')) except DataIdentifierNotFound as error: return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0]) except DuplicateContent as error: return generate_http_error_flask(409, 'DuplicateContent', error.args[0]) except AccessDenied as error: return generate_http_error_flask(401, 'AccessDenied', error.args[0]) except UnsupportedOperation as error: return generate_http_error_flask(409, 'UnsupportedOperation', error.args[0]) except RSENotFound as error: return generate_http_error_flask(404, 'RSENotFound', error.args[0]) except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) return error, 500 return "Created", 201
def get(self, rse, scheme=None): """ Return PFNs for a set of LFNs. Formatted as a JSON object where the key is a LFN and the value is the corresponding PFN. .. :quickref: Attributes; Translate LFNs to PFNs. :param rse: The RSE name. :param scheme: The protocol identifier. :query lfn: One or moref LFN to translate. :query scheme: Optional argument to help with the protocol selection (e.g., http / gsiftp / srm) :query domain: Optional argument used to select the protocol for wan or lan use cases. :query operation: Optional query argument to select the protoco for read-vs-writes. :resheader Content-Type: application/json :status 200: OK. :status 401: Invalid Auth Token. :status 404: RSE Not Found. :status 404: RSE Protocol Not Supported. :status 404: RSE Protocol Domain Not Supported. :status 406: Not Acceptable. :status 500: Internal Error. :returns: A list with detailed PFN information. """ lfns = [] scheme = request.get('scheme', None) domain = request.get('domain', 'wan') operation = request.get('operation', 'write') p_lfns = request.get('lfn', None) if p_lfns: info = p_lfns.split(":", 1) if len(info) != 2: return generate_http_error_flask(400, 'InvalidPath', 'LFN in invalid format') lfn_dict = {'scope': info[0], 'name': info[1]} lfns.append(lfn_dict) rse_settings = None try: rse_settings = get_rse_protocols( rse, issuer=request.environ.get('issuer'), vo=request.environ.get('vo')) except RSENotFound as error: return generate_http_error_flask(404, 'RSENotFound', error.args[0]) except RSEProtocolNotSupported as error: return generate_http_error_flask(404, 'RSEProtocolNotSupported', error.args[0]) except RSEProtocolDomainNotSupported as error: return generate_http_error_flask(404, 'RSEProtocolDomainNotSupported', error.args[0]) except Exception as error: print(error) print(format_exc()) return error, 500 pfns = rsemanager.lfns2pfns(rse_settings, lfns, operation=operation, scheme=scheme, domain=domain) return Response(dumps(pfns), content_type="application/json")
def put(self, scope, name): """ Update data identifier status. .. :quickref: DIDs; Update DID status. .. sourcecode:: http PUT /dids/scope1/container1 HTTP/1.1 Host: rucio.cern.ch {"open": False}, **Example response**: .. sourcecode:: http HTTP/1.1 200 OK Vary: Accept :param scope: data identifier scope. :param name: data identifier name. :<json bool open: open or close did :status 200: DIDs successfully updated :status 401: Invalid Auth Token :status 404: DID not found :status 409: Wrong status :status 500: Database Exception """ json_data = request.data try: kwargs = loads(json_data) except ValueError: return generate_http_error_flask( 400, 'ValueError', 'Cannot decode json data parameter') try: set_status(scope=scope, name=name, issuer=request.environ.get('issuer'), **kwargs) except DataIdentifierNotFound as error: return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0]) except UnsupportedStatus as error: return generate_http_error_flask(409, 'UnsupportedStatus', error.args[0]) except UnsupportedOperation as error: return generate_http_error_flask(409, 'UnsupportedOperation', error.args[0]) except AccessDenied as error: return generate_http_error_flask(401, 'AccessDenied', error.args[0]) except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) return error, 500 return "Ok", 200
def post(self, scope, name): """ Create a new data identifier. .. :quickref: DIDs; Create a new DID. .. sourcecode:: http POST /dids/scope1/container1 HTTP/1.1 Host: rucio.cern.ch {"type": "CONTAINER", "lifetime": 86400}, **Example response**: .. sourcecode:: http HTTP/1.1 201 Created Vary: Accept :reqheader Accept: application/json :param scope: data identifier scope. :param name: data identifier name. :<json string type: the new DID type :<json dict statuses: Dictionary with statuses, e.g. {'monotonic':True} :<json dict meta: Dictionary with metadata, e.g. {'length':1234} :<json dict rules: Replication rules associated with the did. e.g., [{'copies': 2, 'rse_expression': 'TIERS1'}, ] :<json int lifetime: DID's liftime in seconds. :<json list dids: The content. :<json string rse: The RSE name when registering replicas. :status 201: new DIDs created :status 401: Invalid Auth Token :status 409: DID already exists :status 500: Database Exception """ statuses, meta, rules, lifetime, dids, rse = {}, {}, [], None, [], None try: json_data = loads(request.data) type = json_data['type'] if 'statuses' in json_data: statuses = json_data['statuses'] if 'meta' in json_data: meta = json_data['meta'] if 'rules' in json_data: rules = json_data['rules'] if 'lifetime' in json_data: lifetime = json_data['lifetime'] if 'dids' in json_data: dids = json_data['dids'] if 'rse' in json_data: rse = json_data['rse'] except ValueError: return generate_http_error_flask( 400, 'ValueError', 'Cannot decode json parameter list') except KeyError as error: return generate_http_error_flask(400, 'ValueError', str(error)) try: add_did(scope=scope, name=name, type=type, statuses=statuses, meta=meta, rules=rules, lifetime=lifetime, dids=dids, rse=rse, issuer=request.environ.get('issuer')) except DataIdentifierNotFound as error: return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0]) except DuplicateContent as error: return generate_http_error_flask(409, 'DuplicateContent', error.args[0]) except DataIdentifierAlreadyExists as error: return generate_http_error_flask(409, 'DataIdentifierAlreadyExists', error.args[0]) except AccessDenied as error: return generate_http_error_flask(401, 'AccessDenied', error.args[0]) except UnsupportedOperation as error: return generate_http_error_flask(409, 'UnsupportedOperation', error.args[0]) except DatabaseException as error: return generate_http_error_flask(500, 'DatabaseException', error.args[0]) except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) return error, 500 return "Created", 201
def post(self): """ Add new DIDs in bulk. .. :quickref: BulkDID; Bulk add DIDs. **Example request**: .. sourcecode:: http POST /dids/ HTTP/1.1 Host: rucio.cern.ch [ {"scope": "scope1", "type": "CONTAINER", "name": "container1", "account": "jdoe", "length": 1}, {"scope": "scope1", "type": "DATASET", "name": "dataset1", "account": "jdoe", "length": 3} ] **Example response**: .. sourcecode:: http HTTP/1.1 201 Created Vary: Accept :reqheader Accept: application/json :<json string scope: the new DID scope :<json string name: the new DID name :<json string type: the new DID type :<json string account: the owner account of the new DID :<json string statuses: monotonic :status 201: new DIDs created :status 401: Invalid Auth Token :status 409: DID already exists :status 500: Database Exception """ try: json_data = loads(request.data) except ValueError: return generate_http_error_flask( 400, 'ValueError', 'Cannot decode json parameter list') try: print(json_data) add_dids(json_data, issuer=request.environ.get('issuer')) except DataIdentifierNotFound as error: return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0]) except DuplicateContent as error: return generate_http_error_flask(409, 'DuplicateContent', error.args[0]) except DataIdentifierAlreadyExists as error: return generate_http_error_flask(409, 'DataIdentifierAlreadyExists', error.args[0]) except AccessDenied as error: return generate_http_error_flask(401, 'AccessDenied', error.args[0]) except UnsupportedOperation as error: return generate_http_error_flask(409, 'UnsupportedOperation', error.args[0]) except DatabaseException as error: return generate_http_error_flask(500, 'DatabaseException', error.args[0]) except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) return error, 500 return "Created", 201
def get(self): """ Authenticate a Rucio account temporarily via a GSS token. .. :quickref: GSS; Authenticate with GSS token :reqheader Rucio-Account: Account identifier as a string. :reqheader Rucio-AppID: Application identifier as a string. :reqheader SavedCredentials: Apache mod_auth_kerb SavedCredentials. :resheader Access-Control-Allow-Origin: :resheader Access-Control-Allow-Headers: :resheader Access-Control-Allow-Methods: :resheader Access-Control-Allow-Credentials: :resheader Access-Control-Expose-Headers: :resheader X-Rucio-Auth-Token: The authentication token :status 200: Successfully authenticated :status 404: Invalid credentials """ response = Response() response.headers['Access-Control-Allow-Origin'] = request.environ.get( 'HTTP_ORIGIN') response.headers['Access-Control-Allow-Headers'] = request.environ.get( 'HTTP_ACCESS_CONTROL_REQUEST_HEADERS') response.headers['Access-Control-Allow-Methods'] = '*' response.headers['Access-Control-Allow-Credentials'] = 'true' response.headers[ 'Access-Control-Expose-Headers'] = 'X-Rucio-Auth-Token' response.headers['Content-Type'] = 'application/octet-stream' response.headers[ 'Cache-Control'] = 'no-cache, no-store, max-age=0, must-revalidate' response.headers['Cache-Control'] = 'post-check=0, pre-check=0' response.headers['Pragma'] = 'no-cache' account = request.environ.get('HTTP_X_RUCIO_ACCOUNT') gsscred = request.environ.get('REMOTE_USER') appid = request.environ.get('HTTP_X_RUCIO_APPID') if appid is None: appid = 'unknown' ip = request.environ.get('HTTP_X_FORWARDED_FOR') if ip is None: ip = request.remote_addr try: result = get_auth_token_gss(account, gsscred, appid, ip) except AccessDenied: return generate_http_error_flask( 401, 'CannotAuthenticate', 'Cannot authenticate to account %(account)s with given credentials' % locals()) if result is None: return generate_http_error_flask( 401, 'CannotAuthenticate', 'Cannot authenticate to account %(account)s with given credentials' % locals()) else: response.headers('X-Rucio-Auth-Token', result) return str() return 'BadRequest', 400
appid, ip) except AccessDenied: return generate_http_error_flask( 401, 'CannotAuthenticate', 'Cannot authenticate to account %(account)s with given credentials' % locals()) except RucioException, e: return generate_http_error_flask(500, e.__class__.__name__, e.args[0]) except Exception, e: print format_exc() return e, 500 if not result: return generate_http_error_flask( 401, 'CannotAuthenticate', 'Cannot authenticate to account %(account)s with given credentials' % locals()) response.headers['X-Rucio-Auth-Token'] = result return response class GSS(MethodView): """ Authenticate a Rucio account temporarily via a GSS token. """ def options(self): """ HTTP Success: 200 OK
def get(self): """ Authenticate a Rucio account temporarily via an x509 certificate. .. :quickref: x509; Authenticate with x509 certificate. :reqheader Rucio-Account: Account identifier as a string. :reqheader Rucio-AppID: Application identifier as a string. :reqheader SSLStdEnv: Apache mod_ssl SSL Standard Env Variables. :resheader Access-Control-Allow-Origin: :resheader Access-Control-Allow-Headers: :resheader Access-Control-Allow-Methods: :resheader Access-Control-Allow-Credentials: :resheader Access-Control-Expose-Headers: :resheader X-Rucio-Auth-Token: The authentication token :status 200: Successfully authenticated :status 404: Invalid credentials """ response = Response() response.headers['Access-Control-Allow-Origin'] = request.environ.get( 'HTTP_ORIGIN') response.headers['Access-Control-Allow-Headers'] = request.environ.get( 'HTTP_ACCESS_CONTROL_REQUEST_HEADERS') response.headers['Access-Control-Allow-Methods'] = '*' response.headers['Access-Control-Allow-Credentials'] = 'true' response.headers[ 'Access-Control-Expose-Headers'] = 'X-Rucio-Auth-Token' response.headers['Content-Type'] = 'application/octet-stream' response.headers[ 'Cache-Control'] = 'no-cache, no-store, max-age=0, must-revalidate' response.headers['Cache-Control'] = 'post-check=0, pre-check=0' response.headers['Pragma'] = 'no-cache' account = request.environ.get('HTTP_X_RUCIO_ACCOUNT') dn = request.environ.get('SSL_CLIENT_S_DN') if not dn: return generate_http_error_flask(401, 'CannotAuthenticate', 'Cannot get DN') if not dn.startswith('/'): dn = '/%s' % '/'.join(dn.split(',')[::-1]) appid = request.environ.get('HTTP_X_RUCIO_APPID') if appid is None: appid = 'unknown' ip = request.environ.get('HTTP_X_FORWARDED_FOR') if ip is None: ip = request.remote_addr # If we get a valid proxy certificate we have to strip this postfix, # otherwise we would have to store the proxy DN in the database as well. # Alternative: use the SSL_CLIENT_I_DN, but that would require a separate # endpoint as you cannot programmatically decide, by examining the SSL variables, # if you got a proxy or regular certificate while True: if dn.endswith('/CN=limited proxy'): dn = dn[:-17] elif dn.endswith('/CN=proxy'): dn = dn[:-9] elif search('/CN=[0-9]*$', dn): dn = dn.rpartition('/')[0] else: break try: result = get_auth_token_x509(account, dn, appid, ip) except AccessDenied: print('Cannot Authenticate', account, dn, appid, ip) return generate_http_error_flask( 401, 'CannotAuthenticate', 'Cannot authenticate to account %(account)s with given credentials' % locals()) except IdentityError: print('Cannot Authenticate', account, dn, appid, ip) return generate_http_error_flask( 401, 'CannotAuthenticate', 'No default account set for %(dn)s' % locals()) if not result: print('Cannot Authenticate', account, dn, appid, ip) return generate_http_error_flask( 401, 'CannotAuthenticate', 'Cannot authenticate to account %(account)s with given credentials' % locals()) response.headers['X-Rucio-Auth-Token'] = result response.set_data(str()) return response
except ValueError: return generate_http_error_flask( 400, 'ValueError', 'Cannot decode json parameter list') try: print json_data add_dids(json_data, issuer=request.environ.get('issuer')) except DataIdentifierNotFound, error: return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0]) except DuplicateContent, error: return generate_http_error_flask(409, 'DuplicateContent', error.args[0]) except DataIdentifierAlreadyExists, error: return generate_http_error_flask(409, 'DataIdentifierAlreadyExists', error.args[0]) except AccessDenied, error: return generate_http_error_flask(401, 'AccessDenied', error.args[0]) except UnsupportedOperation, error: return generate_http_error_flask(409, 'UnsupportedOperation', error.args[0]) except DatabaseException, error: return generate_http_error_flask(500, 'DatabaseException', error.args[0]) except RucioException, error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception, error: print format_exc()
def get(self): """ Authenticate a Rucio account temporarily via username and password. .. :quickref: UserPass; Authenticate with username/password :reqheader X-Rucio-Account: Account identifier as a string. :reqheader X-Rucio-Username: Username as a string. :reqheader X-Rucio-Password: password as a text-plain string. :reqheader X-Rucio-AppID: Application identifier as a string. :resheader Access-Control-Allow-Origin: :resheader Access-Control-Allow-Headers: :resheader Access-Control-Allow-Methods: :resheader Access-Control-Allow-Credentials: :resheader Access-Control-Expose-Headers: :resheader X-Rucio-Auth-Token: The authentication token :status 200: Successfully authenticated :status 404: Invalid credentials """ response = Response() response.headers['Access-Control-Allow-Origin'] = request.environ.get( 'HTTP_ORIGIN') response.headers['Access-Control-Allow-Headers'] = request.environ.get( 'HTTP_ACCESS_CONTROL_REQUEST_HEADERS') response.headers['Access-Control-Allow-Methods'] = '*' response.headers['Access-Control-Allow-Credentials'] = 'true' response.headers[ 'Access-Control-Expose-Headers'] = 'X-Rucio-Auth-Token' response.headers['Content-Type'] = 'application/octet-stream' response.headers[ 'Cache-Control'] = 'no-cache, no-store, max-age=0, must-revalidate' response.headers['Cache-Control'] = 'post-check=0, pre-check=0' response.headers['Pragma'] = 'no-cache' account = request.environ.get('HTTP_X_RUCIO_ACCOUNT') username = request.environ.get('HTTP_X_RUCIO_USERNAME') password = request.environ.get('HTTP_X_RUCIO_PASSWORD') appid = request.environ.get('HTTP_X_RUCIO_APPID') if appid is None: appid = 'unknown' ip = request.environ.get('HTTP_X_FORWARDED_FOR') if ip is None: ip = request.remote_addr print(account, username, password, appid) try: result = get_auth_token_user_pass(account, username, password, appid, ip) except AccessDenied: return generate_http_error_flask( 401, 'CannotAuthenticate', 'Cannot authenticate to account %(account)s with given credentials' % locals()) except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) return error, 500 if not result: return generate_http_error_flask( 401, 'CannotAuthenticate', 'Cannot authenticate to account %(account)s with given credentials' % locals()) response.headers['X-Rucio-Auth-Token'] = result return response
class Search(MethodView): def get(self, scope): """ List all data identifiers in a scope which match a given metadata. .. :quickref: Search; Search DIDs in a scope with given metadata. **Example request**: .. sourcecode:: http GET /dids/scope1/dids/search?type=collection1&long=True HTTP/1.1 Host: rucio.cern.ch **Example response**: .. sourcecode:: http HTTP/1.1 200 OK Vary: Accept Content-Type: application/x-json-stream {"scope": "scope1", "did_type": "CONTAINER", "name": "container1", "bytes": 1234, "length": 1} {"scope": "scope1", "did_type": "DATASET", "name": "dataset1", "bytes": 234, "length": 3} :query type: specify a DID type to search for :query long: set to True for long output, otherwise only name :resheader Content-Type: application/x-json-stream :status 200: DIDs found :status 401: Invalid Auth Token :status 404: Invalid key in filters :status 409: Wrong DID type :returns: Line separated name of DIDs or dictionaries of DIDs for long option """ filters = {} long = False type = 'collection' for k, v in request.args.items(): if k == 'type': type = v elif k == 'long': long = bool(v) else: filters[k] = v[0] try: data = "" for did in list_dids(scope=scope, filters=filters, type=type, long=long): data += dumps(did) + '\n' return Response(data, content_type='application/x-json-stream') except UnsupportedOperation, error: return generate_http_error_flask(409, 'UnsupportedOperation', error.args[0]) except KeyNotFound, error: return generate_http_error_flask(404, 'KeyNotFound', error.args[0])
def post(self, rse): """ Create RSE with given name. .. :quickref: RSE; create a new RSE. :param rse: The RSE name. :<json bool deterministic: Boolean to know if the pfn is generated deterministically. :<json bool volatile: Boolean for RSE cache. :<json string city: City for the RSE. :<json bool staging_area: Staging area. :<json string region_code: The region code for the RSE. :<json string country_name: The country. :<json string continent: The continent. :<json string time_zone: Timezone. :<json string ISP: Internet Service Provider. :<json string rse_type: RSE type. :<json number latitude: Latitude coordinate of RSE. :<json number longitude: Longitude coordinate of RSE. :<json string ASN: Access service network. :<json integer availability: Availability. :status 201: RSE created successfully. :status 400: Cannot decode json parameter dictionary. :status 401: Invalid Auth Token. :status 409: RSE already exists. :status 409: RSE not found. :status 500: Internal Error. """ json_data = request.data kwargs = { 'deterministic': True, 'volatile': False, 'city': None, 'staging_area': False, 'region_code': None, 'country_name': None, 'continent': None, 'time_zone': None, 'ISP': None, 'rse_type': None, 'latitude': None, 'longitude': None, 'ASN': None, 'availability': None } try: parameters = json_data and loads(json_data) if parameters: for param in kwargs: if param in parameters: kwargs[param] = parameters[param] except ValueError: return generate_http_error_flask( 400, 'ValueError', 'Cannot decode json parameter dictionary') kwargs['issuer'] = request.environ.get('issuer') try: add_rse(rse, **kwargs) except InvalidObject as error: return generate_http_error_flask(400, 'InvalidObject', error.args[0]) except AccessDenied as error: return generate_http_error_flask(401, 'AccessDenied', error.args[0]) except RSENotFound as error: return generate_http_error_flask(404, 'RSENotFound', error.args[0]) except Duplicate as error: return generate_http_error_flask(409, 'Duplicate', error.args[0]) except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: print(error) print(format_exc()) return error, 500 return "Created", 201
return generate_http_error_flask( 400, 'ValueError', 'Cannot decode json parameter list') try: add_key(key=key, key_type=key_type, value_type=value_type, value_regexp=value_regexp, issuer=request.environ.get('issuer')) except Duplicate, e: return generate_http_error_flask(409, 'Duplicate', e[0][0]) except UnsupportedValueType, e: return generate_http_error_flask(400, 'UnsupportedValueType', e[0][0]) except RucioException, e: return generate_http_error_flask(500, e.__class__.__name__, e.args[0][0]) except Exception, e: print e return e, 500 return "Created", 201 class Values(MethodView): """ REST APIs for data identifier attribute values. """ def get(self, key): """ List all values for a key. .. :quickref: Values; List all key values.
if param in parameters: kwargs[param] = parameters[param] except ValueError: return generate_http_error_flask( 400, 'ValueError', 'Cannot decode json parameter dictionary') kwargs['issuer'] = request.environ.get('issuer') try: add_rse(rse, **kwargs) except InvalidObject, error: return generate_http_error_flask(400, 'InvalidObject', error.args[0]) except AccessDenied, error: return generate_http_error_flask(401, 'AccessDenied', error.args[0]) except RSENotFound, error: return generate_http_error_flask(404, 'RSENotFound', error.args[0]) except Duplicate, error: return generate_http_error_flask(409, 'Duplicate', error.args[0]) except RucioException, error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception, error: print error print format_exc() return error, 500 return "Created", 201 def put(self, rse): """ Update RSE properties (e.g. name, availability).
:status 201: scope created :status 404: account does not exist :status 401: unauthorized :status 409: scope already exists :status 500: internal server error """ try: add_scope(scope, account, issuer=request.environ.get('issuer')) except Duplicate, e: return generate_http_error_flask(409, 'Duplicate', e.args[0][0]) except AccountNotFound, e: return generate_http_error_flask(404, 'AccountNotFound', e.args[0][0]) except RucioException, e: print(e) return generate_http_error_flask(500, e.__class__.__name__, e.args[0][0]) except Exception, e: return e, 500 return "OK", 201 bp = Blueprint('scope', __name__) scope_view = Scope.as_view('scope') bp.add_url_rule('/', view_func=scope_view, methods=[ 'GET', ]) bp.add_url_rule('/<account>/<scope>', view_func=scope_view, methods=[ 'POST', ])
def post(self): """ List all replicas for data identifiers. .. :quickref: ListReplicas; List all replicas for did. :reqheader HTTP_ACCEPT: application/metalink4+xml :query schemes: A list of schemes to filter the replicas. :query sort: Requested sorting of the result, e.g., 'geoip', 'closeness', 'dynamic', 'ranking'. :<json list dids: list of DIDs. :<json list schemes: A list of schemes to filter the replicas. :<json bool unavailable: Also include unavailable replicas. :<json bool all_states: Return all replicas whatever state they are in. Adds an extra 'states' entry in the result dictionary. :<json string rse_expression: The RSE expression to restrict on a list of RSEs. :<json dict client_location: Client location dictionary for PFN modification {'ip', 'fqdn', 'site'}. :<json bool sort: Requested sorting of the result, e.g., 'geoip', 'closeness', 'dynamic', 'ranking'. :<json string domain: The network domain for the call, either None, 'wan' or 'lan'. None is fallback to 'wan', 'all' is both ['lan','wan'] :resheader Content-Type: application/x-json-stream :resheader Content-Type: application/metalink4+xml :status 200: OK. :status 400: Cannot decode json parameter list. :status 401: Invalid auth token. :status 404: DID not found. :status 500: Internal Error. :returns: A dictionary containing all replicas information. :returns: A metalink description of replicas if metalink(4)+xml is specified in Accept: """ metalink = False if request.environ.get('HTTP_ACCEPT') is not None: tmp = request.environ.get('HTTP_ACCEPT').split(',') if 'application/metalink4+xml' in tmp: metalink = True client_ip = request.environ.get('HTTP_X_FORWARDED_FOR') if client_ip is None: client_ip = request.remote_addr dids, schemes, select, unavailable, limit = [], None, None, False, None ignore_availability, rse_expression, all_states = False, None, False client_location = {} json_data = request.data try: params = parse_response(json_data) if 'dids' in params: dids = params['dids'] if 'schemes' in params: schemes = params['schemes'] if 'unavailable' in params: unavailable = params['unavailable'] ignore_availability = True if 'all_states' in params: all_states = params['all_states'] if 'rse_expression' in params: rse_expression = params['rse_expression'] if 'client_location' in params: client_location = params['client_location'] client_location['ip'] = params['client_location'].get( 'ip', client_ip) if 'sort' in params: select = params['sort'] if 'domain' in params: domain = params['domain'] except ValueError: return generate_http_error_flask( 400, 'ValueError', 'Cannot decode json parameter list') schemes = request.args.get('schemes', None) select = request.args.get('select', None) select = request.args.get('sort', None) data = "" content_type = 'application/x-json-stream' try: # first, set the appropriate content type, and stream the header if metalink: content_type = 'application/metalink4+xml' data += '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n' # then, stream the replica information for rfile in list_replicas( dids=dids, schemes=schemes, unavailable=unavailable, request_id=request.environ.get('request_id'), ignore_availability=ignore_availability, all_states=all_states, rse_expression=rse_expression, client_location=client_location, domain=domain): replicas = [] dictreplica = {} for rse in rfile['rses']: for replica in rfile['rses'][rse]: replicas.append(replica) dictreplica[replica] = rse if not metalink: data += dumps(rfile, cls=APIEncoder) + '\n' else: data += ' <file name="' + rfile['name'] + '">\n' data += ' <identity>' + rfile['scope'] + ':' + rfile[ 'name'] + '</identity>\n' if rfile['adler32'] is not None: data += ' <hash type="adler32">' + rfile[ 'adler32'] + '</hash>\n' if rfile['md5'] is not None: data += ' <hash type="md5">' + rfile[ 'md5'] + '</hash>\n' data += ' <size>' + str(rfile['bytes']) + '</size>\n' data += ' <glfn name="/atlas/rucio/%s:%s">' % ( rfile['scope'], rfile['name']) data += '</glfn>\n' if select == 'geoip': replicas = sort_geoip(dictreplica, client_location['ip']) elif select == 'closeness': replicas = sort_closeness(dictreplica, client_location) elif select == 'dynamic': replicas = sort_dynamic(dictreplica, client_location) elif select == 'ranking': replicas = sort_ranking(dictreplica, client_location) else: replicas = sort_random(dictreplica) idx = 0 for replica in replicas: data += ' <url location="' + str( dictreplica[replica]) + '" priority="' + str( idx + 1) + '">' + replica + '</url>\n' idx += 1 if limit and limit == idx: break data += ' </file>\n' # don't forget to send the metalink footer if metalink: data += '</metalink>\n' return Response(data, content_type=content_type) except DataIdentifierNotFound as error: return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0]) except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: print format_exc() return error, 500
def before_request(): if request.environ.get('REQUEST_METHOD') == 'OPTIONS': raise "", 200 auth_token = request.environ.get('HTTP_X_RUCIO_AUTH_TOKEN') try: auth = validate_auth_token(auth_token) except RucioException, e: return generate_http_error_flask(500, e.__class__.__name__, e.args[0][0]) except Exception, e: print format_exc() return e, 500 if auth is None: return generate_http_error_flask(401, 'CannotAuthenticate', 'Cannot authenticate with given credentials') request.environ['issuer'] = auth.get('account') request.environ['identity'] = auth.get('identity') request.environ['request_id'] = generate_uuid() request.environ['start_time'] = time() def after_request(response): response.headers['Access-Control-Allow-Origin'] = request.environ.get('HTTP_ORIGIN') response.headers['Access-Control-Allow-Headers'] = request.environ.get('HTTP_ACCESS_CONTROL_REQUEST_HEADERS') response.headers['Access-Control-Allow-Methods'] = '*' response.headers['Access-Control-Allow-Credentials'] = 'true' if request.environ.get('REQUEST_METHOD') == 'GET': response.headers['Cache-Control'] = 'no-cache, no-store, max-age=0, must-revalidate'
def get(self, scope, name): """ Header Redirect .. :quickref: HeaderRedirector; Header redirect. :param scope: The scope name of the file. :param name: The name of the file. :resheader Content-Type: application/metalink+xml'. :status 303: Redirect. :status 401: Invalid Auth Token. :status 404: RSE Not Found. :status 404: DID Not Found. :status 500: Internal Error. """ headers = {} try: # use the default HTTP protocols if no scheme is given client_ip = request.environ.get('HTTP_X_FORWARDED_FOR') if client_ip is None: client_ip = request.remote_addr client_location = {'ip': client_ip, 'fqdn': None, 'site': None} schemes = request.args.get('schemes', ['davs', 'https', 's3']) select = request.args.get('select', 'random') if 'sort' in request.args: select = request.args['sort'] rse = request.args.get('rse', None) site = request.args.get('site', None) client_location['ip'] = request.args.get('ip', client_ip) client_location['fqdn'] = request.args.get('fqdn', None) client_location['site'] = request.args.get('site', None) # correctly forward the schemes and select to potential metalink followups cleaned_url = request.environ.get('REQUEST_URI').split('?')[0] if isinstance(schemes, list): headers[ 'Link'] = '<%s/metalink?schemes=%s&select=%s>; rel=describedby; type="application/metalink+xml"' % ( cleaned_url, ','.join(schemes), select) else: headers[ 'Link'] = '<%s/metalink?schemes=%s&select=%s>; rel=describedby; type="application/metalink+xml"' % ( cleaned_url, schemes, select) schemes = [schemes] # list_replicas needs a list replicas = [ r for r in list_replicas(dids=[{ 'scope': scope, 'name': name, 'type': 'FILE' }], schemes=schemes, client_location=client_location) ] selected_url, selected_rse = None, None for r in replicas: if r['rses']: dictreplica = {} if rse: if rse in r['rses'] and r['rses'][rse]: selected_url = r['rses'][rse][0] selected_rse = rse else: return 'no redirection possible - no valid RSE for HTTP redirection found', 404 else: for rep in r['rses']: for replica in r['rses'][rep]: # since this is HTTP-only redirection, and to ensure compatibility with as many http clients as possible # forcibly replacement davs and s3 URLs to https replica = replica.replace( 'davs://', 'https://').replace('s3://', 'https://') dictreplica[replica] = rep if not dictreplica: return 'no redirection possible - no valid RSE for HTTP redirection found', 404 elif site: rep = site_selector(dictreplica, site) if rep: selected_url = rep[0] else: return 'no redirection possible - no valid RSE for HTTP redirection found', 404 else: if select == 'geoip': rep = sort_geoip(dictreplica, client_location['ip']) elif select == 'closeness': rep = sort_closeness(dictreplica, client_location) elif select == 'dynamic': rep = sort_dynamic(dictreplica, client_location) elif select == 'ranking': rep = sort_ranking(dictreplica, client_location) else: rep = sort_random(dictreplica) selected_url = rep[0] for rep in r['rses']: for replica in r['rses'][rep]: if selected_url == replica: selected_rse = rep if selected_url: if selected_url.startswith('s3+rucio://'): connect(selected_rse, selected_url) signed_URLS = get_signed_urls([selected_url], rse=selected_rse, operation='read') res = redirect(signed_URLS[selected_url], code=303) res.header = headers return res res = redirect(signed_URLS[selected_url], code=303) res.header = headers return res return 'no redirection possible - file does not exist', 404 except ReplicaNotFound as error: return generate_http_error_flask(404, 'ReplicaNotFound', error.args[0]) except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) return error, 500
def get(self): """ Request a challenge token for SSH authentication .. :quickref: SSHChallengeToken; Request SSH Challenge Token :reqheader Rucio-VO: VO name as a string (Multi-VO only). :reqheader Rucio-Account: Account identifier as a string. :reqheader Rucio-AppID: Application identifier as a string. :resheader Access-Control-Allow-Origin: :resheader Access-Control-Allow-Headers: :resheader Access-Control-Allow-Methods: :resheader Access-Control-Allow-Credentials: :resheader Access-Control-Expose-Headers: :resheader X-Rucio-Auth-Token: The authentication token :status 200: Successfully authenticated :status 404: Invalid credentials """ response = Response() response.headers['Access-Control-Allow-Origin'] = request.environ.get( 'HTTP_ORIGIN') response.headers['Access-Control-Allow-Headers'] = request.environ.get( 'HTTP_ACCESS_CONTROL_REQUEST_HEADERS') response.headers['Access-Control-Allow-Methods'] = '*' response.headers['Access-Control-Allow-Credentials'] = 'true' response.headers[ 'Access-Control-Expose-Headers'] = 'X-Rucio-Auth-Token' response.headers['Content-Type'] = 'application/octet-stream' response.headers[ 'Cache-Control'] = 'no-cache, no-store, max-age=0, must-revalidate' response.headers['Cache-Control'] = 'post-check=0, pre-check=0' response.headers['Pragma'] = 'no-cache' vo = request.environ.get('HTTP_X_RUCIO_VO', 'def') account = request.environ.get('HTTP_X_RUCIO_ACCOUNT') appid = request.environ.get('HTTP_X_RUCIO_APPID') if appid is None: appid = 'unknown' ip = request.environ.get('HTTP_X_FORWARDED_FOR') if ip is None: ip = request.remote_addr try: result = get_ssh_challenge_token(account, appid, ip, vo=vo) except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) return error, 500 if not result: return generate_http_error_flask( 401, 'CannotAuthenticate', 'Cannot generate challenge for account %(account)s' % locals()) response.headers['X-Rucio-Auth-Token'] = result.token response.headers['X-Rucio-Auth-Token-Expires'] = date_to_str( result.expired_at) return response