def get(self, scope):
"""
List all data identifiers in a scope which match a given metadata.
.. :quickref: Search; Search DIDs in a scope with given metadata.
**Example request**:
.. sourcecode:: http
GET /dids/scope1/dids/search?type=collection&long=True&length.lt=10 HTTP/1.1
Host: rucio.cern.ch
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: application/x-json-stream
{"scope": "scope1", "did_type": "CONTAINER", "name": "container1",
"bytes": 1234, "length": 1}
{"scope": "scope1", "did_type": "DATASET", "name": "dataset1",
"bytes": 234, "length": 3}
:query type: specify a DID type to search for
:query long: set to True for long output, otherwise only name
:query recursive: set to True to recursively list DIDs content
:query created_before: Date string in RFC-1123 format where the creation date was earlier
:query created_after: Date string in RFC-1123 format where the creation date was later
:query length: Exact number of attached DIDs
:query length.gt: Number of attached DIDs greater than
:query length.lt: Number of attached DIDs less than
:query length.gte: Number of attached DIDs greater than or equal to
:query length.lte: Number of attached DIDs less than or equal to
:query name: Name or pattern of a DID name
:resheader Content-Type: application/x-json-stream
:status 200: DIDs found
:status 401: Invalid Auth Token
:status 404: Invalid key in filters
:status 409: Wrong DID type
:returns: Line separated name of DIDs or dictionaries of DIDs for long option
"""
filters = {}
long = False
recursive = False
type = 'collection'
for k, v in request.args.items():
if k == 'type':
type = v
elif k == 'long':
long = v == '1'
elif k == 'recursive':
recursive = v == 'True'
else:
filters[k] = v
try:
data = ""
for did in list_dids(scope=scope,
filters=filters,
type=type,
long=long,
recursive=recursive):
data += dumps(did) + '\n'
return Response(data, content_type='application/x-json-stream')
except UnsupportedOperation as error:
return generate_http_error_flask(409, 'UnsupportedOperation',
error.args[0])
except KeyNotFound as error:
return generate_http_error_flask(404, 'KeyNotFound', error.args[0])
except Exception as error:
print(format_exc())
return error, 500
def post(self):
"""
Create a new replication rule.
.. :quickref: AllRule; create new rule
:<json list dids: List of data identifiers.
:<json string account: Account issuing the rule.
:<json int copies: The number of replicas.
:<json string rse_expression: RSE expression which gets resolved into a list of RSEs.
:<json string grouping: ALL - All files will be replicated to the same RSE.
DATASET - All files in the same dataset will be replicated to the same RSE.
NONE - Files will be completely spread over all allowed RSEs without any grouping considerations at all
:<json int weight: Weighting scheme to be used.
:<json int lifetime: The lifetime of the replication rule in seconds.
:<json string locked: If the is locked.
:<json string subscription_id: The subscription_id, if the rule is created by a subscription.
:<json string source_replica_expression: Only use replicas as source from these RSEs.
:<json string activity: Activity to be passed to the conveyor.
:<json string notify: Notification setting of the rule ('Y', 'N', 'C'; None = 'N').
:<json bool purge_replicas: Purge setting if a replica should be directly deleted after the rule is deleted.
:<json bool ignore_availability: Option to ignore the availability of RSEs.
:<json string comments: Comment about the rule.
:<json bool ask_approval: Ask for approval for this rule.
:<json bool asynchronous: Create replication rule asynchronously by the judge-injector.
:<json int priority: Priority of the rule and the transfers which should be submitted.
:<json bool split_container: Should a container rule be split into individual dataset rules.
:<json string meta: Dictionary with metadata from the WFMS.
:status 201: rule created
:status 401: Invalid Auth Token
:status 404: DID not found
:status 409: Invalid Replication Rule
:status 409: Duplicate Replication Rule
:status 409: Insufficient Target RSEs
:status 409: Insufficient Account Limit
:status 409: Invalid RSE Expression
:status 409: Replication Rule Creation Temporary Failed
:status 409: Invalid Rule Weight
:status 409: Staging Area Rule Requires Lifetime
:status 409: Scratch Disk Lifetime Conflict
:status 409: Manual Rule Approval Blocked
:status 409: Invalid Object
:returns: List of ids for created rules
"""
json_data = request.data
try:
grouping, weight, lifetime, locked, subscription_id, source_replica_expression, activity, notify,\
purge_replicas, ignore_availability, comment, ask_approval, asynchronous, priority,\
split_container, meta = 'DATASET', None, None, False, None, None, None, None, False, False, None,\
False, False, 3, False, None
params = loads(json_data)
dids = params['dids']
account = params['account']
copies = params['copies']
rse_expression = params['rse_expression']
if 'grouping' in params:
grouping = params['grouping']
if 'weight' in params:
weight = params['weight']
if 'lifetime' in params:
lifetime = params['lifetime']
if 'locked' in params:
locked = params['locked']
if 'subscription_id' in params:
subscription_id = params['subscription_id']
if 'source_replica_expression' in params:
source_replica_expression = params['source_replica_expression']
if 'activity' in params:
activity = params['activity']
if 'notify' in params:
notify = params['notify']
if 'purge_replicas' in params:
purge_replicas = params['purge_replicas']
if 'ignore_availability' in params:
ignore_availability = params['ignore_availability']
if 'comment' in params:
comment = params['comment']
if 'ask_approval' in params:
ask_approval = params['ask_approval']
if 'asynchronous' in params:
asynchronous = params['asynchronous']
if 'priority' in params:
priority = params['priority']
if 'split_container' in params:
split_container = params['split_container']
if 'meta' in params:
meta = params['meta']
except ValueError:
return generate_http_error_flask(400, 'ValueError', 'Cannot decode json parameter list')
try:
rule_ids = add_replication_rule(dids=dids,
copies=copies,
rse_expression=rse_expression,
weight=weight,
lifetime=lifetime,
grouping=grouping,
account=account,
locked=locked,
subscription_id=subscription_id,
source_replica_expression=source_replica_expression,
activity=activity,
notify=notify,
purge_replicas=purge_replicas,
ignore_availability=ignore_availability,
comment=comment,
ask_approval=ask_approval,
asynchronous=asynchronous,
priority=priority,
split_container=split_container,
meta=meta,
issuer=request.environ.get('issuer'),
vo=request.environ.get('vo'))
# TODO: Add all other error cases here
except InvalidReplicationRule as error:
return generate_http_error_flask(409, 'InvalidReplicationRule', error.args[0])
except DuplicateRule as error:
return generate_http_error_flask(409, 'DuplicateRule', error.args[0])
except InsufficientTargetRSEs as error:
return generate_http_error_flask(409, 'InsufficientTargetRSEs', error.args[0])
except InsufficientAccountLimit as error:
return generate_http_error_flask(409, 'InsufficientAccountLimit', error.args[0])
except InvalidRSEExpression as error:
return generate_http_error_flask(409, 'InvalidRSEExpression', error.args[0])
except DataIdentifierNotFound as error:
return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0])
except ReplicationRuleCreationTemporaryFailed as error:
return generate_http_error_flask(409, 'ReplicationRuleCreationTemporaryFailed', error.args[0])
except InvalidRuleWeight as error:
return generate_http_error_flask(409, 'InvalidRuleWeight', error.args[0])
except StagingAreaRuleRequiresLifetime as error:
return generate_http_error_flask(409, 'StagingAreaRuleRequiresLifetime', error.args[0])
except ScratchDiskLifetimeConflict as error:
return generate_http_error_flask(409, 'ScratchDiskLifetimeConflict', error.args[0])
except ManualRuleApprovalBlocked as error:
return generate_http_error_flask(409, 'ManualRuleApprovalBlocked', error.args[0])
except InvalidObject as error:
return generate_http_error_flask(409, 'InvalidObject', error.args[0])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
except Exception as error:
print(error)
print(format_exc())
return error, 500
return Response(dumps(rule_ids), status=201)
def get(self, scope, name):
"""
Metalink redirect
.. :quickref: MetaLinkRedirector; Metalink redirect.
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
500 InternalError
404 Notfound
:param scope: The scope name of the file.
:param name: The name of the file.
:resheader Content-Type: application/metalink4+xml'.
:status 200: OK.
:status 401: Invalid Auth Token.
:status 404: RSE Not Found.
:status 404: DID Not Found.
:status 406: Not Acceptable.
:status 500: Internal Error.
:returns: Metalink file
"""
dids = [{'scope': scope, 'name': name}]
# set the correct client IP
client_ip = request.environ.get('HTTP_X_FORWARDED_FOR')
if client_ip is None:
client_ip = request.remote_addr
client_location = {'ip': client_ip, 'fqdn': None, 'site': None}
schemes = request.args.get(
'schemes', ['http', 'https', 'root', 'gsiftp', 'srm', 'davs'])
select = request.args.get('select', None)
if 'sort' in request.args:
select = request.args['sort']
client_location['ip'] = request.args.get('ip', None)
client_location['fqdn'] = request.args.get('fqdn', None)
client_location['site'] = request.args.get('site', None)
try:
tmp_replicas = [
rep for rep in list_replicas(dids=dids,
schemes=schemes,
client_location=client_location)
]
if not tmp_replicas:
return 'no redirection possible - cannot find the DID', 404
# first, set the appropriate content type, and stream the header
data = '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n'
# iteratively stream the XML per file
for rfile in tmp_replicas:
replicas = []
dictreplica = {}
for rse in rfile['rses']:
for replica in rfile['rses'][rse]:
replicas.append(replica)
dictreplica[replica] = rse
# stream metadata
data += ' <file name="' + rfile['name'] + '">\n'
data += ' <identity>' + rfile['scope'] + ':' + rfile[
'name'] + '</identity>\n'
if rfile['adler32'] is not None:
data += ' <hash type="adler32">' + rfile[
'adler32'] + '</hash>\n'
if rfile['md5'] is not None:
data += ' <hash type="md5">' + rfile['md5'] + '</hash>\n'
data += ' <size>' + str(rfile['bytes']) + '</size>\n'
data += ' <glfn name="/atlas/rucio/%s:%s">' % (rfile['scope'],
rfile['name'])
data += '</glfn>\n'
# sort the actual replicas if necessary
if select == 'geoip':
replicas = sort_geoip(dictreplica,
client_location['ip'],
ignore_error=True)
elif select == 'closeness':
replicas = sort_closeness(dictreplica, client_location)
elif select == 'dynamic':
replicas = sort_dynamic(dictreplica, client_location)
elif select == 'ranking':
replicas = sort_ranking(dictreplica, client_location)
else:
replicas = sort_random(dictreplica)
# stream URLs
idx = 1
for replica in replicas:
data += ' <url location="' + str(
dictreplica[replica]) + '" priority="' + str(
idx) + '">' + replica + '</url>\n'
idx += 1
data += ' </file>\n'
# don't forget to send the metalink footer
data += '</metalink>\n'
return Response(data, content_type='application/metalink4+xml')
except DataIdentifierNotFound as error:
return generate_http_error_flask(404, 'DataIdentifierNotFound',
error.args[0])
except ReplicaNotFound as error:
return generate_http_error_flask(404, 'ReplicaNotFound',
error.args[0])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
return error, 500
def get(self, scope, name):
"""
List all replicas for data identifiers.
.. :quickref: Replicas; List all replicas for did
HTTP Success:
200 OK
HTTP Error:
401 Unauthorized
500 InternalError
:reqheader HTTP_ACCEPT: application/metalink4+xml
:param scope: data identifier scope.
:param name: data identifier name.
:resheader Content-Type: application/x-json-stream
:resheader Content-Type: application/metalink4+xml
:status 200: OK.
:status 401: Invalid auth token.
:status 404: DID not found.
:status 500: Internal Error.
:returns: A dictionary containing all replicas information.
:returns: A metalink description of replicas if metalink(4)+xml is specified in Accept:
"""
metalink = False
if request.environ.get('HTTP_ACCEPT') is not None:
tmp = request.environ.get('HTTP_ACCEPT').split(',')
if 'application/metalink4+xml' in tmp:
metalink = True
dids, schemes, select, limit = [{
'scope': scope,
'name': name
}], None, None, None
schemes = request.args.get('schemes', None)
select = request.args.get('select', None)
limit = request.args.get('limit', None)
if limit:
limit = int(limit)
data = ""
content_type = 'application/x-json-stream'
try:
# first, set the appropriate content type, and stream the header
if metalink:
content_type = 'application/metalink4+xml'
data += '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n'
# then, stream the replica information
for rfile in list_replicas(dids=dids, schemes=schemes):
client_ip = request.environ.get('HTTP_X_FORWARDED_FOR')
if client_ip is None:
client_ip = request.remote_addr
replicas = []
dictreplica = {}
for rse in rfile['rses']:
for replica in rfile['rses'][rse]:
replicas.append(replica)
dictreplica[replica] = rse
if select == 'geoip':
try:
replicas = sort_geoip(dictreplica, client_ip)
except AddressNotFoundError:
pass
else:
replicas = sort_random(dictreplica)
if not metalink:
data += dumps(rfile) + '\n'
else:
data += ' <file name="' + rfile['name'] + '">\n'
data += ' <identity>' + rfile['scope'] + ':' + rfile[
'name'] + '</identity>\n'
if rfile['adler32'] is not None:
data += ' <hash type="adler32">' + rfile[
'adler32'] + '</hash>\n'
if rfile['md5'] is not None:
data += ' <hash type="md5">' + rfile[
'md5'] + '</hash>\n'
data += ' <size>' + str(rfile['bytes']) + '</size>\n'
data += ' <glfn name="/atlas/rucio/%s:%s">' % (
rfile['scope'], rfile['name'])
data += '</glfn>\n'
idx = 0
for replica in replicas:
data += ' <url location="' + str(
dictreplica[replica]) + '" priority="' + str(
idx + 1) + '">' + replica + '</url>\n'
idx += 1
if limit and limit == idx:
break
data += ' </file>\n'
# don't forget to send the metalink footer
if metalink:
data += '</metalink>\n'
return Response(data, content_type=content_type)
except DataIdentifierNotFound as error:
return generate_http_error_flask(404, 'DataIdentifierNotFound',
error.args[0])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print format_exc()
return error, 500
def get(self):
"""
List requests for a given source and destination RSE or site.
.. :quickref: RequestsGet; list requests
:reqheader Content-Type: application/x-json-stream
:status 200: Request found.
:status 404: Request not found.
:status 406: Not Acceptable.
"""
src_rse = f_request.get('src_rse')
dst_rse = f_request.get('dst_rse')
src_site = f_request.get('src_site')
dst_site = f_request.get('dst_site')
request_states = f_request.get('request_states')
if not request_states:
return generate_http_error_flask(400, 'MissingParameter',
'Request state is missing')
if src_rse and not dst_rse:
return generate_http_error_flask(400, 'MissingParameter',
'Destination RSE is missing')
elif dst_rse and not src_rse:
return generate_http_error_flask(400, 'MissingParameter',
'Source RSE is missing')
elif src_site and not dst_site:
return generate_http_error_flask(400, 'MissingParameter',
'Destination site is missing')
elif dst_site and not src_site:
return generate_http_error_flask(400, 'MissingParameter',
'Source site is missing')
try:
states = [
RequestState.from_string(state)
for state in request_states.split(',')
]
except ValueError:
return generate_http_error_flask(400, 'Invalid',
'Request state value is invalid')
src_rses = []
dst_rses = []
if src_site:
src_rses = get_rses_with_attribute_value(
key='site',
value=src_site,
lookup_key='site',
vo=f_request.environ.get('vo'))
if not src_rses:
return generate_http_error_flask(
404, 'NotFound',
'Could not resolve site name %s to RSE' % src_site)
src_rses = [get_rse_name(rse['rse_id']) for rse in src_rses]
dst_rses = get_rses_with_attribute_value(
key='site',
value=dst_site,
lookup_key='site',
vo=f_request.environ.get('vo'))
if not dst_rses:
return generate_http_error_flask(
404, 'NotFound',
'Could not resolve site name %s to RSE' % dst_site)
dst_rses = [get_rse_name(rse['rse_id']) for rse in dst_rses]
else:
dst_rses = [dst_rse]
src_rses = [src_rse]
results = []
for result in request.list_requests(
src_rses,
dst_rses,
states,
issuer=f_request.environ.get('issuer'),
vo=f_request.environ.get('vo')):
del result['_sa_instance_state']
results.append(result)
return json.dumps(results, cls=APIEncoder)
def get(self):
"""
Sign a URL for a limited lifetime for a particular service.
:reqheader X-Rucio-VO: VO name as a string (Multi-VO only).
:reqheader X-Rucio-Account: Account identifier as a string.
:reqheader X-Rucio-AppID: Application identifier as a string.
:resheader Access-Control-Allow-Origin:
:resheader Access-Control-Allow-Headers:
:resheader Access-Control-Allow-Methods:
:resheader Access-Control-Allow-Credentials:
:resheader Access-Control-Expose-Headers:
:resheader X-Rucio-Auth-Token: The authentication token
:status 200: Successfully signed URL
:status 400: Bad Request
:status 401: Unauthorized
:status 406: Not Acceptable
:status 500: Internal Server Error
"""
response = Response()
response.headers['Access-Control-Allow-Origin'] = request.environ.get(
'HTTP_ORIGIN')
response.headers['Access-Control-Allow-Headers'] = request.environ.get(
'HTTP_ACCESS_CONTROL_REQUEST_HEADERS')
response.headers['Access-Control-Allow-Methods'] = '*'
response.headers['Access-Control-Allow-Credentials'] = 'true'
response.headers[
'Access-Control-Expose-Headers'] = 'X-Rucio-Auth-Token'
response.headers['Content-Type'] = 'application/octet-stream'
response.headers[
'Cache-Control'] = 'no-cache, no-store, max-age=0, must-revalidate'
response.headers['Cache-Control'] = 'post-check=0, pre-check=0'
response.headers['Pragma'] = 'no-cache'
vo = request.environ.get('HTTP_X_RUCIO_VO', 'def')
account = request.environ.get('HTTP_X_RUCIO_ACCOUNT')
appid = request.environ.get('HTTP_X_RUCIO_APPID')
if appid is None:
appid = 'unknown'
ip = request.environ.get('HTTP_X_FORWARDED_FOR')
if ip is None:
ip = request.remote_addr
try:
validate_auth_token(request.environ.get('HTTP_X_RUCIO_AUTH_TOKEN'))
except AccessDenied:
return generate_http_error_flask(
401, 'CannotAuthenticate',
'Cannot authenticate to account %(account)s with given credentials'
% locals())
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
return error, 500
rse, svc, operation, url = None, None, None, None
try:
params = parse_qs(request.query[1:])
lifetime = params.get('lifetime', [600])[0]
service = params.get('svc', ['gcs'])[0]
operation = params.get('op', ['read'])[0]
url = params.get('url', [None])[0]
rse = params.get('rse', [None])[0]
except ValueError:
return generate_http_error_flask(
400, 'ValueError', 'Cannot decode json parameter list')
if service not in ['gcs', 's3', 'swift']:
return generate_http_error_flask(
400, 'ValueError',
'Parameter "svc" must be either empty(=gcs), gcs, s3 or swift')
if url is None:
return generate_http_error_flask(400, 'ValueError',
'Parameter "url" not found')
if rse is None:
return generate_http_error_flask(400, 'ValueError',
'Parameter "rse" not found')
if operation not in ['read', 'write', 'delete']:
return generate_http_error_flask(
400, 'ValueError',
'Parameter "op" must be either empty(=read), read, write, or delete.'
)
try:
result = get_signed_url(account,
appid,
ip,
rse=rse,
service=service,
operation=operation,
url=url,
lifetime=lifetime,
vo=vo)
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
return error, 500
if not result:
return generate_http_error_flask(
401, 'CannotAuthenticate',
'Cannot generate signed URL for account %(account)s' %
locals())
return response
def put(self, rse, scheme, hostname=None, port=None):
"""
Updates attributes of an existing protocol entry. Because protocol identifier, hostname,
and port are used as unique identifier they are immutable.
.. :quickref: Protocol; Update RSE protocol.
:param rse: The RSE name.
:param scheme: The protocol identifier.
:param hostname: The hostname defined for the scheme, used if more than one scheme is registered with the same identifier.
:param port: The port registered for the hostname, ued if more than one scheme is registered with the same identifier and hostname.
:<json dict paramaters: parameter of the new protocol entry.
:status 201: Created.
:status 400: Cannot decode json parameter dictionary.
:status 401: Invalid Auth Token.
:status 404: RSE not found.
:status 404: RSE Protocol Not Supported.
:status 404: RSE Protocol Domain Not Supported.
:status 409: RSE Protocol Priority Error.
:status 500: Internal Error.
"""
json_data = request.data
try:
parameter = loads(json_data)
except ValueError:
return generate_http_error_flask(
400, 'ValueError', 'Cannot decode json parameter dictionary')
try:
update_protocols(rse,
issuer=request.environ.get('issuer'),
vo=request.environ.get('vo'),
scheme=scheme,
hostname=hostname,
port=port,
data=parameter)
except InvalidObject as error:
return generate_http_error_flask(400, 'InvalidObject',
error.args[0])
except RSEProtocolNotSupported as error:
return generate_http_error_flask(404, 'RSEProtocolNotSupported',
error.args[0])
except RSENotFound as error:
return generate_http_error_flask(404, 'RSENotFound', error.args[0])
except RSEProtocolDomainNotSupported as error:
return generate_http_error_flask(404,
'RSEProtocolDomainNotSupported',
error.args[0])
except RSEProtocolPriorityError as error:
return generate_http_error_flask(409, 'RSEProtocolPriorityError',
error.args[0])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(error)
print(format_exc())
return error, 500
return "OK", 200
def get(self):
"""
Authenticate a Rucio account temporarily via SSH key exchange.
.. :quickref: SSH; Authenticate with SSH key exchange.
:reqheader Rucio-Account: Account identifier as a string.
:reqheader Rucio-SSH-Signature: Response to server challenge signed with SSH private key as a base64 encoded string.
:reqheader Rucio-AppID: Application identifier as a string.
:resheader Access-Control-Allow-Origin:
:resheader Access-Control-Allow-Headers:
:resheader Access-Control-Allow-Methods:
:resheader Access-Control-Allow-Credentials:
:resheader Access-Control-Expose-Headers:
:resheader X-Rucio-Auth-Token: The authentication token
:status 200: Successfully authenticated
:status 404: Invalid credentials
"""
response = Response()
response.headers['Access-Control-Allow-Origin'] = request.environ.get(
'HTTP_ORIGIN')
response.headers['Access-Control-Allow-Headers'] = request.environ.get(
'HTTP_ACCESS_CONTROL_REQUEST_HEADERS')
response.headers['Access-Control-Allow-Methods'] = '*'
response.headers['Access-Control-Allow-Credentials'] = 'true'
response.headers[
'Access-Control-Expose-Headers'] = 'X-Rucio-Auth-Token'
response.headers['Content-Type'] = 'application/octet-stream'
response.headers[
'Cache-Control'] = 'no-cache, no-store, max-age=0, must-revalidate'
response.headers['Cache-Control'] = 'post-check=0, pre-check=0'
response.headers['Pragma'] = 'no-cache'
account = request.environ.get('HTTP_X_RUCIO_ACCOUNT')
signature = request.environ.get('HTTP_X_RUCIO_SSH_SIGNATURE')
appid = request.environ.get('HTTP_X_RUCIO_APPID')
if appid is None:
appid = 'unknown'
ip = request.environ.get('HTTP_X_FORWARDED_FOR')
if ip is None:
ip = request.remote_addr
# decode the signature which must come in base64 encoded
try:
signature = base64.b64decode(signature)
except Exception as error: # noqa: F841
return generate_http_error_flask(
401, 'CannotAuthenticate',
'Cannot authenticate to account %(account)s with malformed signature'
% locals())
try:
result = get_auth_token_ssh(account, signature, appid, ip)
except AccessDenied:
return generate_http_error_flask(
401, 'CannotAuthenticate',
'Cannot authenticate to account %(account)s with given credentials'
% locals())
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
return error, 500
if not result:
return generate_http_error_flask(
401, 'CannotAuthenticate',
'Cannot authenticate to account %(account)s with given credentials'
% locals())
response.headers['X-Rucio-Auth-Token'] = result
response.set_data(str())
return response
def post(self, scope, name):
"""
Append data identifiers to data identifiers.
.. :quickref: Attachment; Append DID to DID.
**Example request**:
.. sourcecode:: http
POST /dids/scope1/datasets1/dids HTTP/1.1
Host: rucio.cern.ch
[{"scope": "scope1", "name": "file1"},
{"scope": "scope1", "name": "file2"},
{"scope": "scope1", "name": "file3"}]
**Example response**:
.. sourcecode:: http
HTTP/1.1 201 Created
Vary: Accept
:param scope: The scope of the DID to attach to.
:param name: The name of the DID to attach to.
:<json list attachments: List of dicts of DIDs to attach.
:status 201: DIDs successfully attached
:status 401: Invalid Auth Token
:status 404: DID not found
:status 409: DIDs already attached
:status 500: Database Exception
"""
try:
json_data = loads(request.data)
except ValueError:
return generate_http_error_flask(
400, 'ValueError', 'Cannot decode json parameter list')
try:
attach_dids(scope=scope,
name=name,
attachment=json_data,
issuer=request.environ.get('issuer'))
except DataIdentifierNotFound as error:
return generate_http_error_flask(404, 'DataIdentifierNotFound',
error.args[0])
except DuplicateContent as error:
return generate_http_error_flask(409, 'DuplicateContent',
error.args[0])
except AccessDenied as error:
return generate_http_error_flask(401, 'AccessDenied',
error.args[0])
except UnsupportedOperation as error:
return generate_http_error_flask(409, 'UnsupportedOperation',
error.args[0])
except RSENotFound as error:
return generate_http_error_flask(404, 'RSENotFound', error.args[0])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
return error, 500
return "Created", 201
def get(self, rse, scheme=None):
"""
Return PFNs for a set of LFNs. Formatted as a JSON object where the key is a LFN and the
value is the corresponding PFN.
.. :quickref: Attributes; Translate LFNs to PFNs.
:param rse: The RSE name.
:param scheme: The protocol identifier.
:query lfn: One or moref LFN to translate.
:query scheme: Optional argument to help with the protocol selection (e.g., http / gsiftp / srm)
:query domain: Optional argument used to select the protocol for wan or lan use cases.
:query operation: Optional query argument to select the protoco for read-vs-writes.
:resheader Content-Type: application/json
:status 200: OK.
:status 401: Invalid Auth Token.
:status 404: RSE Not Found.
:status 404: RSE Protocol Not Supported.
:status 404: RSE Protocol Domain Not Supported.
:status 406: Not Acceptable.
:status 500: Internal Error.
:returns: A list with detailed PFN information.
"""
lfns = []
scheme = request.get('scheme', None)
domain = request.get('domain', 'wan')
operation = request.get('operation', 'write')
p_lfns = request.get('lfn', None)
if p_lfns:
info = p_lfns.split(":", 1)
if len(info) != 2:
return generate_http_error_flask(400, 'InvalidPath',
'LFN in invalid format')
lfn_dict = {'scope': info[0], 'name': info[1]}
lfns.append(lfn_dict)
rse_settings = None
try:
rse_settings = get_rse_protocols(
rse,
issuer=request.environ.get('issuer'),
vo=request.environ.get('vo'))
except RSENotFound as error:
return generate_http_error_flask(404, 'RSENotFound', error.args[0])
except RSEProtocolNotSupported as error:
return generate_http_error_flask(404, 'RSEProtocolNotSupported',
error.args[0])
except RSEProtocolDomainNotSupported as error:
return generate_http_error_flask(404,
'RSEProtocolDomainNotSupported',
error.args[0])
except Exception as error:
print(error)
print(format_exc())
return error, 500
pfns = rsemanager.lfns2pfns(rse_settings,
lfns,
operation=operation,
scheme=scheme,
domain=domain)
return Response(dumps(pfns), content_type="application/json")
def put(self, scope, name):
"""
Update data identifier status.
.. :quickref: DIDs; Update DID status.
.. sourcecode:: http
PUT /dids/scope1/container1 HTTP/1.1
Host: rucio.cern.ch
{"open": False},
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
:param scope: data identifier scope.
:param name: data identifier name.
:<json bool open: open or close did
:status 200: DIDs successfully updated
:status 401: Invalid Auth Token
:status 404: DID not found
:status 409: Wrong status
:status 500: Database Exception
"""
json_data = request.data
try:
kwargs = loads(json_data)
except ValueError:
return generate_http_error_flask(
400, 'ValueError', 'Cannot decode json data parameter')
try:
set_status(scope=scope,
name=name,
issuer=request.environ.get('issuer'),
**kwargs)
except DataIdentifierNotFound as error:
return generate_http_error_flask(404, 'DataIdentifierNotFound',
error.args[0])
except UnsupportedStatus as error:
return generate_http_error_flask(409, 'UnsupportedStatus',
error.args[0])
except UnsupportedOperation as error:
return generate_http_error_flask(409, 'UnsupportedOperation',
error.args[0])
except AccessDenied as error:
return generate_http_error_flask(401, 'AccessDenied',
error.args[0])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
return error, 500
return "Ok", 200
def post(self, scope, name):
"""
Create a new data identifier.
.. :quickref: DIDs; Create a new DID.
.. sourcecode:: http
POST /dids/scope1/container1 HTTP/1.1
Host: rucio.cern.ch
{"type": "CONTAINER", "lifetime": 86400},
**Example response**:
.. sourcecode:: http
HTTP/1.1 201 Created
Vary: Accept
:reqheader Accept: application/json
:param scope: data identifier scope.
:param name: data identifier name.
:<json string type: the new DID type
:<json dict statuses: Dictionary with statuses, e.g. {'monotonic':True}
:<json dict meta: Dictionary with metadata, e.g. {'length':1234}
:<json dict rules: Replication rules associated with the did. e.g., [{'copies': 2, 'rse_expression': 'TIERS1'}, ]
:<json int lifetime: DID's liftime in seconds.
:<json list dids: The content.
:<json string rse: The RSE name when registering replicas.
:status 201: new DIDs created
:status 401: Invalid Auth Token
:status 409: DID already exists
:status 500: Database Exception
"""
statuses, meta, rules, lifetime, dids, rse = {}, {}, [], None, [], None
try:
json_data = loads(request.data)
type = json_data['type']
if 'statuses' in json_data:
statuses = json_data['statuses']
if 'meta' in json_data:
meta = json_data['meta']
if 'rules' in json_data:
rules = json_data['rules']
if 'lifetime' in json_data:
lifetime = json_data['lifetime']
if 'dids' in json_data:
dids = json_data['dids']
if 'rse' in json_data:
rse = json_data['rse']
except ValueError:
return generate_http_error_flask(
400, 'ValueError', 'Cannot decode json parameter list')
except KeyError as error:
return generate_http_error_flask(400, 'ValueError', str(error))
try:
add_did(scope=scope,
name=name,
type=type,
statuses=statuses,
meta=meta,
rules=rules,
lifetime=lifetime,
dids=dids,
rse=rse,
issuer=request.environ.get('issuer'))
except DataIdentifierNotFound as error:
return generate_http_error_flask(404, 'DataIdentifierNotFound',
error.args[0])
except DuplicateContent as error:
return generate_http_error_flask(409, 'DuplicateContent',
error.args[0])
except DataIdentifierAlreadyExists as error:
return generate_http_error_flask(409,
'DataIdentifierAlreadyExists',
error.args[0])
except AccessDenied as error:
return generate_http_error_flask(401, 'AccessDenied',
error.args[0])
except UnsupportedOperation as error:
return generate_http_error_flask(409, 'UnsupportedOperation',
error.args[0])
except DatabaseException as error:
return generate_http_error_flask(500, 'DatabaseException',
error.args[0])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
return error, 500
return "Created", 201
def post(self):
"""
Add new DIDs in bulk.
.. :quickref: BulkDID; Bulk add DIDs.
**Example request**:
.. sourcecode:: http
POST /dids/ HTTP/1.1
Host: rucio.cern.ch
[
{"scope": "scope1", "type": "CONTAINER", "name": "container1",
"account": "jdoe", "length": 1},
{"scope": "scope1", "type": "DATASET", "name": "dataset1",
"account": "jdoe", "length": 3}
]
**Example response**:
.. sourcecode:: http
HTTP/1.1 201 Created
Vary: Accept
:reqheader Accept: application/json
:<json string scope: the new DID scope
:<json string name: the new DID name
:<json string type: the new DID type
:<json string account: the owner account of the new DID
:<json string statuses: monotonic
:status 201: new DIDs created
:status 401: Invalid Auth Token
:status 409: DID already exists
:status 500: Database Exception
"""
try:
json_data = loads(request.data)
except ValueError:
return generate_http_error_flask(
400, 'ValueError', 'Cannot decode json parameter list')
try:
print(json_data)
add_dids(json_data, issuer=request.environ.get('issuer'))
except DataIdentifierNotFound as error:
return generate_http_error_flask(404, 'DataIdentifierNotFound',
error.args[0])
except DuplicateContent as error:
return generate_http_error_flask(409, 'DuplicateContent',
error.args[0])
except DataIdentifierAlreadyExists as error:
return generate_http_error_flask(409,
'DataIdentifierAlreadyExists',
error.args[0])
except AccessDenied as error:
return generate_http_error_flask(401, 'AccessDenied',
error.args[0])
except UnsupportedOperation as error:
return generate_http_error_flask(409, 'UnsupportedOperation',
error.args[0])
except DatabaseException as error:
return generate_http_error_flask(500, 'DatabaseException',
error.args[0])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
return error, 500
return "Created", 201
def get(self):
"""
Authenticate a Rucio account temporarily via a GSS token.
.. :quickref: GSS; Authenticate with GSS token
:reqheader Rucio-Account: Account identifier as a string.
:reqheader Rucio-AppID: Application identifier as a string.
:reqheader SavedCredentials: Apache mod_auth_kerb SavedCredentials.
:resheader Access-Control-Allow-Origin:
:resheader Access-Control-Allow-Headers:
:resheader Access-Control-Allow-Methods:
:resheader Access-Control-Allow-Credentials:
:resheader Access-Control-Expose-Headers:
:resheader X-Rucio-Auth-Token: The authentication token
:status 200: Successfully authenticated
:status 404: Invalid credentials
"""
response = Response()
response.headers['Access-Control-Allow-Origin'] = request.environ.get(
'HTTP_ORIGIN')
response.headers['Access-Control-Allow-Headers'] = request.environ.get(
'HTTP_ACCESS_CONTROL_REQUEST_HEADERS')
response.headers['Access-Control-Allow-Methods'] = '*'
response.headers['Access-Control-Allow-Credentials'] = 'true'
response.headers[
'Access-Control-Expose-Headers'] = 'X-Rucio-Auth-Token'
response.headers['Content-Type'] = 'application/octet-stream'
response.headers[
'Cache-Control'] = 'no-cache, no-store, max-age=0, must-revalidate'
response.headers['Cache-Control'] = 'post-check=0, pre-check=0'
response.headers['Pragma'] = 'no-cache'
account = request.environ.get('HTTP_X_RUCIO_ACCOUNT')
gsscred = request.environ.get('REMOTE_USER')
appid = request.environ.get('HTTP_X_RUCIO_APPID')
if appid is None:
appid = 'unknown'
ip = request.environ.get('HTTP_X_FORWARDED_FOR')
if ip is None:
ip = request.remote_addr
try:
result = get_auth_token_gss(account, gsscred, appid, ip)
except AccessDenied:
return generate_http_error_flask(
401, 'CannotAuthenticate',
'Cannot authenticate to account %(account)s with given credentials'
% locals())
if result is None:
return generate_http_error_flask(
401, 'CannotAuthenticate',
'Cannot authenticate to account %(account)s with given credentials'
% locals())
else:
response.headers('X-Rucio-Auth-Token', result)
return str()
return 'BadRequest', 400
appid, ip)
except AccessDenied:
return generate_http_error_flask(
401, 'CannotAuthenticate',
'Cannot authenticate to account %(account)s with given credentials'
% locals())
except RucioException, e:
return generate_http_error_flask(500, e.__class__.__name__,
e.args[0])
except Exception, e:
print format_exc()
return e, 500
if not result:
return generate_http_error_flask(
401, 'CannotAuthenticate',
'Cannot authenticate to account %(account)s with given credentials'
% locals())
response.headers['X-Rucio-Auth-Token'] = result
return response
class GSS(MethodView):
"""
Authenticate a Rucio account temporarily via a GSS token.
"""
def options(self):
"""
HTTP Success:
200 OK
def get(self):
"""
Authenticate a Rucio account temporarily via an x509 certificate.
.. :quickref: x509; Authenticate with x509 certificate.
:reqheader Rucio-Account: Account identifier as a string.
:reqheader Rucio-AppID: Application identifier as a string.
:reqheader SSLStdEnv: Apache mod_ssl SSL Standard Env Variables.
:resheader Access-Control-Allow-Origin:
:resheader Access-Control-Allow-Headers:
:resheader Access-Control-Allow-Methods:
:resheader Access-Control-Allow-Credentials:
:resheader Access-Control-Expose-Headers:
:resheader X-Rucio-Auth-Token: The authentication token
:status 200: Successfully authenticated
:status 404: Invalid credentials
"""
response = Response()
response.headers['Access-Control-Allow-Origin'] = request.environ.get(
'HTTP_ORIGIN')
response.headers['Access-Control-Allow-Headers'] = request.environ.get(
'HTTP_ACCESS_CONTROL_REQUEST_HEADERS')
response.headers['Access-Control-Allow-Methods'] = '*'
response.headers['Access-Control-Allow-Credentials'] = 'true'
response.headers[
'Access-Control-Expose-Headers'] = 'X-Rucio-Auth-Token'
response.headers['Content-Type'] = 'application/octet-stream'
response.headers[
'Cache-Control'] = 'no-cache, no-store, max-age=0, must-revalidate'
response.headers['Cache-Control'] = 'post-check=0, pre-check=0'
response.headers['Pragma'] = 'no-cache'
account = request.environ.get('HTTP_X_RUCIO_ACCOUNT')
dn = request.environ.get('SSL_CLIENT_S_DN')
if not dn:
return generate_http_error_flask(401, 'CannotAuthenticate',
'Cannot get DN')
if not dn.startswith('/'):
dn = '/%s' % '/'.join(dn.split(',')[::-1])
appid = request.environ.get('HTTP_X_RUCIO_APPID')
if appid is None:
appid = 'unknown'
ip = request.environ.get('HTTP_X_FORWARDED_FOR')
if ip is None:
ip = request.remote_addr
# If we get a valid proxy certificate we have to strip this postfix,
# otherwise we would have to store the proxy DN in the database as well.
# Alternative: use the SSL_CLIENT_I_DN, but that would require a separate
# endpoint as you cannot programmatically decide, by examining the SSL variables,
# if you got a proxy or regular certificate
while True:
if dn.endswith('/CN=limited proxy'):
dn = dn[:-17]
elif dn.endswith('/CN=proxy'):
dn = dn[:-9]
elif search('/CN=[0-9]*$', dn):
dn = dn.rpartition('/')[0]
else:
break
try:
result = get_auth_token_x509(account, dn, appid, ip)
except AccessDenied:
print('Cannot Authenticate', account, dn, appid, ip)
return generate_http_error_flask(
401, 'CannotAuthenticate',
'Cannot authenticate to account %(account)s with given credentials'
% locals())
except IdentityError:
print('Cannot Authenticate', account, dn, appid, ip)
return generate_http_error_flask(
401, 'CannotAuthenticate',
'No default account set for %(dn)s' % locals())
if not result:
print('Cannot Authenticate', account, dn, appid, ip)
return generate_http_error_flask(
401, 'CannotAuthenticate',
'Cannot authenticate to account %(account)s with given credentials'
% locals())
response.headers['X-Rucio-Auth-Token'] = result
response.set_data(str())
return response
except ValueError:
return generate_http_error_flask(
400, 'ValueError', 'Cannot decode json parameter list')
try:
print json_data
add_dids(json_data, issuer=request.environ.get('issuer'))
except DataIdentifierNotFound, error:
return generate_http_error_flask(404, 'DataIdentifierNotFound',
error.args[0])
except DuplicateContent, error:
return generate_http_error_flask(409, 'DuplicateContent',
error.args[0])
except DataIdentifierAlreadyExists, error:
return generate_http_error_flask(409,
'DataIdentifierAlreadyExists',
error.args[0])
except AccessDenied, error:
return generate_http_error_flask(401, 'AccessDenied',
error.args[0])
except UnsupportedOperation, error:
return generate_http_error_flask(409, 'UnsupportedOperation',
error.args[0])
except DatabaseException, error:
return generate_http_error_flask(500, 'DatabaseException',
error.args[0])
except RucioException, error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception, error:
print format_exc()
def get(self):
"""
Authenticate a Rucio account temporarily via username and password.
.. :quickref: UserPass; Authenticate with username/password
:reqheader X-Rucio-Account: Account identifier as a string.
:reqheader X-Rucio-Username: Username as a string.
:reqheader X-Rucio-Password: password as a text-plain string.
:reqheader X-Rucio-AppID: Application identifier as a string.
:resheader Access-Control-Allow-Origin:
:resheader Access-Control-Allow-Headers:
:resheader Access-Control-Allow-Methods:
:resheader Access-Control-Allow-Credentials:
:resheader Access-Control-Expose-Headers:
:resheader X-Rucio-Auth-Token: The authentication token
:status 200: Successfully authenticated
:status 404: Invalid credentials
"""
response = Response()
response.headers['Access-Control-Allow-Origin'] = request.environ.get(
'HTTP_ORIGIN')
response.headers['Access-Control-Allow-Headers'] = request.environ.get(
'HTTP_ACCESS_CONTROL_REQUEST_HEADERS')
response.headers['Access-Control-Allow-Methods'] = '*'
response.headers['Access-Control-Allow-Credentials'] = 'true'
response.headers[
'Access-Control-Expose-Headers'] = 'X-Rucio-Auth-Token'
response.headers['Content-Type'] = 'application/octet-stream'
response.headers[
'Cache-Control'] = 'no-cache, no-store, max-age=0, must-revalidate'
response.headers['Cache-Control'] = 'post-check=0, pre-check=0'
response.headers['Pragma'] = 'no-cache'
account = request.environ.get('HTTP_X_RUCIO_ACCOUNT')
username = request.environ.get('HTTP_X_RUCIO_USERNAME')
password = request.environ.get('HTTP_X_RUCIO_PASSWORD')
appid = request.environ.get('HTTP_X_RUCIO_APPID')
if appid is None:
appid = 'unknown'
ip = request.environ.get('HTTP_X_FORWARDED_FOR')
if ip is None:
ip = request.remote_addr
print(account, username, password, appid)
try:
result = get_auth_token_user_pass(account, username, password,
appid, ip)
except AccessDenied:
return generate_http_error_flask(
401, 'CannotAuthenticate',
'Cannot authenticate to account %(account)s with given credentials'
% locals())
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
return error, 500
if not result:
return generate_http_error_flask(
401, 'CannotAuthenticate',
'Cannot authenticate to account %(account)s with given credentials'
% locals())
response.headers['X-Rucio-Auth-Token'] = result
return response
class Search(MethodView):
def get(self, scope):
"""
List all data identifiers in a scope which match a given metadata.
.. :quickref: Search; Search DIDs in a scope with given metadata.
**Example request**:
.. sourcecode:: http
GET /dids/scope1/dids/search?type=collection1&long=True HTTP/1.1
Host: rucio.cern.ch
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: application/x-json-stream
{"scope": "scope1", "did_type": "CONTAINER", "name": "container1",
"bytes": 1234, "length": 1}
{"scope": "scope1", "did_type": "DATASET", "name": "dataset1",
"bytes": 234, "length": 3}
:query type: specify a DID type to search for
:query long: set to True for long output, otherwise only name
:resheader Content-Type: application/x-json-stream
:status 200: DIDs found
:status 401: Invalid Auth Token
:status 404: Invalid key in filters
:status 409: Wrong DID type
:returns: Line separated name of DIDs or dictionaries of DIDs for long option
"""
filters = {}
long = False
type = 'collection'
for k, v in request.args.items():
if k == 'type':
type = v
elif k == 'long':
long = bool(v)
else:
filters[k] = v[0]
try:
data = ""
for did in list_dids(scope=scope,
filters=filters,
type=type,
long=long):
data += dumps(did) + '\n'
return Response(data, content_type='application/x-json-stream')
except UnsupportedOperation, error:
return generate_http_error_flask(409, 'UnsupportedOperation',
error.args[0])
except KeyNotFound, error:
return generate_http_error_flask(404, 'KeyNotFound', error.args[0])
def post(self, rse):
""" Create RSE with given name.
.. :quickref: RSE; create a new RSE.
:param rse: The RSE name.
:<json bool deterministic: Boolean to know if the pfn is generated deterministically.
:<json bool volatile: Boolean for RSE cache.
:<json string city: City for the RSE.
:<json bool staging_area: Staging area.
:<json string region_code: The region code for the RSE.
:<json string country_name: The country.
:<json string continent: The continent.
:<json string time_zone: Timezone.
:<json string ISP: Internet Service Provider.
:<json string rse_type: RSE type.
:<json number latitude: Latitude coordinate of RSE.
:<json number longitude: Longitude coordinate of RSE.
:<json string ASN: Access service network.
:<json integer availability: Availability.
:status 201: RSE created successfully.
:status 400: Cannot decode json parameter dictionary.
:status 401: Invalid Auth Token.
:status 409: RSE already exists.
:status 409: RSE not found.
:status 500: Internal Error.
"""
json_data = request.data
kwargs = {
'deterministic': True,
'volatile': False,
'city': None,
'staging_area': False,
'region_code': None,
'country_name': None,
'continent': None,
'time_zone': None,
'ISP': None,
'rse_type': None,
'latitude': None,
'longitude': None,
'ASN': None,
'availability': None
}
try:
parameters = json_data and loads(json_data)
if parameters:
for param in kwargs:
if param in parameters:
kwargs[param] = parameters[param]
except ValueError:
return generate_http_error_flask(
400, 'ValueError', 'Cannot decode json parameter dictionary')
kwargs['issuer'] = request.environ.get('issuer')
try:
add_rse(rse, **kwargs)
except InvalidObject as error:
return generate_http_error_flask(400, 'InvalidObject',
error.args[0])
except AccessDenied as error:
return generate_http_error_flask(401, 'AccessDenied',
error.args[0])
except RSENotFound as error:
return generate_http_error_flask(404, 'RSENotFound', error.args[0])
except Duplicate as error:
return generate_http_error_flask(409, 'Duplicate', error.args[0])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(error)
print(format_exc())
return error, 500
return "Created", 201
return generate_http_error_flask(
400, 'ValueError', 'Cannot decode json parameter list')
try:
add_key(key=key,
key_type=key_type,
value_type=value_type,
value_regexp=value_regexp,
issuer=request.environ.get('issuer'))
except Duplicate, e:
return generate_http_error_flask(409, 'Duplicate', e[0][0])
except UnsupportedValueType, e:
return generate_http_error_flask(400, 'UnsupportedValueType',
e[0][0])
except RucioException, e:
return generate_http_error_flask(500, e.__class__.__name__,
e.args[0][0])
except Exception, e:
print e
return e, 500
return "Created", 201
class Values(MethodView):
""" REST APIs for data identifier attribute values. """
def get(self, key):
"""
List all values for a key.
.. :quickref: Values; List all key values.
if param in parameters:
kwargs[param] = parameters[param]
except ValueError:
return generate_http_error_flask(
400, 'ValueError', 'Cannot decode json parameter dictionary')
kwargs['issuer'] = request.environ.get('issuer')
try:
add_rse(rse, **kwargs)
except InvalidObject, error:
return generate_http_error_flask(400, 'InvalidObject',
error.args[0])
except AccessDenied, error:
return generate_http_error_flask(401, 'AccessDenied',
error.args[0])
except RSENotFound, error:
return generate_http_error_flask(404, 'RSENotFound', error.args[0])
except Duplicate, error:
return generate_http_error_flask(409, 'Duplicate', error.args[0])
except RucioException, error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception, error:
print error
print format_exc()
return error, 500
return "Created", 201
def put(self, rse):
""" Update RSE properties (e.g. name, availability).
:status 201: scope created
:status 404: account does not exist
:status 401: unauthorized
:status 409: scope already exists
:status 500: internal server error
"""
try:
add_scope(scope, account, issuer=request.environ.get('issuer'))
except Duplicate, e:
return generate_http_error_flask(409, 'Duplicate', e.args[0][0])
except AccountNotFound, e:
return generate_http_error_flask(404, 'AccountNotFound',
e.args[0][0])
except RucioException, e:
print(e)
return generate_http_error_flask(500, e.__class__.__name__,
e.args[0][0])
except Exception, e:
return e, 500
return "OK", 201
bp = Blueprint('scope', __name__)
scope_view = Scope.as_view('scope')
bp.add_url_rule('/', view_func=scope_view, methods=[
'GET',
])
bp.add_url_rule('/<account>/<scope>', view_func=scope_view, methods=[
'POST',
])
def post(self):
"""
List all replicas for data identifiers.
.. :quickref: ListReplicas; List all replicas for did.
:reqheader HTTP_ACCEPT: application/metalink4+xml
:query schemes: A list of schemes to filter the replicas.
:query sort: Requested sorting of the result, e.g., 'geoip', 'closeness', 'dynamic', 'ranking'.
:<json list dids: list of DIDs.
:<json list schemes: A list of schemes to filter the replicas.
:<json bool unavailable: Also include unavailable replicas.
:<json bool all_states: Return all replicas whatever state they are in. Adds an extra 'states' entry in the result dictionary.
:<json string rse_expression: The RSE expression to restrict on a list of RSEs.
:<json dict client_location: Client location dictionary for PFN modification {'ip', 'fqdn', 'site'}.
:<json bool sort: Requested sorting of the result, e.g., 'geoip', 'closeness', 'dynamic', 'ranking'.
:<json string domain: The network domain for the call, either None, 'wan' or 'lan'. None is fallback to 'wan', 'all' is both ['lan','wan']
:resheader Content-Type: application/x-json-stream
:resheader Content-Type: application/metalink4+xml
:status 200: OK.
:status 400: Cannot decode json parameter list.
:status 401: Invalid auth token.
:status 404: DID not found.
:status 500: Internal Error.
:returns: A dictionary containing all replicas information.
:returns: A metalink description of replicas if metalink(4)+xml is specified in Accept:
"""
metalink = False
if request.environ.get('HTTP_ACCEPT') is not None:
tmp = request.environ.get('HTTP_ACCEPT').split(',')
if 'application/metalink4+xml' in tmp:
metalink = True
client_ip = request.environ.get('HTTP_X_FORWARDED_FOR')
if client_ip is None:
client_ip = request.remote_addr
dids, schemes, select, unavailable, limit = [], None, None, False, None
ignore_availability, rse_expression, all_states = False, None, False
client_location = {}
json_data = request.data
try:
params = parse_response(json_data)
if 'dids' in params:
dids = params['dids']
if 'schemes' in params:
schemes = params['schemes']
if 'unavailable' in params:
unavailable = params['unavailable']
ignore_availability = True
if 'all_states' in params:
all_states = params['all_states']
if 'rse_expression' in params:
rse_expression = params['rse_expression']
if 'client_location' in params:
client_location = params['client_location']
client_location['ip'] = params['client_location'].get(
'ip', client_ip)
if 'sort' in params:
select = params['sort']
if 'domain' in params:
domain = params['domain']
except ValueError:
return generate_http_error_flask(
400, 'ValueError', 'Cannot decode json parameter list')
schemes = request.args.get('schemes', None)
select = request.args.get('select', None)
select = request.args.get('sort', None)
data = ""
content_type = 'application/x-json-stream'
try:
# first, set the appropriate content type, and stream the header
if metalink:
content_type = 'application/metalink4+xml'
data += '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n'
# then, stream the replica information
for rfile in list_replicas(
dids=dids,
schemes=schemes,
unavailable=unavailable,
request_id=request.environ.get('request_id'),
ignore_availability=ignore_availability,
all_states=all_states,
rse_expression=rse_expression,
client_location=client_location,
domain=domain):
replicas = []
dictreplica = {}
for rse in rfile['rses']:
for replica in rfile['rses'][rse]:
replicas.append(replica)
dictreplica[replica] = rse
if not metalink:
data += dumps(rfile, cls=APIEncoder) + '\n'
else:
data += ' <file name="' + rfile['name'] + '">\n'
data += ' <identity>' + rfile['scope'] + ':' + rfile[
'name'] + '</identity>\n'
if rfile['adler32'] is not None:
data += ' <hash type="adler32">' + rfile[
'adler32'] + '</hash>\n'
if rfile['md5'] is not None:
data += ' <hash type="md5">' + rfile[
'md5'] + '</hash>\n'
data += ' <size>' + str(rfile['bytes']) + '</size>\n'
data += ' <glfn name="/atlas/rucio/%s:%s">' % (
rfile['scope'], rfile['name'])
data += '</glfn>\n'
if select == 'geoip':
replicas = sort_geoip(dictreplica,
client_location['ip'])
elif select == 'closeness':
replicas = sort_closeness(dictreplica, client_location)
elif select == 'dynamic':
replicas = sort_dynamic(dictreplica, client_location)
elif select == 'ranking':
replicas = sort_ranking(dictreplica, client_location)
else:
replicas = sort_random(dictreplica)
idx = 0
for replica in replicas:
data += ' <url location="' + str(
dictreplica[replica]) + '" priority="' + str(
idx + 1) + '">' + replica + '</url>\n'
idx += 1
if limit and limit == idx:
break
data += ' </file>\n'
# don't forget to send the metalink footer
if metalink:
data += '</metalink>\n'
return Response(data, content_type=content_type)
except DataIdentifierNotFound as error:
return generate_http_error_flask(404, 'DataIdentifierNotFound',
error.args[0])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print format_exc()
return error, 500
def before_request():
if request.environ.get('REQUEST_METHOD') == 'OPTIONS':
raise "", 200
auth_token = request.environ.get('HTTP_X_RUCIO_AUTH_TOKEN')
try:
auth = validate_auth_token(auth_token)
except RucioException, e:
return generate_http_error_flask(500, e.__class__.__name__, e.args[0][0])
except Exception, e:
print format_exc()
return e, 500
if auth is None:
return generate_http_error_flask(401, 'CannotAuthenticate', 'Cannot authenticate with given credentials')
request.environ['issuer'] = auth.get('account')
request.environ['identity'] = auth.get('identity')
request.environ['request_id'] = generate_uuid()
request.environ['start_time'] = time()
def after_request(response):
response.headers['Access-Control-Allow-Origin'] = request.environ.get('HTTP_ORIGIN')
response.headers['Access-Control-Allow-Headers'] = request.environ.get('HTTP_ACCESS_CONTROL_REQUEST_HEADERS')
response.headers['Access-Control-Allow-Methods'] = '*'
response.headers['Access-Control-Allow-Credentials'] = 'true'
if request.environ.get('REQUEST_METHOD') == 'GET':
response.headers['Cache-Control'] = 'no-cache, no-store, max-age=0, must-revalidate'
def get(self, scope, name):
"""
Header Redirect
.. :quickref: HeaderRedirector; Header redirect.
:param scope: The scope name of the file.
:param name: The name of the file.
:resheader Content-Type: application/metalink+xml'.
:status 303: Redirect.
:status 401: Invalid Auth Token.
:status 404: RSE Not Found.
:status 404: DID Not Found.
:status 500: Internal Error.
"""
headers = {}
try:
# use the default HTTP protocols if no scheme is given
client_ip = request.environ.get('HTTP_X_FORWARDED_FOR')
if client_ip is None:
client_ip = request.remote_addr
client_location = {'ip': client_ip, 'fqdn': None, 'site': None}
schemes = request.args.get('schemes', ['davs', 'https', 's3'])
select = request.args.get('select', 'random')
if 'sort' in request.args:
select = request.args['sort']
rse = request.args.get('rse', None)
site = request.args.get('site', None)
client_location['ip'] = request.args.get('ip', client_ip)
client_location['fqdn'] = request.args.get('fqdn', None)
client_location['site'] = request.args.get('site', None)
# correctly forward the schemes and select to potential metalink followups
cleaned_url = request.environ.get('REQUEST_URI').split('?')[0]
if isinstance(schemes, list):
headers[
'Link'] = '<%s/metalink?schemes=%s&select=%s>; rel=describedby; type="application/metalink+xml"' % (
cleaned_url, ','.join(schemes), select)
else:
headers[
'Link'] = '<%s/metalink?schemes=%s&select=%s>; rel=describedby; type="application/metalink+xml"' % (
cleaned_url, schemes, select)
schemes = [schemes] # list_replicas needs a list
replicas = [
r for r in list_replicas(dids=[{
'scope': scope,
'name': name,
'type': 'FILE'
}],
schemes=schemes,
client_location=client_location)
]
selected_url, selected_rse = None, None
for r in replicas:
if r['rses']:
dictreplica = {}
if rse:
if rse in r['rses'] and r['rses'][rse]:
selected_url = r['rses'][rse][0]
selected_rse = rse
else:
return 'no redirection possible - no valid RSE for HTTP redirection found', 404
else:
for rep in r['rses']:
for replica in r['rses'][rep]:
# since this is HTTP-only redirection, and to ensure compatibility with as many http clients as possible
# forcibly replacement davs and s3 URLs to https
replica = replica.replace(
'davs://',
'https://').replace('s3://', 'https://')
dictreplica[replica] = rep
if not dictreplica:
return 'no redirection possible - no valid RSE for HTTP redirection found', 404
elif site:
rep = site_selector(dictreplica, site)
if rep:
selected_url = rep[0]
else:
return 'no redirection possible - no valid RSE for HTTP redirection found', 404
else:
if select == 'geoip':
rep = sort_geoip(dictreplica,
client_location['ip'])
elif select == 'closeness':
rep = sort_closeness(dictreplica,
client_location)
elif select == 'dynamic':
rep = sort_dynamic(dictreplica,
client_location)
elif select == 'ranking':
rep = sort_ranking(dictreplica,
client_location)
else:
rep = sort_random(dictreplica)
selected_url = rep[0]
for rep in r['rses']:
for replica in r['rses'][rep]:
if selected_url == replica:
selected_rse = rep
if selected_url:
if selected_url.startswith('s3+rucio://'):
connect(selected_rse, selected_url)
signed_URLS = get_signed_urls([selected_url],
rse=selected_rse,
operation='read')
res = redirect(signed_URLS[selected_url], code=303)
res.header = headers
return res
res = redirect(signed_URLS[selected_url], code=303)
res.header = headers
return res
return 'no redirection possible - file does not exist', 404
except ReplicaNotFound as error:
return generate_http_error_flask(404, 'ReplicaNotFound',
error.args[0])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
return error, 500
def get(self):
"""
Request a challenge token for SSH authentication
.. :quickref: SSHChallengeToken; Request SSH Challenge Token
:reqheader Rucio-VO: VO name as a string (Multi-VO only).
:reqheader Rucio-Account: Account identifier as a string.
:reqheader Rucio-AppID: Application identifier as a string.
:resheader Access-Control-Allow-Origin:
:resheader Access-Control-Allow-Headers:
:resheader Access-Control-Allow-Methods:
:resheader Access-Control-Allow-Credentials:
:resheader Access-Control-Expose-Headers:
:resheader X-Rucio-Auth-Token: The authentication token
:status 200: Successfully authenticated
:status 404: Invalid credentials
"""
response = Response()
response.headers['Access-Control-Allow-Origin'] = request.environ.get(
'HTTP_ORIGIN')
response.headers['Access-Control-Allow-Headers'] = request.environ.get(
'HTTP_ACCESS_CONTROL_REQUEST_HEADERS')
response.headers['Access-Control-Allow-Methods'] = '*'
response.headers['Access-Control-Allow-Credentials'] = 'true'
response.headers[
'Access-Control-Expose-Headers'] = 'X-Rucio-Auth-Token'
response.headers['Content-Type'] = 'application/octet-stream'
response.headers[
'Cache-Control'] = 'no-cache, no-store, max-age=0, must-revalidate'
response.headers['Cache-Control'] = 'post-check=0, pre-check=0'
response.headers['Pragma'] = 'no-cache'
vo = request.environ.get('HTTP_X_RUCIO_VO', 'def')
account = request.environ.get('HTTP_X_RUCIO_ACCOUNT')
appid = request.environ.get('HTTP_X_RUCIO_APPID')
if appid is None:
appid = 'unknown'
ip = request.environ.get('HTTP_X_FORWARDED_FOR')
if ip is None:
ip = request.remote_addr
try:
result = get_ssh_challenge_token(account, appid, ip, vo=vo)
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
return error, 500
if not result:
return generate_http_error_flask(
401, 'CannotAuthenticate',
'Cannot generate challenge for account %(account)s' % locals())
response.headers['X-Rucio-Auth-Token'] = result.token
response.headers['X-Rucio-Auth-Token-Expires'] = date_to_str(
result.expired_at)
return response
