def put(self, account):
"""
Create a new identity and map it to an account.
.. :quickref: X509; add new x509 identity.
:param account: the affected account.
:reqheader X-Rucio-Email: the desired email.
:status 201: Created.
:status 401: Invalid Auth Token.
:status 500: Internal Error.
"""
dn = request.environ.get('SSL_CLIENT_S_DN')
email = request_header_ensure_string('X-Rucio-Email')
try:
add_identity(dn, 'x509', email=email)
except Exception as error:
print(format_exc())
return str(error), 500
try:
add_account_identity(dn,
'x509',
account,
email=email,
issuer=request.environ.get('issuer'),
vo=request.environ.get('vo'))
except Exception as error:
print(format_exc())
return str(error), 500
return 'Created', 201
def put(self, account):
"""
Create a new identity and map it to an account.
.. :quickref: UserPass; add new userpass identity.
:reqheader X-Rucio-Username: the desired username.
:reqheader X-Rucio-Password: the desired password.
:reqheader X-Rucio-Email: the desired email.
:param account: the affected account.
:status 201: Created.
:status 400: Missing username or password.
:status 401: Invalid Auth Token.
:status 500: Internal Error.
"""
username = request_header_ensure_string('X-Rucio-Username')
password = request_header_ensure_string('X-Rucio-Password')
email = request_header_ensure_string('X-Rucio-Email')
if not username or not password:
return 'Username and Password must be set.', 400
try:
add_identity(username, 'userpass', email, password)
except Exception as error:
print(format_exc())
return str(error), 500
try:
add_account_identity(username,
'userpass',
account,
email=email,
password=password,
issuer=request.environ.get('issuer'),
vo=request.environ.get('vo'))
except Exception as error:
print(format_exc())
return str(error), 500
return 'Created', 201
def post(self):
"""
Trace endpoint used by the pilot and CLI clients to post data access information.
.. :quickref: Trace; Send trace.
:<json dict payload: Dictionary contain the trace information.
:status 201: Created.
:status 400: Cannot decode json data.
:status 500: Internal Error.
"""
headers = Headers()
headers.set('Content-Type', 'application/octet-stream')
headers.set('Access-Control-Allow-Origin',
request.environ.get('HTTP_ORIGIN'))
headers.set('Access-Control-Allow-Headers',
request.environ.get('HTTP_ACCESS_CONTROL_REQUEST_HEADERS'))
headers.set('Access-Control-Allow-Methods', '*')
headers.set('Access-Control-Allow-Credentials', 'true')
try:
payload = json.loads(request.data)
# generate entry timestamp
payload['traceTimeentry'] = datetime.datetime.utcnow()
payload['traceTimeentryUnix'] = calendar.timegm(
payload['traceTimeentry'].timetuple(
)) + payload['traceTimeentry'].microsecond / 1e6
# guess client IP
payload['traceIp'] = request_header_ensure_string(
'X-Forwarded-For', request.remote_addr)
# generate unique ID
payload['traceId'] = str(uuid.uuid4()).replace('-', '').lower()
trace(payload=payload)
except ValueError:
return generate_http_error_flask(
400,
'ValueError',
'Cannot decode json parameter list',
headers=headers)
except Exception as error:
print(traceback.format_exc())
return str(error), 500, headers
return 'Created', 201, headers
def post(self):
"""
Trace endpoint used by the XAOD framework to post data access information.
.. :quickref: XAODTrace; Send XAOD trace.
:<json dict payload: Dictionary contain the trace information.
:status 201: Created.
:status 400: Cannot decode json data.
:status 500: Internal Error.
"""
headers = Headers()
headers.set('Content-Type', 'application/octet-stream')
headers.set('Access-Control-Allow-Origin',
request.environ.get('HTTP_ORIGIN'))
headers.set('Access-Control-Allow-Headers',
request.environ.get('HTTP_ACCESS_CONTROL_REQUEST_HEADERS'))
headers.set('Access-Control-Allow-Methods', '*')
headers.set('Access-Control-Allow-Credentials', 'true')
try:
payload = json.loads(request.data)
# generate entry timestamp
payload['timeentry'] = int(time.time())
# guess client IP
payload['ip'] = request_header_ensure_string(
'X-Forwarded-For', request.remote_addr)
trace(payload=payload)
except ValueError:
return generate_http_error_flask(
400,
'ValueError',
'Cannot decode json parameter list',
headers=headers)
except Exception as error:
print(traceback.format_exc())
return str(error), 500, headers
return 'Created', 201, headers
def get(self, account):
""" get account parameters for given account name.
.. :quickref: AccountParameter; Get account parameters.
:param account: The account identifier.
:status 200: OK.
:status 401: Invalid auth token.
:status 404: Account not found.
:status 406: Not Acceptable.
:status 500: Database exception.
:returns: JSON dict containing informations about the requested user.
"""
if account == 'whoami':
# Redirect to the account uri
frontend = request_header_ensure_string('X-Requested-Host')
if frontend:
return redirect(frontend + "/accounts/%s" % (request.environ.get('issuer')), code=302)
return redirect(request.environ.get('issuer'), code=303)
acc = None
try:
acc = get_account_info(account, vo=request.environ.get('vo'))
except AccountNotFound as error:
return generate_http_error_flask(404, 'AccountNotFound', error.args[0])
except AccessDenied as error:
return generate_http_error_flask(401, 'AccessDenied', error.args[0])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
except Exception as error:
print(format_exc())
return str(error), 500
accdict = acc.to_dict()
for key, value in accdict.items():
if isinstance(value, datetime):
accdict[key] = value.strftime('%Y-%m-%dT%H:%M:%S')
del accdict['_sa_instance_state']
return Response(render_json(**accdict), content_type="application/json")
def get(self, scope_name):
"""
Metalink redirect
.. :quickref: MetaLinkRedirector; Metalink redirect.
:param scope_name: data identifier (scope)/(name).
:resheader Content-Type: application/metalink4+xml'.
:status 200: OK.
:status 401: Invalid Auth Token.
:status 404: RSE Not Found.
:status 404: DID Not Found.
:status 406: Not Acceptable.
:status 500: Internal Error.
:returns: Metalink file
"""
headers = Headers()
headers.set('Access-Control-Allow-Origin',
request.environ.get('HTTP_ORIGIN'))
headers.set('Access-Control-Allow-Headers',
request.environ.get('HTTP_ACCESS_CONTROL_REQUEST_HEADERS'))
headers.set('Access-Control-Allow-Methods', '*')
headers.set('Access-Control-Allow-Credentials', 'true')
try:
scope, name = parse_scope_name(scope_name)
except ValueError as error:
return generate_http_error_flask(400,
'ValueError',
error.args[0],
headers=headers)
except Exception as error:
print(format_exc())
return str(error), 500, headers
dids, schemes, select = [{
'scope': scope,
'name': name
}], ['http', 'https', 'root', 'gsiftp', 'srm', 'davs'], None
# set the correct client IP
client_ip = request_header_ensure_string('X-Forwarded-For',
request.remote_addr)
client_location = {'ip': client_ip, 'fqdn': None, 'site': None}
if request.query_string:
query_string = request.query_string.decode(encoding='utf-8')
params = parse_qs(query_string)
if 'schemes' in params:
schemes = params['schemes']
if 'select' in params:
select = params['select'][0]
if 'sort' in params:
select = params['sort'][0]
if 'ip' in params:
client_location['ip'] = params['ip'][0]
if 'fqdn' in params:
client_location['fqdn'] = params['fqdn'][0]
if 'site' in params:
client_location['site'] = params['site'][0]
# get vo if given
vo = request_header_ensure_string('X-Rucio-VO', 'def')
try:
replicas_iter = list_replicas(dids=dids,
schemes=schemes,
client_location=client_location,
vo=vo)
try:
first = next(replicas_iter)
except StopIteration:
return 'no redirection possible - cannot find the DID', 404
def generate():
# first, set the appropriate content type, and stream the header
yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n'
# iteratively stream the XML per file
for rfile in itertools.chain((first, ), replicas_iter):
replicas = []
dictreplica = {}
for rse in rfile['rses']:
for replica in rfile['rses'][rse]:
replicas.append(replica)
dictreplica[replica] = rse
# stream metadata
yield ' <file name="' + rfile['name'] + '">\n'
yield ' <identity>' + rfile['scope'] + ':' + rfile[
'name'] + '</identity>\n'
if rfile['adler32'] is not None:
yield ' <hash type="adler32">' + rfile[
'adler32'] + '</hash>\n'
if rfile['md5'] is not None:
yield ' <hash type="md5">' + rfile['md5'] + '</hash>\n'
yield ' <size>' + str(rfile['bytes']) + '</size>\n'
yield ' <glfn name="/atlas/rucio/%s:%s">' % (
rfile['scope'], rfile['name'])
yield '</glfn>\n'
# sort the actual replicas if necessary
if select == 'geoip':
replicas = sort_geoip(dictreplica,
client_location['ip'],
ignore_error=True)
elif select == 'closeness':
replicas = sort_closeness(dictreplica, client_location)
elif select == 'dynamic':
replicas = sort_dynamic(dictreplica, client_location)
elif select == 'ranking':
replicas = sort_ranking(dictreplica, client_location)
else:
replicas = sort_random(dictreplica)
# stream URLs
idx = 1
for replica in replicas:
yield ' <url location="' + str(
dictreplica[replica]) + '" priority="' + str(
idx) + '">' + replica + '</url>\n'
idx += 1
yield ' </file>\n'
# don't forget to send the metalink footer
yield '</metalink>\n'
return try_stream(generate(),
content_type='application/metalink4+xml')
except DataIdentifierNotFound as error:
return generate_http_error_flask(404,
'DataIdentifierNotFound',
error.args[0],
headers=headers)
except ReplicaNotFound as error:
return generate_http_error_flask(404,
'ReplicaNotFound',
error.args[0],
headers=headers)
except RucioException as error:
return generate_http_error_flask(500,
error.__class__.__name__,
error.args[0],
headers=headers)
except Exception as error:
print(format_exc())
return str(error), 500, headers
def get(self, scope_name):
"""
Header Redirect
.. :quickref: HeaderRedirector; Header redirect.
:param scope_name: data identifier (scope)/(name).
:resheader Content-Type: application/metalink+xml'.
:status 303: Redirect.
:status 401: Invalid Auth Token.
:status 404: RSE Not Found.
:status 404: DID Not Found.
:status 500: Internal Error.
"""
headers = Headers()
headers.set('Access-Control-Allow-Origin',
request.environ.get('HTTP_ORIGIN'))
headers.set('Access-Control-Allow-Headers',
request.environ.get('HTTP_ACCESS_CONTROL_REQUEST_HEADERS'))
headers.set('Access-Control-Allow-Methods', '*')
headers.set('Access-Control-Allow-Credentials', 'true')
try:
scope, name = parse_scope_name(scope_name)
except ValueError as error:
return generate_http_error_flask(400,
'ValueError',
error.args[0],
headers=headers)
except Exception as error:
print(format_exc())
return str(error), 500, headers
try:
# use the default HTTP protocols if no scheme is given
select, rse, site, schemes = 'random', None, None, [
'davs', 'http', 'https'
]
client_ip = request_header_ensure_string('X-Forwarded-For',
request.remote_addr)
client_location = {'ip': client_ip, 'fqdn': None, 'site': None}
if request.query_string:
query_string = request.query_string.decode(encoding='utf-8')
params = parse_qs(query_string)
if 'select' in params:
select = params['select'][0]
if 'sort' in params:
select = params['sort'][0]
if 'rse' in params:
rse = params['rse'][0]
if 'site' in params:
site = params['site'][0]
if 'schemes' in params:
schemes = params['schemes'][0]
else:
schemes = ['davs', 'https', 's3']
if 'ip' in params:
client_location['ip'] = params['ip'][0]
if 'fqdn' in params:
client_location['fqdn'] = params['fqdn'][0]
if 'site' in params:
client_location['site'] = params['site'][0]
# correctly forward the schemes and select to potential metalink followups
cleaned_url = request.environ.get('REQUEST_URI').split('?')[0]
if isinstance(schemes, list):
headers.set(
'Link',
'<%s/metalink?schemes=%s&select=%s>; rel=describedby; type="application/metalink+xml"'
% (cleaned_url, ','.join(schemes), select))
else:
headers.set(
'Link',
'<%s/metalink?schemes=%s&select=%s>; rel=describedby; type="application/metalink+xml"'
% (cleaned_url, schemes, select))
schemes = [schemes] # list_replicas needs a list
# get vo if given
vo = request_header_ensure_string('X-Rucio-VO', 'def')
replicas = [
r for r in list_replicas(dids=[{
'scope': scope,
'name': name,
'type': 'FILE'
}],
schemes=schemes,
client_location=client_location,
vo=vo)
]
selected_url = None
for r in replicas:
if r['rses']:
dictreplica = {}
if rse:
if rse in r['rses'] and r['rses'][rse]:
selected_url = r['rses'][rse][0]
else:
return 'no redirection possible - no valid RSE for HTTP redirection found', 404, headers
else:
for rep in r['rses']:
for replica in r['rses'][rep]:
# since this is HTTP-only redirection, and to ensure compatibility with as many http clients as possible
# forcibly replacement davs and s3 URLs to https
replica = replica.replace(
'davs://',
'https://').replace('s3://', 'https://')
dictreplica[replica] = rep
if not dictreplica:
return 'no redirection possible - no valid RSE for HTTP redirection found', 404, headers
elif site:
rep = site_selector(dictreplica, site, vo)
if rep:
selected_url = rep[0]
else:
return 'no redirection possible - no valid RSE for HTTP redirection found', 404, headers
else:
if select == 'geoip':
rep = sort_geoip(dictreplica,
client_location['ip'])
elif select == 'closeness':
rep = sort_closeness(dictreplica,
client_location)
elif select == 'dynamic':
rep = sort_dynamic(dictreplica,
client_location)
elif select == 'ranking':
rep = sort_ranking(dictreplica,
client_location)
else:
rep = sort_random(dictreplica)
selected_url = rep[0]
if selected_url:
response = redirect(selected_url, code=303)
response.headers.extend(headers)
return response
return 'no redirection possible - file does not exist', 404, headers
except ReplicaNotFound as error:
return generate_http_error_flask(404,
'ReplicaNotFound',
error.args[0],
headers=headers)
except RucioException as error:
return generate_http_error_flask(500,
error.__class__.__name__,
error.args[0],
headers=headers)
except Exception as error:
print(format_exc())
return str(error), 500, headers
def get(self):
"""
Sign a URL for a limited lifetime for a particular service.
:reqheader X-Rucio-VO: VO name as a string (Multi-VO only).
:reqheader X-Rucio-Account: Account identifier as a string.
:reqheader X-Rucio-AppID: Application identifier as a string.
:status 200: Successfully signed URL
:status 400: Bad Request
:status 401: Unauthorized
:status 406: Not Acceptable
:status 500: Internal Server Error
"""
vo = request_header_ensure_string('X-Rucio-VO', 'def')
account = request_header_ensure_string('X-Rucio-Account')
appid = request_header_ensure_string('X-Rucio-AppID', 'unknown')
ip = request_header_ensure_string('X-Forwarded-For',
request.remote_addr)
rse, svc, operation, url = None, None, None, None
try:
query_string = request.query_string.decode(encoding='utf-8')
params = parse_qs(query_string)
rse = params.get('rse', [None])[0]
lifetime = params.get('lifetime', [600])[0]
service = params.get('svc', ['gcs'])[0]
operation = params.get('op', ['read'])[0]
url = params.get('url', [None])[0]
except ValueError:
return generate_http_error_flask(
400, 'ValueError', 'Cannot decode json parameter list')
if service not in ['gcs', 's3', 'swift']:
return generate_http_error_flask(
400, 'ValueError',
'Parameter "svc" must be either empty(=gcs), gcs, s3 or swift')
if url is None:
return generate_http_error_flask(400, 'ValueError',
'Parameter "url" not found')
if rse is None:
return generate_http_error_flask(400, 'ValueError',
'Parameter "rse" not found')
if operation not in ['read', 'write', 'delete']:
return generate_http_error_flask(
400, 'ValueError',
'Parameter "op" must be either empty(=read), read, write, or delete.'
)
try:
result = get_signed_url(account,
appid,
ip,
rse=rse,
service=service,
operation=operation,
url=url,
lifetime=lifetime,
vo=vo)
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
return str(error), 500
if not result:
return generate_http_error_flask(
401, 'CannotAuthenticate',
'Cannot generate signed URL for account %(account)s' %
locals())
return str(result), 200