def test_delete(self):
""" OBJECTSTORE (COMMON): Delete urls """
urls = []
for i in range(10):
url = '%s_%s' % (self.url, i)
urls.append(url)
ret = objectstore.get_signed_urls(urls, rse=self.rse, operation='write')
for url in urls:
if isinstance(url, Exception):
raise ret[self.url]
# write
command = 'curl --request PUT --upload-file /bin/hostname "%s"' % ret[url]
status, output = commands.getstatusoutput(command)
if status:
raise Exception(output)
if 'AccessDenied' in output:
raise Exception(output)
ret = objectstore.delete(urls, rse=self.rse)
for url in urls:
if isinstance(ret[url], Exception):
raise ret[url]
def get(self, url, rse, operation):
"""
Pass a url and return the signed url.
.. :quickref: ObjectStoreGet; get signed url.
:param url: A URL string.
:param rse: RSE name.
:param operation: the request operation (default: 'read').
:status 200: OK.
:status 401: Invalid Auth Token.
:status 406: Not Acceptable.
:status 500: Internal Error.
:returns: the signed URL.
get redirect URL.
"""
try:
pos = url.index('/')
url = ''.join([url[:pos], '/', url[pos:]])
if operation == 'connect':
objectstore.connect(rse, url)
else:
result = objectstore.get_signed_urls([url],
rse=rse,
operation=operation)
if isinstance(result[url], Exception):
return Response(result[url])
return Response(result[url])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(traceback.format_exc())
return error, 500
return "OK"
def GET(self, scope, name):
"""
Header Redirect
HTTP Success:
303 See Other
HTTP Error:
401 Unauthorized
500 InternalError
404 Notfound
:param scope: The scope name of the file.
:param name: The name of the file.
"""
header('Access-Control-Allow-Origin', ctx.env.get('HTTP_ORIGIN'))
header('Access-Control-Allow-Headers',
ctx.env.get('HTTP_ACCESS_CONTROL_REQUEST_HEADERS'))
header('Access-Control-Allow-Methods', '*')
header('Access-Control-Allow-Credentials', 'true')
try:
# use the default HTTP protocols if no scheme is given
select, rse, site, schemes = 'random', None, None, [
'davs', 'http', 'https', 's3+rucio'
]
client_ip = ctx.env.get('HTTP_X_FORWARDED_FOR')
if client_ip is None:
client_ip = ctx.ip
client_location = {'ip': client_ip, 'fqdn': None, 'site': None}
if ctx.query:
params = parse_qs(ctx.query[1:])
if 'select' in params:
select = params['select'][0]
if 'sort' in params:
select = params['sort'][0]
if 'rse' in params:
rse = params['rse'][0]
if 'site' in params:
site = params['site'][0]
if 'schemes' in params:
schemes = params['schemes'][0]
else:
schemes = ['davs', 'https', 's3']
if 'ip' in params:
client_location['ip'] = params['ip'][0]
if 'fqdn' in params:
client_location['fqdn'] = params['fqdn'][0]
if 'site' in params:
client_location['site'] = params['site'][0]
# correctly forward the schemes and select to potential metalink followups
cleaned_url = ctx.env.get('REQUEST_URI').split('?')[0]
if isinstance(schemes, list):
header(
'Link',
'<%s/metalink?schemes=%s&select=%s>; rel=describedby; type="application/metalink+xml"'
% (cleaned_url, ','.join(schemes), select))
else:
header(
'Link',
'<%s/metalink?schemes=%s&select=%s>; rel=describedby; type="application/metalink+xml"'
% (cleaned_url, schemes, select))
schemes = [schemes] # list_replicas needs a list
replicas = [
r for r in list_replicas(dids=[{
'scope': scope,
'name': name,
'type': 'FILE'
}],
schemes=schemes,
client_location=client_location)
]
selected_url, selected_rse = None, None
for r in replicas:
if r['rses']:
dictreplica = {}
if rse:
if rse in r['rses'] and r['rses'][rse]:
selected_url = r['rses'][rse][0]
selected_rse = rse
else:
raise ReplicaNotFound(
'no redirection possible - no valid RSE for HTTP redirection found'
)
else:
for rep in r['rses']:
for replica in r['rses'][rep]:
# since this is HTTP-only redirection, and to ensure compatibility with as many http clients as possible
# forcibly replacement davs and s3 URLs to https
replica = replica.replace(
'davs://',
'https://').replace('s3://', 'https://')
dictreplica[replica] = rep
if not dictreplica:
raise ReplicaNotFound(
'no redirection possible - no valid RSE for HTTP redirection found'
)
elif site:
rep = site_selector(dictreplica, site)
if rep:
selected_url = rep[0]
else:
raise ReplicaNotFound(
'no redirection possible - no valid RSE for HTTP redirection found'
)
else:
if select == 'geoip':
rep = sort_geoip(dictreplica,
client_location['ip'])
elif select == 'closeness':
rep = sort_closeness(dictreplica,
client_location)
elif select == 'dynamic':
rep = sort_dynamic(dictreplica,
client_location)
elif select == 'ranking':
rep = sort_ranking(dictreplica,
client_location)
else:
rep = sort_random(dictreplica)
selected_url = rep[0]
for rep in r['rses']:
for replica in r['rses'][rep]:
if selected_url == replica:
selected_rse = rep
if selected_url:
if selected_url.startswith('s3+rucio://'):
connect(selected_rse, selected_url)
signed_URLS = get_signed_urls([selected_url],
rse=selected_rse,
operation='read')
raise seeother(signed_URLS[selected_url])
raise seeother(selected_url)
raise ReplicaNotFound(
'no redirection possible - file does not exist')
except seeother:
raise
except ReplicaNotFound as error:
raise generate_http_error(404, 'ReplicaNotFound', error.args[0])
except RucioException as error:
raise generate_http_error(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
raise InternalError(error)
def get(self, scope, name):
"""
Header Redirect
.. :quickref: HeaderRedirector; Header redirect.
:param scope: The scope name of the file.
:param name: The name of the file.
:resheader Content-Type: application/metalink+xml'.
:status 303: Redirect.
:status 401: Invalid Auth Token.
:status 404: RSE Not Found.
:status 404: DID Not Found.
:status 500: Internal Error.
"""
headers = {}
try:
# use the default HTTP protocols if no scheme is given
client_ip = request.environ.get('HTTP_X_FORWARDED_FOR')
if client_ip is None:
client_ip = request.remote_addr
client_location = {'ip': client_ip, 'fqdn': None, 'site': None}
schemes = request.args.get('schemes', ['davs', 'https', 's3'])
select = request.args.get('select', 'random')
if 'sort' in request.args:
select = request.args['sort']
rse = request.args.get('rse', None)
site = request.args.get('site', None)
client_location['ip'] = request.args.get('ip', client_ip)
client_location['fqdn'] = request.args.get('fqdn', None)
client_location['site'] = request.args.get('site', None)
# correctly forward the schemes and select to potential metalink followups
cleaned_url = request.environ.get('REQUEST_URI').split('?')[0]
if isinstance(schemes, list):
headers[
'Link'] = '<%s/metalink?schemes=%s&select=%s>; rel=describedby; type="application/metalink+xml"' % (
cleaned_url, ','.join(schemes), select)
else:
headers[
'Link'] = '<%s/metalink?schemes=%s&select=%s>; rel=describedby; type="application/metalink+xml"' % (
cleaned_url, schemes, select)
schemes = [schemes] # list_replicas needs a list
replicas = [
r for r in list_replicas(dids=[{
'scope': scope,
'name': name,
'type': 'FILE'
}],
schemes=schemes,
client_location=client_location)
]
selected_url, selected_rse = None, None
for r in replicas:
if r['rses']:
dictreplica = {}
if rse:
if rse in r['rses'] and r['rses'][rse]:
selected_url = r['rses'][rse][0]
selected_rse = rse
else:
return 'no redirection possible - no valid RSE for HTTP redirection found', 404
else:
for rep in r['rses']:
for replica in r['rses'][rep]:
# since this is HTTP-only redirection, and to ensure compatibility with as many http clients as possible
# forcibly replacement davs and s3 URLs to https
replica = replica.replace(
'davs://',
'https://').replace('s3://', 'https://')
dictreplica[replica] = rep
if not dictreplica:
return 'no redirection possible - no valid RSE for HTTP redirection found', 404
elif site:
rep = site_selector(dictreplica, site)
if rep:
selected_url = rep[0]
else:
return 'no redirection possible - no valid RSE for HTTP redirection found', 404
else:
if select == 'geoip':
rep = sort_geoip(dictreplica,
client_location['ip'])
elif select == 'closeness':
rep = sort_closeness(dictreplica,
client_location)
elif select == 'dynamic':
rep = sort_dynamic(dictreplica,
client_location)
elif select == 'ranking':
rep = sort_ranking(dictreplica,
client_location)
else:
rep = sort_random(dictreplica)
selected_url = rep[0]
for rep in r['rses']:
for replica in r['rses'][rep]:
if selected_url == replica:
selected_rse = rep
if selected_url:
if selected_url.startswith('s3+rucio://'):
connect(selected_rse, selected_url)
signed_URLS = get_signed_urls([selected_url],
rse=selected_rse,
operation='read')
res = redirect(signed_URLS[selected_url], code=303)
res.header = headers
return res
res = redirect(signed_URLS[selected_url], code=303)
res.header = headers
return res
return 'no redirection possible - file does not exist', 404
except ReplicaNotFound as error:
return generate_http_error_flask(404, 'ReplicaNotFound',
error.args[0])
except RucioException as error:
return generate_http_error_flask(500, error.__class__.__name__,
error.args[0])
except Exception as error:
print(format_exc())
return error, 500
def GET(self, scope, name):
"""
Header Redirect
HTTP Success:
303 See Other
HTTP Error:
401 Unauthorized
500 InternalError
404 Notfound
:param scope: The scope name of the file.
:param name: The name of the file.
"""
header('Access-Control-Allow-Origin', ctx.env.get('HTTP_ORIGIN'))
header('Access-Control-Allow-Headers',
ctx.env.get('HTTP_ACCESS_CONTROL_REQUEST_HEADERS'))
header('Access-Control-Allow-Methods', '*')
header('Access-Control-Allow-Credentials', 'true')
try:
# use the default HTTP protocols if no scheme is given
select, rse, site, schemes = 'random', None, None, [
'http', 'https', 's3+rucio'
]
if ctx.query:
params = parse_qs(ctx.query[1:])
if 'select' in params:
select = params['select'][0]
if 'rse' in params:
rse = params['rse'][0]
if 'site' in params:
site = params['site'][0]
if 'schemes' in params:
schemes = params['schemes'][0]
# correctly forward the schemes and select to potential metalink followups
cleaned_url = ctx.env.get('REQUEST_URI').split('?')[0]
if isinstance(schemes, list):
header(
'Link',
'<%s/metalink?schemes=%s&select=%s>; rel=describedby; type="application/metalink+xml"'
% (cleaned_url, ','.join(schemes), select))
else:
header(
'Link',
'<%s/metalink?schemes=%s&select=%s>; rel=describedby; type="application/metalink+xml"'
% (cleaned_url, schemes, select))
schemes = [schemes] # list_replicas needs a list
replicas = [
r for r in list_replicas(dids=[{
'scope': scope,
'name': name,
'type': 'FILE'
}],
schemes=schemes)
]
selected_url, selected_rse = None, None
for r in replicas:
if r['rses']:
replicadict = {}
if rse:
if rse in r['rses'] and r['rses'][rse]:
selected_url = r['rses'][rse][0]
selected_rse = rse
else:
raise ReplicaNotFound(
'no redirection possible - no valid RSE for HTTP redirection found'
)
else:
for rep in r['rses']:
for replica in r['rses'][rep]:
replicadict[replica] = rep
if not replicadict:
raise ReplicaNotFound(
'no redirection possible - no valid RSE for HTTP redirection found'
)
elif site:
rep = site_selector(replicadict, site)
if rep:
selected_url = rep[0]
else:
raise ReplicaNotFound(
'no redirection possible - no valid RSE for HTTP redirection found'
)
else:
client_ip = ctx.env.get('HTTP_X_FORWARDED_FOR')
if client_ip is None:
client_ip = ctx.ip
if select == 'geoip':
rep = geoIP_order(replicadict, client_ip)
else:
rep = random_order(replicadict, client_ip)
selected_url = rep[0]
for rep in r['rses']:
for replica in r['rses'][rep]:
if selected_url == replica:
selected_rse = rep
if selected_url:
if selected_url.startswith('s3+rucio://'):
connect(selected_rse, selected_url)
signed_URLS = get_signed_urls([selected_url],
rse=selected_rse,
operation='read')
raise seeother(signed_URLS[selected_url])
raise seeother(selected_url)
raise ReplicaNotFound(
'no redirection possible - file does not exist')
except seeother:
raise
except ReplicaNotFound, e:
raise generate_http_error(404, 'ReplicaNotFound', e.args[0][0])