def GET(self, scope, name): """ List all replicas for data identifiers. HTTP Success: 200 OK HTTP Error: 401 Unauthorized 500 InternalError :returns: A dictionary containing all replicas information. :returns: A metalink description of replicas if metalink(4)+xml is specified in Accept: """ metalink = None if ctx.env.get('HTTP_ACCEPT') is not None: tmp = ctx.env.get('HTTP_ACCEPT').split(',') # first check if client accepts metalink if 'application/metalink+xml' in tmp: metalink = 3 # but prefer metalink4 if the client has support for it # (clients can put both in their ACCEPT header!) if 'application/metalink4+xml' in tmp: metalink = 4 dids, schemes, select, limit = [{'scope': scope, 'name': name}], None, None, None if ctx.query: try: params = loads(unquote(ctx.query[1:])) if 'schemes' in params: schemes = params['schemes'] except ValueError: params = parse_qs(ctx.query[1:]) if 'select' in params: select = params['select'][0] if 'limit' in params: limit = int(params['limit'][0]) try: # first, set the appropriate content type, and stream the header if metalink is None: header('Content-Type', 'application/x-json-stream') elif metalink == 3: header('Content-Type', 'application/metalink+xml') schemes = ['http', 'https'] yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink version="3.0" xmlns="http://www.metalinker.org/">\n<files>\n' elif metalink == 4: header('Content-Type', 'application/metalink4+xml') schemes = ['http', 'https'] yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n' # then, stream the replica information for rfile in list_replicas(dids=dids, schemes=schemes): client_ip = ctx.get('ip') replicas = [] dictreplica = {} for rse in rfile['rses']: for replica in rfile['rses'][rse]: replicas.append(replica) dictreplica[replica] = rse if select == 'geoip': try: replicas = geoIP_order(dictreplica, client_ip) except AddressNotFoundError: pass else: replicas = random_order(dictreplica, client_ip) if metalink is None: yield dumps(rfile) + '\n' elif metalink == 3: idx = 0 yield ' <file name="' + rfile['name'] + '">\n <resources>\n' for replica in replicas: yield ' <url type="http" preference="' + str(idx) + '">' + replica + '</url>\n' idx += 1 if limit and limit == idx: break yield ' </resources>\n </file>\n' elif metalink == 4: yield ' <file name="' + rfile['name'] + '">\n' yield ' <identity>' + rfile['scope'] + ':' + rfile['name'] + '</identity>\n' if rfile['adler32'] is not None: yield ' <hash type="adler32">' + rfile['adler32'] + '</hash>\n' if rfile['md5'] is not None: yield ' <hash type="md5">' + rfile['md5'] + '</hash>\n' yield ' <size>' + str(rfile['bytes']) + '</size>\n' idx = 0 for replica in replicas: yield ' <url location="' + str(dictreplica[replica]) + '" priority="' + str(idx + 1) + '">' + replica + '</url>\n' idx += 1 if limit and limit == idx: break yield ' </file>\n' # don't forget to send the metalink footer if metalink: if metalink == 3: yield '</files>\n</metalink>\n' elif metalink == 4: yield '</metalink>\n' except DataIdentifierNotFound, e: raise generate_http_error(404, 'DataIdentifierNotFound', e.args[0][0])
def GET(self, scope, name): """ List all replicas for data identifiers. HTTP Success: 200 OK HTTP Error: 401 Unauthorized 500 InternalError :returns: A dictionary containing all replicas information. :returns: A metalink description of replicas if metalink(4)+xml is specified in Accept: """ metalink = None if ctx.env.get('HTTP_ACCEPT') is not None: tmp = ctx.env.get('HTTP_ACCEPT').split(',') # first check if client accepts metalink if 'application/metalink+xml' in tmp: metalink = 3 # but prefer metalink4 if the client has support for it # (clients can put both in their ACCEPT header!) if 'application/metalink4+xml' in tmp: metalink = 4 dids, schemes, select, limit = [{ 'scope': scope, 'name': name }], None, None, None if ctx.query: params = parse_qs(ctx.query[1:]) if 'schemes' in params: schemes = params['schemes'] if 'select' in params: select = params['select'][0] if 'limit' in params: limit = int(params['limit'][0]) try: # first, set the APPropriate content type, and stream the header if metalink is None: header('Content-Type', 'application/x-json-stream') elif metalink == 3: header('Content-Type', 'application/metalink+xml') yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink version="3.0" xmlns="http://www.metalinker.org/">\n<files>\n' elif metalink == 4: header('Content-Type', 'application/metalink4+xml') yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n' # then, stream the replica information for rfile in list_replicas(dids=dids, schemes=schemes): client_ip = ctx.env.get('HTTP_X_FORWARDED_FOR') if client_ip is None: client_ip = ctx.ip replicas = [] dictreplica = {} for rse in rfile['rses']: for replica in rfile['rses'][rse]: replicas.append(replica) dictreplica[replica] = rse if select == 'geoip': try: replicas = geoIP_order(dictreplica, client_ip) except AddressNotFoundError: pass else: replicas = random_order(dictreplica, client_ip) if metalink is None: yield dumps(rfile) + '\n' elif metalink == 3: idx = 0 yield ' <file name="' + rfile['name'] + '">\n' yield ' <glfn name="/atlas/rucio/%s:%s">' % ( rfile['scope'], rfile['name']) yield '</glfn>\n' yield ' <resources>\n' for replica in replicas: yield ' <url type="http" preference="' + str( idx) + '">' + replica + '</url>\n' idx += 1 if limit and limit == idx: break yield ' </resources>\n </file>\n' elif metalink == 4: yield ' <file name="' + rfile['name'] + '">\n' yield ' <identity>' + rfile['scope'] + ':' + rfile[ 'name'] + '</identity>\n' if rfile['adler32'] is not None: yield ' <hash type="adler32">' + rfile[ 'adler32'] + '</hash>\n' if rfile['md5'] is not None: yield ' <hash type="md5">' + rfile['md5'] + '</hash>\n' yield ' <size>' + str(rfile['bytes']) + '</size>\n' yield ' <glfn name="/atlas/rucio/%s:%s">' % ( rfile['scope'], rfile['name']) yield '</glfn>\n' idx = 0 for replica in replicas: yield ' <url location="' + str( dictreplica[replica]) + '" priority="' + str( idx + 1) + '">' + replica + '</url>\n' idx += 1 if limit and limit == idx: break yield ' </file>\n' # don't forget to send the metalink footer if metalink: if metalink == 3: yield '</files>\n</metalink>\n' elif metalink == 4: yield '</metalink>\n' except DataIdentifierNotFound, e: raise generate_http_error(404, 'DataIdentifierNotFound', e.args[0][0])
def POST(self): """ List all replicas for data identifiers. HTTP Success: 200 OK HTTP Error: 401 Unauthorized 500 InternalError :returns: A dictionary containing all replicas information. :returns: A metalink description of replicas if metalink(4)+xml is specified in Accept: """ metalink = None if ctx.env.get('HTTP_ACCEPT') is not None: tmp = ctx.env.get('HTTP_ACCEPT').split(',') # first check if client accepts metalink if 'application/metalink+xml' in tmp: metalink = 3 # but prefer metalink4 if the client has support for it # (clients can put both in their ACCEPT header!) if 'application/metalink4+xml' in tmp: metalink = 4 dids, schemes, select, unavailable, limit = [], None, None, False, None ignore_availability, rse_expression, all_states = False, None, False json_data = data() try: params = parse_response(json_data) if 'dids' in params: dids = params['dids'] if 'schemes' in params: schemes = params['schemes'] if 'unavailable' in params: unavailable = params['unavailable'] ignore_availability = True if 'all_states' in params: all_states = params['all_states'] if 'rse_expression' in params: rse_expression = params['rse_expression'] except ValueError: raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list') if ctx.query: params = parse_qs(ctx.query[1:]) if 'select' in params: select = params['select'][0] if 'limit' in params: limit = params['limit'][0] try: # first, set the APPropriate content type, and stream the header if metalink is None: header('Content-Type', 'application/x-json-stream') elif metalink == 3: header('Content-Type', 'application/metalink+xml') yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink version="3.0" xmlns="http://www.metalinker.org/">\n<files>\n' elif metalink == 4: header('Content-Type', 'application/metalink4+xml') yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n' # then, stream the replica information for rfile in list_replicas(dids=dids, schemes=schemes, unavailable=unavailable, request_id=ctx.env.get('request_id'), ignore_availability=ignore_availability, all_states=all_states, rse_expression=rse_expression): client_ip = ctx.env.get('HTTP_X_FORWARDED_FOR') if client_ip is None: client_ip = ctx.ip replicas = [] dictreplica = {} for rse in rfile['rses']: for replica in rfile['rses'][rse]: replicas.append(replica) dictreplica[replica] = rse if select == 'geoip': replicas = geoIP_order(dictreplica, client_ip) else: replicas = random_order(dictreplica, client_ip) if metalink is None: yield dumps(rfile, cls=APIEncoder) + '\n' elif metalink == 3: idx = 0 yield ' <file name="' + rfile[ 'name'] + '">\n <resources>\n' for replica in replicas: yield ' <url type="http" preference="' + str( idx) + '">' + replica + '</url>\n' idx += 1 if limit and limit == idx: break yield ' </resources>\n </file>\n' elif metalink == 4: yield ' <file name="' + rfile['name'] + '">\n' yield ' <identity>' + rfile['scope'] + ':' + rfile[ 'name'] + '</identity>\n' if rfile['adler32'] is not None: yield ' <hash type="adler32">' + rfile[ 'adler32'] + '</hash>\n' if rfile['md5'] is not None: yield ' <hash type="md5">' + rfile['md5'] + '</hash>\n' yield ' <size>' + str(rfile['bytes']) + '</size>\n' idx = 0 for replica in replicas: yield ' <url location="' + str( dictreplica[replica]) + '" priority="' + str( idx + 1) + '">' + replica + '</url>\n' idx += 1 if limit and limit == idx: break yield ' </file>\n' # don't forget to send the metalink footer if metalink: if metalink == 3: yield '</files>\n</metalink>\n' elif metalink == 4: yield '</metalink>\n' except DataIdentifierNotFound, e: raise generate_http_error(404, 'DataIdentifierNotFound', e.args[0][0])
def GET(self, scope, name): """ Redirect download HTTP Success: 303 See Other HTTP Error: 401 Unauthorized 500 InternalError 404 Notfound :param scope: The scope name of the file. :param name: The name of the file. """ header("Access-Control-Allow-Origin", ctx.env.get("HTTP_ORIGIN")) header("Access-Control-Allow-Headers", ctx.env.get("HTTP_ACCESS_CONTROL_REQUEST_HEADERS")) header("Access-Control-Allow-Methods", "*") header("Access-Control-Allow-Credentials", "true") try: replicas = [ r for r in list_replicas(dids=[{"scope": scope, "name": name, "type": "FILE"}], schemes=["http", "https"]) ] select = "random" rse = None site = None if ctx.query: params = parse_qs(ctx.query[1:]) if "select" in params: select = params["select"][0] if "rse" in params: rse = params["rse"][0] if "site" in params: site = params["site"][0] for r in replicas: if r["rses"]: replicadict = {} if rse: if rse in r["rses"] and r["rses"][rse]: return found(r["rses"][rse][0]) return notfound("Sorry, the replica you were looking for was not found.") else: for rep in r["rses"]: for replica in r["rses"][rep]: replicadict[replica] = rep if not replicadict: return notfound("Sorry, the replica you were looking for was not found.") elif site: rep = site_selector(replicadict, site) if rep: return found(rep[0]) return notfound("Sorry, the replica you were looking for was not found.") else: client_ip = ctx.get("ip") if select == "geoip": rep = geoIP_order(replicadict, client_ip) else: rep = random_order(replicadict, client_ip) return found(rep[0]) return notfound("Sorry, the replica you were looking for was not found.") except RucioException, e: raise generate_http_error(500, e.__class__.__name__, e.args[0][0])
def GET(self, scope, name): """ Metalink redirect HTTP Success: 200 OK HTTP Error: 401 Unauthorized 500 InternalError 404 Notfound :param scope: The scope name of the file. :param name: The name of the file. """ header('Access-Control-Allow-Origin', ctx.env.get('HTTP_ORIGIN')) header('Access-Control-Allow-Headers', ctx.env.get('HTTP_ACCESS_CONTROL_REQUEST_HEADERS')) header('Access-Control-Allow-Methods', '*') header('Access-Control-Allow-Credentials', 'true') dids, schemes, select = [{ 'scope': scope, 'name': name }], ['http', 'https', 's3+rucio', 's3+https', 'root', 'gsiftp', 'srm'], None if ctx.query: params = parse_qs(ctx.query[1:]) if 'schemes' in params: schemes = params['schemes'] if 'select' in params: select = params['select'][0] try: tmp_replicas = [ rep for rep in list_replicas(dids=dids, schemes=schemes) ] if not tmp_replicas: raise ReplicaNotFound( 'no redirection possible - cannot find the DID') # first, set the APPropriate content type, and stream the header header('Content-Type', 'application/metalink4+xml') yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n' # set the correct client IP client_ip = ctx.env.get('HTTP_X_FORWARDED_FOR') if client_ip is None: client_ip = ctx.ip # iteratively stream the XML per file for rfile in tmp_replicas: replicas = [] dictreplica = {} for rse in rfile['rses']: for replica in rfile['rses'][rse]: replicas.append(replica) dictreplica[replica] = rse # sort the actual replicas if necessary if select == 'geoip': try: replicas = geoIP_order(dictreplica, client_ip) except AddressNotFoundError: pass else: replicas = random_order(dictreplica, client_ip) # stream metadata yield ' <file name="' + rfile['name'] + '">\n' yield ' <identity>' + rfile['scope'] + ':' + rfile[ 'name'] + '</identity>\n' if rfile['adler32'] is not None: yield ' <hash type="adler32">' + rfile[ 'adler32'] + '</hash>\n' if rfile['md5'] is not None: yield ' <hash type="md5">' + rfile['md5'] + '</hash>\n' yield ' <size>' + str(rfile['bytes']) + '</size>\n' yield ' <glfn name="%s%s">' % ( 'root://atlas-xrd-eu.cern.ch:1094//atlas/rucio/', '%s:%s' % (rfile['scope'], rfile['name'])) yield '</glfn>\n' # stream URLs idx = 1 for replica in replicas: yield ' <url location="' + str( dictreplica[replica]) + '" priority="' + str( idx) + '">' + replica + '</url>\n' idx += 1 yield ' </file>\n' # don't forget to send the metalink footer yield '</metalink>\n' except DataIdentifierNotFound, e: raise generate_http_error(404, 'DataIdentifierNotFound', e.args[0][0])
def GET(self, scope, name): """ Header Redirect HTTP Success: 303 See Other HTTP Error: 401 Unauthorized 500 InternalError 404 Notfound :param scope: The scope name of the file. :param name: The name of the file. """ header('Access-Control-Allow-Origin', ctx.env.get('HTTP_ORIGIN')) header('Access-Control-Allow-Headers', ctx.env.get('HTTP_ACCESS_CONTROL_REQUEST_HEADERS')) header('Access-Control-Allow-Methods', '*') header('Access-Control-Allow-Credentials', 'true') try: # use the default HTTP protocols if no scheme is given select, rse, site, schemes = 'random', None, None, [ 'http', 'https', 's3+rucio' ] if ctx.query: params = parse_qs(ctx.query[1:]) if 'select' in params: select = params['select'][0] if 'rse' in params: rse = params['rse'][0] if 'site' in params: site = params['site'][0] if 'schemes' in params: schemes = params['schemes'][0] # correctly forward the schemes and select to potential metalink followups cleaned_url = ctx.env.get('REQUEST_URI').split('?')[0] if isinstance(schemes, list): header( 'Link', '<%s/metalink?schemes=%s&select=%s>; rel=describedby; type="application/metalink+xml"' % (cleaned_url, ','.join(schemes), select)) else: header( 'Link', '<%s/metalink?schemes=%s&select=%s>; rel=describedby; type="application/metalink+xml"' % (cleaned_url, schemes, select)) schemes = [schemes] # list_replicas needs a list replicas = [ r for r in list_replicas(dids=[{ 'scope': scope, 'name': name, 'type': 'FILE' }], schemes=schemes) ] selected_url, selected_rse = None, None for r in replicas: if r['rses']: replicadict = {} if rse: if rse in r['rses'] and r['rses'][rse]: selected_url = r['rses'][rse][0] selected_rse = rse else: raise ReplicaNotFound( 'no redirection possible - no valid RSE for HTTP redirection found' ) else: for rep in r['rses']: for replica in r['rses'][rep]: replicadict[replica] = rep if not replicadict: raise ReplicaNotFound( 'no redirection possible - no valid RSE for HTTP redirection found' ) elif site: rep = site_selector(replicadict, site) if rep: selected_url = rep[0] else: raise ReplicaNotFound( 'no redirection possible - no valid RSE for HTTP redirection found' ) else: client_ip = ctx.env.get('HTTP_X_FORWARDED_FOR') if client_ip is None: client_ip = ctx.ip if select == 'geoip': rep = geoIP_order(replicadict, client_ip) else: rep = random_order(replicadict, client_ip) selected_url = rep[0] for rep in r['rses']: for replica in r['rses'][rep]: if selected_url == replica: selected_rse = rep if selected_url: if selected_url.startswith('s3+rucio://'): connect(selected_rse, selected_url) signed_URLS = get_signed_urls([selected_url], rse=selected_rse, operation='read') raise seeother(signed_URLS[selected_url]) raise seeother(selected_url) raise ReplicaNotFound( 'no redirection possible - file does not exist') except seeother: raise except ReplicaNotFound, e: raise generate_http_error(404, 'ReplicaNotFound', e.args[0][0])