def _load_json_data(self, response): """ Helper method to correctly load json data based on the content type of the http response. :param response: the response received from the server. """ if 'content-type' in response.headers and response.headers['content-type'] == 'application/x-json-stream': for line in response.iter_lines(): if line: yield parse_response(line) elif 'content-type' in response.headers and response.headers['content-type'] == 'application/json': yield parse_response(response.text) else: # Exception ? yield response.text
def _get_exception(self, headers, status_code=None, data=None): """ Helper method to parse an error string send by the server and transform it into the corresponding rucio exception. :param headers: The http response header containing the Rucio exception details. :param status_code: The http status code. :param data: The data with the ExceptionMessage. :return: A rucio exception class and an error string. """ try: data = parse_response(data) except ValueError: data = {} if 'ExceptionClass' not in data: if 'ExceptionMessage' not in data: human_http_code = _codes.get( status_code, None) # NOQA, pylint: disable-msg=W0612 return getattr( exception, 'RucioException' ), 'no error information passed (http status code: %(status_code)s %(human_http_code)s)' % locals( ) return getattr(exception, 'RucioException'), data['ExceptionMessage'] exc_cls = None try: exc_cls = getattr(exception, data['ExceptionClass']) except AttributeError: return getattr(exception, 'RucioException'), data['ExceptionMessage'] return exc_cls, data['ExceptionMessage']
def POST(self): """ List all meta of a list of data identifiers. HTTP Success: 200 OK HTTP Error: 400 Bad Request 401 Unauthorized 404 DataIdentifierNotFound 500 InternalError :returns: A list of dictionaries containing all meta. """ header('Content-Type', 'application/x-json-stream') json_data = data() try: params = parse_response(json_data) dids = params['dids'] except KeyError as error: raise generate_http_error(400, 'ValueError', 'Cannot find mandatory parameter : %s' % str(error)) except ValueError: raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list') try: for meta in get_metadata_bulk(dids, vo=ctx.env.get('vo')): yield render_json(**meta) + '\n' except ValueError: raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list') except DataIdentifierNotFound as error: raise generate_http_error(404, 'DataIdentifierNotFound', error.args[0]) except RucioException as error: raise generate_http_error(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) raise InternalError(error)
def post(self): """ Set a tombstone on a list of replicas. .. :quickref: Tombstone; Set a tombstone on a list of replicas. :<json string replicas: list fo replicas :resheader Content-Type: application/x-json-string :status 201: Created. :status 401: Invalid auth token. :status 404: ReplicaNotFound. :status 500: Internal Error. """ json_data = request.data replicas = [] try: params = parse_response(json_data) if 'replicas' in params: replicas = params['replicas'] except ValueError: return generate_http_error_flask(400, 'ValueError', 'Cannot decode json parameter list') try: for replica in replicas: set_tombstone(replica['rse'], replica['scope'], replica['name'], issuer=request.environ.get('issuer'), vo=request.environ.get('vo')) except ReplicaNotFound as error: return generate_http_error_flask(404, 'ReplicaNotFound', error.args[0]) except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) return str(error), 500 return 'Created', 201
def put(self): """ Update a file replicas state at a given RSE. .. :quickref: Replicas; update replicas state. :<json string rse: The RSE name. :<json list files: list of dicts with 'scope', 'name' and 'state'. :status 201: Replica successfully updated. :status 400: Cannot decode json parameter list. :status 401: Invalid auth token. :status 500: Internal Error. """ json_data = request.data try: parameters = parse_response(json_data) except ValueError: return generate_http_error_flask(400, 'ValueError', 'Cannot decode json parameter list') try: update_replicas_states(rse=parameters['rse'], files=parameters['files'], issuer=request.environ.get('issuer'), vo=request.environ.get('vo')) except AccessDenied as error: return generate_http_error_flask(401, 'AccessDenied', error.args[0]) except UnsupportedOperation as error: return generate_http_error_flask(500, 'UnsupportedOperation', error.args[0]) except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) return str(error), 500 return '', 200
def POST(self): """ Declare a list of suspicious replicas. HTTP Success: 200 OK HTTP Error: 401 Unauthorized 500 InternalError """ json_data = data() pfns = [] header('Content-Type', 'application/x-json-stream') try: params = parse_response(json_data) if 'pfns' in params: pfns = params['pfns'] if 'reason' in params: reason = params['reason'] except ValueError: raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list') not_declared_files = {} try: not_declared_files = declare_suspicious_file_replicas( pfns=pfns, reason=reason, issuer=ctx.env.get('issuer')) except ReplicaNotFound, e: raise generate_http_error(404, 'ReplicaNotFound', e.args[0][0])
def POST(self): """ List the DIDs associated to a list of replicas. HTTP Success: 200 OK HTTP Error: 401 Unauthorized 500 InternalError :returns: A list of dictionaries containing the mAPPing PFNs to DIDs. """ json_data = data() rse, pfns = None, [] header('Content-Type', 'application/x-json-stream') rse = None try: params = parse_response(json_data) if 'pfns' in params: pfns = params['pfns'] if 'rse' in params: rse = params['rse'] except ValueError: raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list') try: for pfn in get_did_from_pfns(pfns, rse): yield dumps(pfn) + '\n' except RucioException, e: raise generate_http_error(500, e.__class__.__name__, e.args[0][0])
def DELETE(self): """ Delete file replicas at a given RSE. HTTP Success: 200 Ok HTTP Error: 401 Unauthorized 409 Conflict 500 Internal Error """ json_data = data() try: parameters = parse_response(json_data) except ValueError: raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list') try: delete_replicas(rse=parameters['rse'], files=parameters['files'], issuer=ctx.env.get('issuer'), ignore_availability=parameters.get( 'ignore_availability', False)) except AccessDenied, e: raise generate_http_error(401, 'AccessDenied', e.args[0][0])
def POST(self): """ Create file replicas at a given RSE. HTTP Success: 201 Created HTTP Error: 401 Unauthorized 409 Conflict 500 Internal Error """ json_data = data() try: parameters = parse_response(json_data) except ValueError: raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list') try: add_replicas(rse=parameters['rse'], files=parameters['files'], issuer=ctx.env.get('issuer'), ignore_availability=parameters.get( 'ignore_availability', False)) except InvalidPath, e: raise generate_http_error(400, 'InvalidPath', e.args[0][0])
def POST(self, rse, operation): """ Get URLs for files at a given RSE. HTTP Success: 200 OK HTTP Error: 401 Unauthorized 500 Internal Error """ header('Content-Type', 'application/json') json_data = data() try: parameters = parse_response(json_data) except ValueError: raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list') try: result = objectstore.get_signed_urls(parameters, rse=rse, operation=operation) for url in result: if isinstance(result[url], Exception): raise result[url] return render_json(**result) except RucioException as error: raise generate_http_error(500, error.__class__.__name__, error.args[0]) except Exception as error: print traceback.format_exc() raise InternalError(error)
def POST(self, rse): """ Get files metadata at a given RSE. HTTP Success: 200 OK HTTP Error: 401 Unauthorized 500 Internal Error """ header('Content-Type', 'application/json') json_data = data() try: parameters = parse_response(json_data) except ValueError: raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list') try: url = parameters['url'] new_url = parameters['new_url'] objectstore.rename(url, new_url, rse=rse) except RucioException as error: raise generate_http_error(500, error.__class__.__name__, error.args[0]) except Exception as error: print traceback.format_exc() raise InternalError(error) raise OK()
def POST(self): """ Set a tombstone on a list of replicas. HTTP Success: 201 OK HTTP Error: 401 Unauthorized 404 ReplicaNotFound 500 InternalError """ json_data = data() replicas = [] try: params = parse_response(json_data) if 'replicas' in params: replicas = params['replicas'] except ValueError: raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list') try: for replica in replicas: set_tombstone(replica['rse'], replica['scope'], replica['name'], issuer=ctx.env.get('issuer'), vo=ctx.env.get('vo')) except ReplicaNotFound as error: raise generate_http_error(404, 'ReplicaNotFound', error.args[0]) except RucioException as error: raise generate_http_error(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) raise InternalError(error) raise Created()
def POST(self): """ Import data into Rucio. HTTP Success: 200 OK HTTP Error: 400 Bad request 401 Unauthorized 404 Resource not Found 500 InternalError """ header('Content-Type', 'application/x-json-stream') json_data = data() try: data_to_import = json_data and parse_response(json_data) except ValueError: raise generate_http_error( 400, 'ValueError', 'Cannot decode json parameter dictionary') try: import_data(data=data_to_import, issuer=ctx.env.get('issuer'), vo=ctx.env.get('vo')) except RucioException as error: raise generate_http_error(500, error.__class__.__name__, error.args[0]) raise Created()
def _load_json_data(self, response): """ Helper method to correctly load json data based on the content type of the http response. :param response: the response received from the server. """ if 'content-type' in response.headers and response.headers[ 'content-type'] == 'application/x-json-stream': for line in response.iter_lines(): if line: yield parse_response(line) elif 'content-type' in response.headers and response.headers[ 'content-type'] == 'application/json': yield parse_response(response.text) else: # Exception ? yield response.text
def post(self, rse): """ Pass list of urls and return their metadata. .. :quickref: ObjectStoreInfo; Get files metadata at a given RSE. :param rse: RSE name. :<json string urls: A list of URL string. :resheader Content-Type: application/json :status 200: OK. :status 400: Cannot decode json parameter list. :status 401: Invalid Auth Token. :status 500: Internal Error. :returns: Dictonary of metadata. """ json_data = request.data try: parameters = parse_response(json_data) except ValueError: return generate_http_error_flask( 400, 'ValueError', 'Cannot decode json parameter list') try: result = objectstore.get_metadata(parameters, rse=rse) for url in result: if isinstance(result[url], Exception): return Response(result[url]) return Response(render_json(**result), content_type="application/json") except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: print(traceback.format_exc()) return error, 500
def POST(self): """ Declare a list of bad replicas. HTTP Success: 200 OK HTTP Error: 401 Unauthorized 500 InternalError """ json_data = data() rse, pfns = None, [] header('Content-Type', 'application/x-json-stream') try: params = parse_response(json_data) if 'pfns' in params: pfns = params['pfns'] if 'rse' in params: rse = params['rse'] except ValueError: raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list') try: declare_bad_file_replicas(rse=rse, pfns=pfns, issuer=ctx.env.get('issuer')) except ReplicaNotFound, e: raise generate_http_error(404, 'ReplicaNotFound', e.args[0][0])
def PUT(self): """ Update a file replicas state at a given RSE. HTTP Success: 200 OK HTTP Error: 401 Unauthorized 500 Internal Error """ json_data = data() try: parameters = parse_response(json_data) except ValueError: raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list') try: update_replicas_states(rse=parameters['rse'], files=parameters['files'], issuer=ctx.env.get('issuer')) except AccessDenied as error: raise generate_http_error(401, 'AccessDenied', error.args[0]) except UnsupportedOperation as error: raise generate_http_error(500, 'UnsupportedOperation', error.args[0]) except RucioException as error: raise generate_http_error(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) raise InternalError(error) raise OK()
def post(self, rse): """ Rename object. .. :quickref: ObjectStoreRename; Rename object. :param rse: the RSE name. :<json string url: the URL string. :<json string new_url: the new URL string. :<json string urls: A list of URL string. :resheader Content-Type: application/json :status 200: OK. :status 400: Cannot decode json parameter list. :status 401: Invalid Auth Token. :status 500: Internal Error. """ json_data = request.data try: parameters = parse_response(json_data) except ValueError: return generate_http_error_flask( 400, 'ValueError', 'Cannot decode json parameter list') try: url = parameters['url'] new_url = parameters['new_url'] objectstore.rename(url, new_url, rse=rse) except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: print(traceback.format_exc()) return error, 500 return "OK", 200
def check_error_api(params, exception_class, exception_message, code): headers_dict = {'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root'} response = rest_client.get('/requests/list', query_string=params, headers=headers(auth(auth_token), vohdr(vo), hdrdict(headers_dict))) assert response.status_code == code body = parse_response(response.get_data(as_text=True)) assert body['ExceptionClass'] == exception_class assert body['ExceptionMessage'] == exception_message
def POST(self): """ List the DIDs associated to a list of replicas. HTTP Success: 200 OK HTTP Error: 401 Unauthorized 500 InternalError :returns: A list of dictionaries containing the mapping PFNs to DIDs. """ json_data = data() rse, pfns = None, [] header('Content-Type', 'application/x-json-stream') try: params = parse_response(json_data) if 'pfns' in params: pfns = params['pfns'] if 'rse' in params: rse = params['rse'] except ValueError: raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list') try: for pfn in get_did_from_pfns(pfns, rse): yield dumps(pfn) + '\n' except RucioException, e: raise generate_http_error(500, e.__class__.__name__, e.args[0][0])
def post(self): """ Declare a list of bad PFNs. .. :quickref: BadPFNs; Declare bad replicas. :<json string pfns: The list of PFNs. :<json string reason: The reason of the loss. :<json string state: The state is eiher BAD, SUSPICIOUS or TEMPORARY_UNAVAILABLE. :<json string expires_at: The expiration date. Only apply to TEMPORARY_UNAVAILABLE. :resheader Content-Type: application/x-json-string :status 201: Created. :status 400: Cannot decode json parameter list. :status 401: Invalid auth token. :status 404: Replica not found. :status 500: Internal Error. :returns: A list of not successfully declared files. """ json_data = request.data pfns = [] reason = None state = None expires_at = None try: params = parse_response(json_data) if 'pfns' in params: pfns = params['pfns'] if 'reason' in params: reason = params['reason'] if 'reason' in params: reason = params['reason'] if 'state' in params: reason = params['state'] if 'expires_at' in params: expires_at = datetime.strptime(params['expires_at'], "%Y-%m-%dT%H:%M:%S.%f") add_bad_pfns(pfns=pfns, issuer=request.environ.get('issuer'), state=state, reason=reason, expires_at=expires_at) except (ValueError, InvalidType) as error: return generate_http_error_flask(400, 'ValueError', error.args[0]) except AccessDenied as error: return generate_http_error_flask(401, 'AccessDenied', error.args[0]) except ReplicaNotFound as error: return generate_http_error_flask(404, 'ReplicaNotFound', error.args[0]) except Duplicate as error: return generate_http_error_flask(409, 'Duplicate', error.args[0]) except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) return error, 500 return 'Created', 201
def test_auth_ssh(self): """ MULTI VO (REST): Test ssh authentication to multiple VOs """ mw = [] try: add_account_identity(PUBLIC_KEY, 'SSH', 'root', '*****@*****.**', 'root', **self.vo) add_account_identity(PUBLIC_KEY, 'SSH', 'root', '*****@*****.**', 'root', **self.new_vo) except Duplicate: pass # Might already exist, can skip headers_tst = {'X-Rucio-Account': 'root'} headers_tst.update(self.vo_header) res_tst = TestApp(auth_app.wsgifunc(*mw)).get('/ssh_challenge_token', headers=headers_tst, expect_errors=True) assert_equal(res_tst.status, 200) challenge_tst = str(res_tst.header('X-Rucio-SSH-Challenge-Token')) headers_tst.update({'X-Rucio-SSH-Signature': ssh_sign(PRIVATE_KEY, challenge_tst)}) res_tst = TestApp(auth_app.wsgifunc(*mw)).get('/ssh', headers=headers_tst, expect_errors=True) assert_equal(res_tst.status, 200) token_tst = str(res_tst.header('X-Rucio-Auth-Token')) headers_new = {'X-Rucio-Account': 'root'} headers_new.update(self.new_header) res_new = TestApp(auth_app.wsgifunc(*mw)).get('/ssh_challenge_token', headers=headers_new, expect_errors=True) assert_equal(res_new.status, 200) challenge_tst = str(res_new.header('X-Rucio-SSH-Challenge-Token')) headers_new.update({'X-Rucio-SSH-Signature': ssh_sign(PRIVATE_KEY, challenge_tst)}) res_new = TestApp(auth_app.wsgifunc(*mw)).get('/ssh', headers=headers_new, expect_errors=True) assert_equal(res_new.status, 200) token_new = str(res_new.header('X-Rucio-Auth-Token')) headers_tst = {'X-Rucio-Auth-Token': str(token_tst)} res_tst = TestApp(account_app.wsgifunc(*mw)).get('/', headers=headers_tst, expect_errors=True) assert_equal(res_tst.status, 200) accounts_tst = [parse_response(a)['account'] for a in res_tst.body.decode().split('\n')[:-1]] assert_not_equal(len(accounts_tst), 0) assert_in(self.account_tst, accounts_tst) assert_not_in(self.account_new, accounts_tst) headers_new = {'X-Rucio-Auth-Token': str(token_new)} res_new = TestApp(account_app.wsgifunc(*mw)).get('/', headers=headers_new, expect_errors=True) assert_equal(res_new.status, 200) accounts_new = [parse_response(a)['account'] for a in res_new.body.decode().split('\n')[:-1]] assert_not_equal(len(accounts_new), 0) assert_in(self.account_new, accounts_new) assert_not_in(self.account_tst, accounts_new)
def check_correct_api(params, expected_requests): headers_dict = {'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root'} response = rest_client.get('/requests/history/list', query_string=params, headers=headers(auth(auth_token), vohdr(vo), hdrdict(headers_dict))) assert response.status_code == 200 requests = set() for request in response.get_data(as_text=True).split('\n')[:-1]: request = parse_response(request) requests.add((request['state'], request['source_rse_id'], request['dest_rse_id'], request['name'])) assert requests == expected_requests
def post(self): """ Create file replicas at a given RSE. .. :quickref: Replicas; create replicas at RSE :<json string rse: The RSE name. :<json list files: list of dicts with 'scope', 'name', 'bytes', 'meta' and 'adler32'. :<json bool ignore_availability: Flag to ignore the RSE blacklisting. :status 201: Replica Successfully created. :status 400: Invalid Path. :status 401: Invalid auth token. :status 404: RSE not found. :status 409: Replica already exists. :status 409: DID already exists. :status 503: Resource Temporary Unavailable. """ json_data = request.data try: parameters = parse_response(json_data) except ValueError: return generate_http_error_flask( 400, 'ValueError', 'Cannot decode json parameter list') try: add_replicas(rse=parameters['rse'], files=parameters['files'], issuer=request.environ.get('issuer'), vo=request.environ.get('vo'), ignore_availability=parameters.get( 'ignore_availability', False)) except InvalidPath as error: return generate_http_error_flask(400, 'InvalidPath', error.args[0]) except AccessDenied as error: return generate_http_error_flask(401, 'AccessDenied', error.args[0]) except Duplicate as error: return generate_http_error_flask(409, 'Duplicate', error.args[0]) except DataIdentifierAlreadyExists as error: return generate_http_error_flask(409, 'DataIdentifierAlreadyExists', error.args[0]) except RSENotFound as error: return generate_http_error_flask(404, 'RSENotFound', error.args[0]) except ResourceTemporaryUnavailable as error: return generate_http_error_flask(503, 'ResourceTemporaryUnavailable', error.args[0]) except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) return error, 500 return 'Created', 201
def POST(self): """ Declare a list of bad replicas by DID. HTTP Success: 200 OK HTTP Error: 400 BadRequest 401 Unauthorized 409 Conflict 500 InternalError """ json_data = data() dids = [] rse = None reason = None expires_at = None header('Content-Type', 'application/x-json-stream') try: params = parse_response(json_data) if 'dids' in params: dids = params['dids'] if 'rse' in params: rse = params['rse'] if 'reason' in params: reason = params['reason'] state = BadFilesStatus.BAD if 'expires_at' in params and params['expires_at']: expires_at = datetime.strptime(params['expires_at'], "%Y-%m-%dT%H:%M:%S.%f") not_declared_files = add_bad_dids(dids=dids, rse=rse, issuer=ctx.env.get('issuer'), state=state, reason=reason, expires_at=expires_at, vo=ctx.env.get('vo')) except (ValueError, InvalidType) as error: raise generate_http_error(400, 'ValueError', error.args[0]) except AccessDenied as error: raise generate_http_error(401, 'AccessDenied', error.args[0]) except ReplicaNotFound as error: raise generate_http_error(404, 'ReplicaNotFound', error.args[0]) except Duplicate as error: raise generate_http_error(409, 'Duplicate', error.args[0]) except RucioException as error: raise generate_http_error(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) raise InternalError(error) raise Created(dumps(not_declared_files))
def post(self): """ List dataset replicas for multiple DIDs. .. :quickref: DatasetReplicasBulk; List dataset replicas (bulk). :<json list dids: List of DIDs for querying the datasets. :resheader Content-Type: application/x-json-stream :status 200: OK. :status 400: Bad Request. :status 401: Invalid auth token. :status 406: Not Acceptable. :status 500: Internal Error. :returns: A dictionary containing all replicas information. """ json_data = request.data try: params = parse_response(json_data) dids = params['dids'] didslength = len(dids) except KeyError as error: return generate_http_error_flask( 400, 'KeyError', 'Cannot find mandatory parameter : %s' % str(error)) except ValueError: return generate_http_error_flask( 400, 'ValueError', 'Cannot decode json parameter list') except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) return error, 500 if didslength == 0: return generate_http_error_flask(400, 'ValueError', 'List of DIDs is empty') try: data = "" for row in list_dataset_replicas_bulk( dids=dids, vo=request.environ.get('vo')): data += dumps(row, cls=APIEncoder) + '\n' return Response(data, content_type='application/x-json-stream') except InvalidObject as error: return generate_http_error_flask( 400, 'InvalidObject', 'Cannot validate DIDs: %s' % (str(error))) except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) return error, 500
def POST(self): """ Create file replicas at a given RSE. HTTP Success: 201 Created HTTP Error: 401 Unauthorized 405 Method Not Allowed 409 Conflict 500 Internal Error 503 Service Unavailable """ json_data = data() try: parameters = parse_response(json_data) except (ValueError, InvalidType): raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list') try: add_files(lfns=parameters['lfns'], issuer=ctx.env.get('issuer'), ignore_availability=parameters.get( 'ignore_availability', False)) except InvalidPath as error: raise generate_http_error(400, 'InvalidPath', error.args[0]) except AccessDenied as error: raise generate_http_error(401, 'AccessDenied', error.args[0]) except UnsupportedOperation as error: raise generate_http_error(405, 'UnsupportedOperation', error.args[0]) except Duplicate as error: raise generate_http_error(409, 'Duplicate', error.args[0]) except DataIdentifierAlreadyExists as error: raise generate_http_error(409, 'DataIdentifierAlreadyExists', error.args[0]) except RSENotFound as error: raise generate_http_error(404, 'RSENotFound', error.args[0]) except DatabaseException as error: raise generate_http_error(503, 'DatabaseException', error.args[0]) except ResourceTemporaryUnavailable as error: raise generate_http_error(503, 'ResourceTemporaryUnavailable', error.args[0]) except RucioException as error: print(format_exc()) raise generate_http_error(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) raise InternalError(error) raise Created()
def post(self): """ Atomic method used by the RucioFileCatalog plugin in Dirac that : - Creates files and their replicas - Creates the dataset containing the files and attach the files to the dataset - Creates a rule on the dataset with RSE expression ANY and grouping NONE - Creates all the container hierarchy containing the dataset ..:quickref: AddFiles; Method used by the RucioFileCatalog plugin in Dirac. :<json list lfns: List of lfn (dictionary {'lfn': <lfn>, 'rse': <rse>, 'bytes': <bytes>, 'adler32': <adler32>, 'guid': <guid>, 'pfn': <pfn>}. :<json bool ignore_availability: A boolean to choose if unavailable sites need to be ignored. :status 201: Created. :status 400: Cannot decode json parameter list. :status 401: Invalid auth token. :status 404: DID not found. :status 405: Unsupported Operation. :status 409: Duplicate. :status 500: Internal Error. :status 503: Temporary error. """ try: parameters = parse_response(request.data) except (ValueError, InvalidType): return generate_http_error_flask(400, 'ValueError', 'Cannot decode json parameter list') try: add_files(lfns=parameters['lfns'], issuer=request.environ.get('issuer'), ignore_availability=parameters.get('ignore_availability', False)) except InvalidPath as error: return generate_http_error_flask(400, 'InvalidPath', error.args[0]) except AccessDenied as error: return generate_http_error_flask(401, 'AccessDenied', error.args[0]) except UnsupportedOperation as error: return generate_http_error_flask(405, 'UnsupportedOperation', error.args[0]) except Duplicate as error: return generate_http_error_flask(409, 'Duplicate', error.args[0]) except DataIdentifierAlreadyExists as error: return generate_http_error_flask(409, 'DataIdentifierAlreadyExists', error.args[0]) except RSENotFound as error: return generate_http_error_flask(404, 'RSENotFound', error.args[0]) except DatabaseException as error: return generate_http_error_flask(503, 'DatabaseException', error.args[0]) except ResourceTemporaryUnavailable as error: return generate_http_error_flask(503, 'ResourceTemporaryUnavailable', error.args[0]) except RucioException as error: logging.exception("Internal Error") return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: logging.exception("Internal Error") return str(error), 500 return 'Created', 201
def POST(self): """ Declare a list of bad PFNs. HTTP Success: 200 OK HTTP Error: 400 BadRequest 401 Unauthorized 409 Conflict 500 InternalError """ json_data = data() pfns = [] reason = None state = None expires_at = None header('Content-Type', 'application/x-json-stream') try: params = parse_response(json_data) if 'pfns' in params: pfns = params['pfns'] if 'reason' in params: reason = params['reason'] if 'state' in params: state = params['state'] if 'expires_at' in params: expires_at = datetime.strptime(params['expires_at'], "%Y-%m-%dT%H:%M:%S.%f") add_bad_pfns(pfns=pfns, issuer=ctx.env.get('issuer'), state=state, reason=reason, expires_at=expires_at) except (ValueError, InvalidType) as error: raise generate_http_error(400, 'ValueError', error.args[0]) except AccessDenied as error: raise generate_http_error(401, 'AccessDenied', error.args[0]) except ReplicaNotFound as error: raise generate_http_error(404, 'ReplicaNotFound', error.args[0]) except Duplicate as error: raise generate_http_error(409, 'Duplicate', error.args[0]) except RucioException as error: raise generate_http_error(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) raise InternalError(error) raise Created()
def POST(self): """ List dataset replicas for multiple DIDs. HTTP Success: 200 OK HTTP Error: 400 Bad Request 401 Unauthorized 406 Not Acceptable 500 InternalError :returns: A dictionary containing all replicas information. """ header('Content-Type', 'application/x-json-stream') json_data = data() try: params = parse_response(json_data) dids = params['dids'] didslength = len(dids) except KeyError as error: raise generate_http_error( 400, 'KeyError', 'Cannot find mandatory parameter : %s' % str(error)) except ValueError: raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list') except RucioException as error: raise generate_http_error(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) raise InternalError(error) if didslength == 0: raise generate_http_error(400, 'ValueError', 'List of DIDs is empty') try: for row in list_dataset_replicas_bulk(dids=dids, vo=ctx.env.get('vo')): yield dumps(row, cls=APIEncoder) + '\n' except InvalidObject as error: raise generate_http_error( 400, 'InvalidObject', 'Cannot validate DIDs: %s' % (str(error))) except RucioException as error: raise generate_http_error(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) raise InternalError(error)
def test_auth_gss(self): """ MULTI VO (REST): Test gss authentication to multiple VOs """ mw = [] # Can't rely on `requests_kerberos` module being present, so get tokens from API instead token_tst = get_auth_token_gss('root', '*****@*****.**', 'unknown', None, **self.vo).token token_new = get_auth_token_gss('root', '*****@*****.**', 'unknown', None, **self.new_vo).token headers_tst = {'X-Rucio-Auth-Token': str(token_tst)} res_tst = TestApp(account_app.wsgifunc(*mw)).get('/', headers=headers_tst, expect_errors=True) assert_equal(res_tst.status, 200) accounts_tst = [parse_response(a)['account'] for a in res_tst.body.decode().split('\n')[:-1]] assert_not_equal(len(accounts_tst), 0) assert_in(self.account_tst, accounts_tst) assert_not_in(self.account_new, accounts_tst) headers_new = {'X-Rucio-Auth-Token': str(token_new)} res_new = TestApp(account_app.wsgifunc(*mw)).get('/', headers=headers_new, expect_errors=True) assert_equal(res_new.status, 200) accounts_new = [parse_response(a)['account'] for a in res_new.body.decode().split('\n')[:-1]] assert_not_equal(len(accounts_new), 0) assert_in(self.account_new, accounts_new) assert_not_in(self.account_tst, accounts_new)
def test_auth_x509(self): """ MULTI VO (REST): Test X509 authentication to multiple VOs """ mw = [] # TestApp doesn't support `cert` argument, so get tokens from API instead token_tst = get_auth_token_x509('root', '/CN=Rucio User', 'unknown', None, **self.vo).token token_new = get_auth_token_x509('root', '/CN=Rucio User', 'unknown', None, **self.new_vo).token headers_tst = {'X-Rucio-Auth-Token': str(token_tst)} res_tst = TestApp(account_app.wsgifunc(*mw)).get('/', headers=headers_tst, expect_errors=True) assert_equal(res_tst.status, 200) accounts_tst = [parse_response(a)['account'] for a in res_tst.body.decode().split('\n')[:-1]] assert_not_equal(len(accounts_tst), 0) assert_in(self.account_tst, accounts_tst) assert_not_in(self.account_new, accounts_tst) headers_new = {'X-Rucio-Auth-Token': str(token_new)} res_new = TestApp(account_app.wsgifunc(*mw)).get('/', headers=headers_new, expect_errors=True) assert_equal(res_new.status, 200) accounts_new = [parse_response(a)['account'] for a in res_new.body.decode().split('\n')[:-1]] assert_not_equal(len(accounts_new), 0) assert_in(self.account_new, accounts_new) assert_not_in(self.account_tst, accounts_new)
def delete(self): """ Delete file replicas at a given RSE. .. :quickref: Replicas; Delete replica at RSE. :<json string rse: The RSE name. :<json list files: list of dicts with 'scope', 'name'. :<json bool ignore_availability: Flag to ignore the RSE blacklisting. :status 200: Replica successfully deleted. :status 400: Cannot decode json parameter list. :status 401: Invalid auth token. :status 404: RSE not found. :status 404: Replica not found. :status 500: Internal Error. """ json_data = request.data try: parameters = parse_response(json_data) except ValueError: return generate_http_error_flask( 400, 'ValueError', 'Cannot decode json parameter list') try: delete_replicas(rse=parameters['rse'], files=parameters['files'], issuer=request.environ.get('issuer'), vo=request.environ.get('vo'), ignore_availability=parameters.get( 'ignore_availability', False)) except AccessDenied as error: return generate_http_error_flask(401, 'AccessDenied', error.args[0]) except RSENotFound as error: return generate_http_error_flask(404, 'RSENotFound', error.args[0]) except ResourceTemporaryUnavailable as error: return generate_http_error_flask(503, 'ResourceTemporaryUnavailable', error.args[0]) except ReplicaNotFound as error: return generate_http_error_flask(404, 'ReplicaNotFound', error.args[0]) except RucioException as error: return generate_http_error_flask(500, error.__class__.__name__, error.args[0]) except Exception as error: print(format_exc()) return error, 500 return 'OK', 200
def PUT(self): """ Update a file replicas state at a given RSE. HTTP Success: 200 OK HTTP Error: 401 Unauthorized 500 Internal Error """ json_data = data() try: parameters = parse_response(json_data) except ValueError: raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list') try: update_replicas_states(rse=parameters['rse'], files=parameters['files'], issuer=ctx.env.get('issuer')) except AccessDenied, e: raise generate_http_error(401, 'AccessDenied', e.args[0][0])
def DELETE(self): """ Delete file replicas at a given RSE. HTTP Success: 200 Ok HTTP Error: 401 Unauthorized 409 Conflict 500 Internal Error """ json_data = data() try: parameters = parse_response(json_data) except ValueError: raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list') try: delete_replicas(rse=parameters['rse'], files=parameters['files'], issuer=ctx.env.get('issuer'), ignore_availability=parameters.get('ignore_availability', False)) except AccessDenied, e: raise generate_http_error(401, 'AccessDenied', e.args[0][0])
def POST(self): """ List all replicas for data identifiers. HTTP Success: 200 OK HTTP Error: 401 Unauthorized 500 InternalError :returns: A dictionary containing all replicas information. :returns: A metalink description of replicas if metalink(4)+xml is specified in Accept: """ metalink = None if ctx.env.get('HTTP_ACCEPT') is not None: tmp = ctx.env.get('HTTP_ACCEPT').split(',') # first check if client accepts metalink if 'application/metalink+xml' in tmp: metalink = 3 # but prefer metalink4 if the client has support for it # (clients can put both in their ACCEPT header!) if 'application/metalink4+xml' in tmp: metalink = 4 dids, schemes, select, unavailable, limit = [], None, None, False, None ignore_availability = False all_states = False json_data = data() try: params = parse_response(json_data) if 'dids' in params: dids = params['dids'] if 'schemes' in params: schemes = params['schemes'] if 'unavailable' in params: unavailable = params['unavailable'] ignore_availability = True if 'all_states' in params: all_states = params['all_states'] except ValueError: raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter list') if ctx.query: params = parse_qs(ctx.query[1:]) if 'select' in params: select = params['select'][0] if 'limit' in params: limit = params['limit'][0] try: # first, set the appropriate content type, and stream the header if metalink is None: header('Content-Type', 'application/x-json-stream') elif metalink == 3: header('Content-Type', 'application/metalink+xml') yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink version="3.0" xmlns="http://www.metalinker.org/">\n<files>\n' elif metalink == 4: header('Content-Type', 'application/metalink4+xml') yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n' # then, stream the replica information for rfile in list_replicas(dids=dids, schemes=schemes, unavailable=unavailable, request_id=ctx.env.get('request_id'), ignore_availability=ignore_availability, all_states=all_states): client_ip = ctx.get('ip') replicas = [] dictreplica = {} for rse in rfile['rses']: for replica in rfile['rses'][rse]: replicas.append(replica) dictreplica[replica] = rse if select == 'geoip': replicas = geoIP_order(dictreplica, client_ip) else: replicas = random_order(dictreplica, client_ip) if metalink is None: yield dumps(rfile, cls=APIEncoder) + '\n' elif metalink == 3: idx = 0 yield ' <file name="' + rfile['name'] + '">\n <resources>\n' for replica in replicas: yield ' <url type="http" preference="' + str(idx) + '">' + replica + '</url>\n' idx += 1 if limit and limit == idx: break yield ' </resources>\n </file>\n' elif metalink == 4: yield ' <file name="' + rfile['name'] + '">\n' yield ' <identity>' + rfile['scope'] + ':' + rfile['name'] + '</identity>\n' if rfile['adler32'] is not None: yield ' <hash type="adler32">' + rfile['adler32'] + '</hash>\n' if rfile['md5'] is not None: yield ' <hash type="md5">' + rfile['md5'] + '</hash>\n' yield ' <size>' + str(rfile['bytes']) + '</size>\n' idx = 0 for replica in replicas: yield ' <url location="' + str(dictreplica[replica]) + '" priority="' + str(idx + 1) + '">' + replica + '</url>\n' idx += 1 if limit and limit == idx: break yield ' </file>\n' # don't forget to send the metalink footer if metalink: if metalink == 3: yield '</files>\n</metalink>\n' elif metalink == 4: yield '</metalink>\n' except DataIdentifierNotFound, e: raise generate_http_error(404, 'DataIdentifierNotFound', e.args[0][0])