def ping(self): ping_url = 'http://{}:{}/ping'.format(self.host, self.port) up = False while not up: try: retry_response_returning_functions(requests.get, ping_url) up = True except requests.exceptions.ConnectionError: time.sleep(0.01)
def test_authorized_users_only(): session = external_requests_client_session() endpoints = [ (session.get, '/api/v1alpha/billing_projects', 401), (session.get, '/api/v1alpha/billing_projects/foo', 401), (session.post, '/api/v1alpha/billing_projects/foo/users/foo/add', 401), (session.post, '/api/v1alpha/billing_projects/foo/users/foo/remove', 401), (session.post, '/api/v1alpha/billing_projects/foo/create', 401), (session.post, '/api/v1alpha/billing_projects/foo/close', 401), (session.post, '/api/v1alpha/billing_projects/foo/reopen', 401), (session.post, '/api/v1alpha/billing_projects/foo/delete', 401), (session.post, '/api/v1alpha/billing_limits/foo/edit', 401), (session.get, '/api/v1alpha/batches/0/jobs/0', 401), (session.get, '/api/v1alpha/batches/0/jobs/0/log', 401), (session.get, '/api/v1alpha/batches', 401), (session.post, '/api/v1alpha/batches/create', 401), (session.post, '/api/v1alpha/batches/0/jobs/create', 401), (session.get, '/api/v1alpha/batches/0', 401), (session.delete, '/api/v1alpha/batches/0', 401), (session.patch, '/api/v1alpha/batches/0/close', 401), # redirect to auth/login (session.get, '/batches', 302), (session.get, '/batches/0', 302), (session.post, '/batches/0/cancel', 401), (session.get, '/batches/0/jobs/0', 302), ] for method, url, expected in endpoints: full_url = deploy_config.url('batch', url) r = retry_response_returning_functions(method, full_url, allow_redirects=False) assert r.status_code == expected, (full_url, r, expected)
def test_batch_create_validation(self): bad_configs = [ # unexpected field fleep {'billing_project': 'foo', 'n_jobs': 5, 'token': 'baz', 'fleep': 'quam'}, # billing project None/missing {'billing_project': None, 'n_jobs': 5, 'token': 'baz'}, {'n_jobs': 5, 'token': 'baz'}, # n_jobs None/missing {'billing_project': 'foo', 'n_jobs': None, 'token': 'baz'}, {'billing_project': 'foo', 'token': 'baz'}, # n_jobs wrong type {'billing_project': 'foo', 'n_jobs': '5', 'token': 'baz'}, # token None/missing {'billing_project': 'foo', 'n_jobs': 5, 'token': None}, {'billing_project': 'foo', 'n_jobs': 5}, # attribute key/value None {'attributes': {'k': None}, 'billing_project': 'foo', 'n_jobs': 5, 'token': 'baz'}, ] url = deploy_config.url('batch', '/api/v1alpha/batches/create') headers = service_auth_headers(deploy_config, 'batch') for config in bad_configs: r = retry_response_returning_functions( requests.post, url, json=config, allow_redirects=True, headers=headers) assert r.status_code == 400, (config, r)
def __init__(self, *, url=None, config=None): if config is not None and url is not None: raise ValueError( f'Only specify one of the parameters url and config, ' f'received: url={url} and config={config}') if config is None: if url is None: config_path = pkg_resources.resource_filename( __name__, "annotation_db.json") assert os.path.exists( config_path), f'{config_path} does not exist' with open(config_path) as f: config = json.load(f) else: response = retry_response_returning_functions( requests.get, url) config = response.json() assert isinstance(config, dict) else: if not isinstance(config, dict): raise ValueError(f'expected a dict mapping dataset names to ' f'configurations, but found {config}') self.__by_name = { k: Dataset.from_name_and_json(k, v) for k, v in config.items() }
def add_reference(self, config): resp = retry_response_returning_functions( requests.post, f'{self.url}/references/create', json=config, headers=self.headers) if resp.status_code == 400 or resp.status_code == 500: resp_json = resp.json() raise FatalError(resp_json['message']) resp.raise_for_status()
def _request_type(self, ir, kind): code = self._render(ir) resp = retry_response_returning_functions( requests.post, f'{self.url}/type/{kind}', json=code, headers=self.headers) if resp.status_code == 400 or resp.status_code == 500: raise FatalError(resp.text) resp.raise_for_status() return resp.json()
def remove_sequence(self, name): resp = retry_response_returning_functions( requests.delete, f'{self.url}/references/sequence/delete', json={'name': name}, headers=self.headers) if resp.status_code == 400 or resp.status_code == 500: resp_json = resp.json() raise FatalError(resp_json['message']) resp.raise_for_status()
def get_reference(self, name): resp = retry_response_returning_functions(requests.get, f'{self.url}/references/get', json={'name': name}, headers=self.headers) if resp.status_code == 400: resp_json = resp.json() raise FatalError(resp_json['message']) resp.raise_for_status() return resp.json()
def add_sequence(self, name, fasta_file, index_file): resp = retry_response_returning_functions( requests.post, f'{self.url}/references/sequence/set', json={'name': name, 'fasta_file': fasta_file, 'index_file': index_file}, headers=self.headers) if resp.status_code == 400 or resp.status_code == 500: resp_json = resp.json() raise FatalError(resp_json['message']) resp.raise_for_status()
def add_liftover(self, name, chain_file, dest_reference_genome): resp = retry_response_returning_functions( requests.post, f'{self.url}/references/liftover/add', json={'name': name, 'chain_file': chain_file, 'dest_reference_genome': dest_reference_genome}, headers=self.headers) if resp.status_code == 400 or resp.status_code == 500: resp_json = resp.json() raise FatalError(resp_json['message']) resp.raise_for_status()
def parse_vcf_metadata(self, path): resp = retry_response_returning_functions( requests.post, f'{self.url}/parse-vcf-metadata', json={'path': path}, headers=self.headers) if resp.status_code == 400 or resp.status_code == 500: resp_json = resp.json() raise FatalError(resp_json['message']) resp.raise_for_status() return resp.json()
def remove_liftover(self, name, dest_reference_genome): resp = retry_response_returning_functions( requests.delete, f'{self.url}/references/liftover/remove', json={ 'name': name, 'dest_reference_genome': dest_reference_genome }, headers=self.headers) if resp.status_code == 400: resp_json = resp.json() raise FatalError(resp_json['message']) resp.raise_for_status()
def __init__(self, *, region: str = 'us', cloud: str = 'gcp', url: Optional[str] = None, config: Optional[dict] = None): if region not in DB._valid_regions: raise ValueError(f'Specify valid region parameter,' f' received: region={repr(region)}.\n' f'Valid regions are {DB._valid_regions}.') if cloud not in DB._valid_clouds: raise ValueError(f'Specify valid cloud parameter,' f' received: cloud={repr(cloud)}.\n' f'Valid cloud platforms are {DB._valid_clouds}.') if (region, cloud) not in DB._valid_combinations: raise ValueError(f'The {repr(region)} region is not available for' f' the {repr(cloud)} cloud platform. ' f'Valid region, cloud combinations are' f' {DB._valid_combinations}.') if config is not None and url is not None: raise ValueError( f'Only specify one of the parameters url and' f' config, received: url={url} and config={config}') if config is None: if url is None: config_path = pkg_resources.resource_filename( __name__, 'datasets.json') assert os.path.exists(config_path), \ f'{config_path} does not exist' with open(config_path) as f: config = json.load(f) else: session = external_requests_client_session() response = retry_response_returning_functions(session.get, url) config = response.json() assert isinstance(config, dict) else: if not isinstance(config, dict): raise ValueError(f'expected a dict mapping dataset names to ' f'configurations, but found {config}') config = {k: v for k, v in config.items() if 'annotation_db' in v} self.region = region self.cloud = cloud self.url = url self.config = config self.__by_name = { k: Dataset.from_name_and_json(k, v, region, cloud) for k, v in config.items() if Dataset.from_name_and_json(k, v, region, cloud) is not None }
def execute(self, ir, timed=False): code = self._render(ir) resp = retry_response_returning_functions(requests.post, f'{self.url}/execute', json=code, headers=self.headers) if resp.status_code == 400 or resp.status_code == 500: raise FatalError(resp.text) resp.raise_for_status() resp_json = resp.json() typ = dtype(resp_json['type']) value = typ._convert_from_json_na(resp_json['value']) # FIXME put back timings return (value, None) if timed else value
def import_fam(self, path: str, quant_pheno: bool, delimiter: str, missing: str): resp = retry_response_returning_functions( requests.post, f'{self.url}/import-fam', json={ 'path': path, 'quant_pheno': quant_pheno, 'delimiter': delimiter, 'missing': missing }, headers=self.headers) if resp.status_code == 400 or resp.status_code == 500: resp_json = resp.json() raise FatalError(resp_json['message']) resp.raise_for_status() return resp.json()
def from_fasta_file(self, name, fasta_file, index_file, x_contigs, y_contigs, mt_contigs, par): resp = retry_response_returning_functions( requests.post, f'{self.url}/references/create/fasta', json={ 'name': name, 'fasta_file': fasta_file, 'index_file': index_file, 'x_contigs': x_contigs, 'y_contigs': y_contigs, 'mt_contigs': mt_contigs, 'par': par }, headers=self.headers) if resp.status_code == 400 or resp.status_code == 500: resp_json = resp.json() raise FatalError(resp_json['message']) resp.raise_for_status()
def index_bgen(self, files, index_file_map, rg, contig_recoding, skip_invalid_loci): resp = retry_response_returning_functions( requests.post, f'{self.url}/index-bgen', json={ 'files': files, 'index_file_map': index_file_map, 'rg': rg, 'contig_recoding': contig_recoding, 'skip_invalid_loci': skip_invalid_loci }, headers=self.headers) if resp.status_code == 400 or resp.status_code == 500: resp_json = resp.json() raise FatalError(resp_json['message']) resp.raise_for_status() return resp.json()
def test_authorized_users_only(self): endpoints = [ (requests.get, '/api/v1alpha/batches/0/jobs/0', 401), (requests.get, '/api/v1alpha/batches/0/jobs/0/log', 401), (requests.get, '/api/v1alpha/batches', 401), (requests.post, '/api/v1alpha/batches/create', 401), (requests.post, '/api/v1alpha/batches/0/jobs/create', 401), (requests.get, '/api/v1alpha/batches/0', 401), (requests.delete, '/api/v1alpha/batches/0', 401), (requests.patch, '/api/v1alpha/batches/0/close', 401), # redirect to auth/login (requests.get, '/batches', 302), (requests.get, '/batches/0', 302), (requests.post, '/batches/0/cancel', 401), (requests.get, '/batches/0/jobs/0', 302)] for method, url, expected in endpoints: full_url = deploy_config.url('batch', url) r = retry_response_returning_functions( method, full_url, allow_redirects=False) assert r.status_code == expected, (full_url, r, expected)
def test_batch_create_validation(): bad_configs = [ # unexpected field fleep { 'billing_project': 'foo', 'n_jobs': 5, 'token': 'baz', 'fleep': 'quam' }, # billing project None/missing { 'billing_project': None, 'n_jobs': 5, 'token': 'baz' }, { 'n_jobs': 5, 'token': 'baz' }, # n_jobs None/missing { 'billing_project': 'foo', 'n_jobs': None, 'token': 'baz' }, { 'billing_project': 'foo', 'token': 'baz' }, # n_jobs wrong type { 'billing_project': 'foo', 'n_jobs': '5', 'token': 'baz' }, # token None/missing { 'billing_project': 'foo', 'n_jobs': 5, 'token': None }, { 'billing_project': 'foo', 'n_jobs': 5 }, # empty gcsfuse bucket name { 'billing_project': 'foo', 'n_jobs': 5, 'token': 'baz', 'gcsfuse': [{ 'bucket': '', 'mount_path': '/bucket', 'read_only': False }], }, # empty gcsfuse mount_path name { 'billing_project': 'foo', 'n_jobs': 5, 'token': 'baz', 'gcsfuse': [{ 'bucket': 'foo', 'mount_path': '', 'read_only': False }], }, # attribute key/value None { 'attributes': { 'k': None }, 'billing_project': 'foo', 'n_jobs': 5, 'token': 'baz' }, ] url = deploy_config.url('batch', '/api/v1alpha/batches/create') headers = service_auth_headers(deploy_config, 'batch') session = external_requests_client_session() for config in bad_configs: r = retry_response_returning_functions(session.post, url, json=config, allow_redirects=True, headers=headers) assert r.status_code == 400, (config, r)