def ping(self):
ping_url = 'http://{}:{}/ping'.format(self.host, self.port)
up = False
while not up:
try:
retry_response_returning_functions(requests.get, ping_url)
up = True
except requests.exceptions.ConnectionError:
time.sleep(0.01)
def test_authorized_users_only():
session = external_requests_client_session()
endpoints = [
(session.get, '/api/v1alpha/billing_projects', 401),
(session.get, '/api/v1alpha/billing_projects/foo', 401),
(session.post, '/api/v1alpha/billing_projects/foo/users/foo/add', 401),
(session.post, '/api/v1alpha/billing_projects/foo/users/foo/remove',
401),
(session.post, '/api/v1alpha/billing_projects/foo/create', 401),
(session.post, '/api/v1alpha/billing_projects/foo/close', 401),
(session.post, '/api/v1alpha/billing_projects/foo/reopen', 401),
(session.post, '/api/v1alpha/billing_projects/foo/delete', 401),
(session.post, '/api/v1alpha/billing_limits/foo/edit', 401),
(session.get, '/api/v1alpha/batches/0/jobs/0', 401),
(session.get, '/api/v1alpha/batches/0/jobs/0/log', 401),
(session.get, '/api/v1alpha/batches', 401),
(session.post, '/api/v1alpha/batches/create', 401),
(session.post, '/api/v1alpha/batches/0/jobs/create', 401),
(session.get, '/api/v1alpha/batches/0', 401),
(session.delete, '/api/v1alpha/batches/0', 401),
(session.patch, '/api/v1alpha/batches/0/close', 401),
# redirect to auth/login
(session.get, '/batches', 302),
(session.get, '/batches/0', 302),
(session.post, '/batches/0/cancel', 401),
(session.get, '/batches/0/jobs/0', 302),
]
for method, url, expected in endpoints:
full_url = deploy_config.url('batch', url)
r = retry_response_returning_functions(method,
full_url,
allow_redirects=False)
assert r.status_code == expected, (full_url, r, expected)
def test_batch_create_validation(self):
bad_configs = [
# unexpected field fleep
{'billing_project': 'foo', 'n_jobs': 5, 'token': 'baz', 'fleep': 'quam'},
# billing project None/missing
{'billing_project': None, 'n_jobs': 5, 'token': 'baz'},
{'n_jobs': 5, 'token': 'baz'},
# n_jobs None/missing
{'billing_project': 'foo', 'n_jobs': None, 'token': 'baz'},
{'billing_project': 'foo', 'token': 'baz'},
# n_jobs wrong type
{'billing_project': 'foo', 'n_jobs': '5', 'token': 'baz'},
# token None/missing
{'billing_project': 'foo', 'n_jobs': 5, 'token': None},
{'billing_project': 'foo', 'n_jobs': 5},
# attribute key/value None
{'attributes': {'k': None}, 'billing_project': 'foo', 'n_jobs': 5, 'token': 'baz'},
]
url = deploy_config.url('batch', '/api/v1alpha/batches/create')
headers = service_auth_headers(deploy_config, 'batch')
for config in bad_configs:
r = retry_response_returning_functions(
requests.post,
url,
json=config,
allow_redirects=True,
headers=headers)
assert r.status_code == 400, (config, r)
def __init__(self, *, url=None, config=None):
if config is not None and url is not None:
raise ValueError(
f'Only specify one of the parameters url and config, '
f'received: url={url} and config={config}')
if config is None:
if url is None:
config_path = pkg_resources.resource_filename(
__name__, "annotation_db.json")
assert os.path.exists(
config_path), f'{config_path} does not exist'
with open(config_path) as f:
config = json.load(f)
else:
response = retry_response_returning_functions(
requests.get, url)
config = response.json()
assert isinstance(config, dict)
else:
if not isinstance(config, dict):
raise ValueError(f'expected a dict mapping dataset names to '
f'configurations, but found {config}')
self.__by_name = {
k: Dataset.from_name_and_json(k, v)
for k, v in config.items()
}
def add_reference(self, config):
resp = retry_response_returning_functions(
requests.post,
f'{self.url}/references/create', json=config, headers=self.headers)
if resp.status_code == 400 or resp.status_code == 500:
resp_json = resp.json()
raise FatalError(resp_json['message'])
resp.raise_for_status()
def _request_type(self, ir, kind):
code = self._render(ir)
resp = retry_response_returning_functions(
requests.post,
f'{self.url}/type/{kind}', json=code, headers=self.headers)
if resp.status_code == 400 or resp.status_code == 500:
raise FatalError(resp.text)
resp.raise_for_status()
return resp.json()
def remove_sequence(self, name):
resp = retry_response_returning_functions(
requests.delete,
f'{self.url}/references/sequence/delete',
json={'name': name},
headers=self.headers)
if resp.status_code == 400 or resp.status_code == 500:
resp_json = resp.json()
raise FatalError(resp_json['message'])
resp.raise_for_status()
def get_reference(self, name):
resp = retry_response_returning_functions(requests.get,
f'{self.url}/references/get',
json={'name': name},
headers=self.headers)
if resp.status_code == 400:
resp_json = resp.json()
raise FatalError(resp_json['message'])
resp.raise_for_status()
return resp.json()
def add_sequence(self, name, fasta_file, index_file):
resp = retry_response_returning_functions(
requests.post,
f'{self.url}/references/sequence/set',
json={'name': name, 'fasta_file': fasta_file, 'index_file': index_file},
headers=self.headers)
if resp.status_code == 400 or resp.status_code == 500:
resp_json = resp.json()
raise FatalError(resp_json['message'])
resp.raise_for_status()
def add_liftover(self, name, chain_file, dest_reference_genome):
resp = retry_response_returning_functions(
requests.post,
f'{self.url}/references/liftover/add',
json={'name': name, 'chain_file': chain_file,
'dest_reference_genome': dest_reference_genome},
headers=self.headers)
if resp.status_code == 400 or resp.status_code == 500:
resp_json = resp.json()
raise FatalError(resp_json['message'])
resp.raise_for_status()
def parse_vcf_metadata(self, path):
resp = retry_response_returning_functions(
requests.post,
f'{self.url}/parse-vcf-metadata',
json={'path': path},
headers=self.headers)
if resp.status_code == 400 or resp.status_code == 500:
resp_json = resp.json()
raise FatalError(resp_json['message'])
resp.raise_for_status()
return resp.json()
def remove_liftover(self, name, dest_reference_genome):
resp = retry_response_returning_functions(
requests.delete,
f'{self.url}/references/liftover/remove',
json={
'name': name,
'dest_reference_genome': dest_reference_genome
},
headers=self.headers)
if resp.status_code == 400:
resp_json = resp.json()
raise FatalError(resp_json['message'])
resp.raise_for_status()
def __init__(self,
*,
region: str = 'us',
cloud: str = 'gcp',
url: Optional[str] = None,
config: Optional[dict] = None):
if region not in DB._valid_regions:
raise ValueError(f'Specify valid region parameter,'
f' received: region={repr(region)}.\n'
f'Valid regions are {DB._valid_regions}.')
if cloud not in DB._valid_clouds:
raise ValueError(f'Specify valid cloud parameter,'
f' received: cloud={repr(cloud)}.\n'
f'Valid cloud platforms are {DB._valid_clouds}.')
if (region, cloud) not in DB._valid_combinations:
raise ValueError(f'The {repr(region)} region is not available for'
f' the {repr(cloud)} cloud platform. '
f'Valid region, cloud combinations are'
f' {DB._valid_combinations}.')
if config is not None and url is not None:
raise ValueError(
f'Only specify one of the parameters url and'
f' config, received: url={url} and config={config}')
if config is None:
if url is None:
config_path = pkg_resources.resource_filename(
__name__, 'datasets.json')
assert os.path.exists(config_path), \
f'{config_path} does not exist'
with open(config_path) as f:
config = json.load(f)
else:
session = external_requests_client_session()
response = retry_response_returning_functions(session.get, url)
config = response.json()
assert isinstance(config, dict)
else:
if not isinstance(config, dict):
raise ValueError(f'expected a dict mapping dataset names to '
f'configurations, but found {config}')
config = {k: v for k, v in config.items() if 'annotation_db' in v}
self.region = region
self.cloud = cloud
self.url = url
self.config = config
self.__by_name = {
k: Dataset.from_name_and_json(k, v, region, cloud)
for k, v in config.items()
if Dataset.from_name_and_json(k, v, region, cloud) is not None
}
def execute(self, ir, timed=False):
code = self._render(ir)
resp = retry_response_returning_functions(requests.post,
f'{self.url}/execute',
json=code,
headers=self.headers)
if resp.status_code == 400 or resp.status_code == 500:
raise FatalError(resp.text)
resp.raise_for_status()
resp_json = resp.json()
typ = dtype(resp_json['type'])
value = typ._convert_from_json_na(resp_json['value'])
# FIXME put back timings
return (value, None) if timed else value
def import_fam(self, path: str, quant_pheno: bool, delimiter: str, missing: str):
resp = retry_response_returning_functions(
requests.post,
f'{self.url}/import-fam',
json={
'path': path,
'quant_pheno': quant_pheno,
'delimiter': delimiter,
'missing': missing
},
headers=self.headers)
if resp.status_code == 400 or resp.status_code == 500:
resp_json = resp.json()
raise FatalError(resp_json['message'])
resp.raise_for_status()
return resp.json()
def from_fasta_file(self, name, fasta_file, index_file, x_contigs, y_contigs, mt_contigs, par):
resp = retry_response_returning_functions(
requests.post,
f'{self.url}/references/create/fasta',
json={
'name': name,
'fasta_file': fasta_file,
'index_file': index_file,
'x_contigs': x_contigs,
'y_contigs': y_contigs,
'mt_contigs': mt_contigs,
'par': par
}, headers=self.headers)
if resp.status_code == 400 or resp.status_code == 500:
resp_json = resp.json()
raise FatalError(resp_json['message'])
resp.raise_for_status()
def index_bgen(self, files, index_file_map, rg, contig_recoding, skip_invalid_loci):
resp = retry_response_returning_functions(
requests.post,
f'{self.url}/index-bgen',
json={
'files': files,
'index_file_map': index_file_map,
'rg': rg,
'contig_recoding': contig_recoding,
'skip_invalid_loci': skip_invalid_loci
},
headers=self.headers)
if resp.status_code == 400 or resp.status_code == 500:
resp_json = resp.json()
raise FatalError(resp_json['message'])
resp.raise_for_status()
return resp.json()
def test_authorized_users_only(self):
endpoints = [
(requests.get, '/api/v1alpha/batches/0/jobs/0', 401),
(requests.get, '/api/v1alpha/batches/0/jobs/0/log', 401),
(requests.get, '/api/v1alpha/batches', 401),
(requests.post, '/api/v1alpha/batches/create', 401),
(requests.post, '/api/v1alpha/batches/0/jobs/create', 401),
(requests.get, '/api/v1alpha/batches/0', 401),
(requests.delete, '/api/v1alpha/batches/0', 401),
(requests.patch, '/api/v1alpha/batches/0/close', 401),
# redirect to auth/login
(requests.get, '/batches', 302),
(requests.get, '/batches/0', 302),
(requests.post, '/batches/0/cancel', 401),
(requests.get, '/batches/0/jobs/0', 302)]
for method, url, expected in endpoints:
full_url = deploy_config.url('batch', url)
r = retry_response_returning_functions(
method, full_url, allow_redirects=False)
assert r.status_code == expected, (full_url, r, expected)
def test_batch_create_validation():
bad_configs = [
# unexpected field fleep
{
'billing_project': 'foo',
'n_jobs': 5,
'token': 'baz',
'fleep': 'quam'
},
# billing project None/missing
{
'billing_project': None,
'n_jobs': 5,
'token': 'baz'
},
{
'n_jobs': 5,
'token': 'baz'
},
# n_jobs None/missing
{
'billing_project': 'foo',
'n_jobs': None,
'token': 'baz'
},
{
'billing_project': 'foo',
'token': 'baz'
},
# n_jobs wrong type
{
'billing_project': 'foo',
'n_jobs': '5',
'token': 'baz'
},
# token None/missing
{
'billing_project': 'foo',
'n_jobs': 5,
'token': None
},
{
'billing_project': 'foo',
'n_jobs': 5
},
# empty gcsfuse bucket name
{
'billing_project':
'foo',
'n_jobs':
5,
'token':
'baz',
'gcsfuse': [{
'bucket': '',
'mount_path': '/bucket',
'read_only': False
}],
},
# empty gcsfuse mount_path name
{
'billing_project': 'foo',
'n_jobs': 5,
'token': 'baz',
'gcsfuse': [{
'bucket': 'foo',
'mount_path': '',
'read_only': False
}],
},
# attribute key/value None
{
'attributes': {
'k': None
},
'billing_project': 'foo',
'n_jobs': 5,
'token': 'baz'
},
]
url = deploy_config.url('batch', '/api/v1alpha/batches/create')
headers = service_auth_headers(deploy_config, 'batch')
session = external_requests_client_session()
for config in bad_configs:
r = retry_response_returning_functions(session.post,
url,
json=config,
allow_redirects=True,
headers=headers)
assert r.status_code == 400, (config, r)
