def _test_other_endpoints(self):
service_paths = (
'/',
'/openapi',
'/version',
'/index/summary',
'/index/files/order',
)
service_routes = ((config.service_endpoint(), path)
for path in service_paths)
health_endpoints = (config.service_endpoint(),
config.indexer_endpoint())
health_paths = (
'', # default keys for lambda
'/', # all keys
'/basic',
'/elasticsearch',
'/queues',
'/progress',
'/api_endpoints',
'/other_lambdas')
health_routes = ((endpoint, '/health' + path)
for endpoint in health_endpoints
for path in health_paths)
for endpoint, path in (*service_routes, *health_routes):
with self.subTest('other_endpoints', endpoint=endpoint, path=path):
self._check_endpoint(endpoint, path)
def main():
catalogs = {
'dcp2':
config.Catalog(name='dcp2',
atlas='hca',
internal=False,
plugins=dict(
metadata=config.Catalog.Plugin(name='hca'),
repository=config.Catalog.Plugin(name='tdr')),
sources=set())
}
# To create a normalized OpenAPI document, we patch any
# deployment-specific variables that affect the document.
with patch.object(target=type(config),
attribute='catalogs',
new_callable=PropertyMock,
return_value=catalogs):
assert config.catalogs == catalogs
with patch.object(target=config,
attribute='service_function_name',
return_value='azul_service'):
assert config.service_name == 'azul_service'
with patch.object(target=config,
attribute='service_endpoint',
return_value='localhost'):
assert config.service_endpoint() == 'localhost'
app_module = load_app_module('service')
app_spec = app_module.app.spec()
doc_path = os.path.join(config.project_root,
'lambdas/service/openapi.json')
with write_file_atomically(doc_path) as file:
json.dump(app_spec, file, indent=4)
def request_manifest(filters):
url = str(furl(url=config.service_endpoint(), path='fetch/manifest/files'))
params = {
'catalog': 'dcp2',
'filters': json.dumps(filters),
'format': 'terra.bdbag'
}
logger.debug(params)
resp = requests.get(url=url, params=params)
body = resp.json()
assert body['Status'] == 301, body['Status']
while True:
url = body['Location']
logger.debug(f'location is {url}')
time.sleep(0.5)
resp = requests.get(url=url)
body = resp.json()
if body['Status'] != 301:
break
assert body['Status'] == 302, body['Status']
url = body['Location']
logger.debug(f'location is {url}')
return url
def _mock_service_endpoints(self, helper: responses.RequestsMock,
endpoint_states: Mapping[str, bool]) -> None:
for endpoint, endpoint_up in endpoint_states.items():
helper.add(
responses.Response(method='HEAD',
url=config.service_endpoint() + endpoint,
status=200 if endpoint_up else 503,
json={}))
def _api_endpoint(self, path: str) -> Tuple[str, JSON]:
url = config.service_endpoint() + path
response = requests.head(url)
try:
response.raise_for_status()
except requests.exceptions.HTTPError as e:
return url, {'up': False, 'error': repr(e)}
else:
return url, {'up': True}
def filter_projects():
# Tabula Muris causes 500 errors due to memory constraints
# https://github.com/DataBiosphere/azul/issues/2442
resp = requests.get(url=config.service_endpoint() + '/index/projects',
params=dict(size=1000))
projects = resp.json()['termFacets']['project']['terms']
return [
one(project['projectId']) for project in projects
if project['term'] != 'Tabula Muris'
]
def test_openapi(self):
service = config.service_endpoint()
response = requests.get(service + '/')
self.assertEqual(response.status_code, 200)
self.assertEqual(response.headers['content-type'], 'text/html')
self.assertGreater(len(response.content), 0)
# validate OpenAPI spec
response = requests.get(service + '/openapi')
response.raise_for_status()
spec = response.json()
validate_spec(spec)
def _get_one_file_uuid(self, catalog: CatalogName) -> str:
filters = {'fileFormat': {'is': ['fastq.gz', 'fastq']}}
response = self._check_endpoint(endpoint=config.service_endpoint(),
path='/index/files',
query=dict(catalog=catalog,
filters=json.dumps(filters),
size=1,
order='asc',
sort='fileSize'))
hits = json.loads(response)
return one(one(hits['hits'])['files'])['uuid']
def write_to(self, output: IO[str]) -> Optional[str]:
output.write('--create-dirs\n\n'
'--compressed\n\n'
'--location\n\n')
for hit in self._create_request().scan():
doc = self._hit_to_doc(hit)
file = one(doc['contents']['files'])
uuid, name, version = file['uuid'], file['name'], file['version']
url = furl(config.service_endpoint(),
path=f'/repository/files/{uuid}',
args=dict(version=version, catalog=self.catalog))
output.write(f'url={self._option(url.url)}\n'
f'output={self._option(name)}\n\n')
return None
def _test_repository_files(self, catalog: str):
with self.subTest('repository_files', catalog=catalog):
file_uuid = self._get_one_file_uuid(catalog)
response = self._check_endpoint(
endpoint=config.service_endpoint(),
path=f'/fetch/repository/files/{file_uuid}',
query=dict(catalog=catalog))
response = json.loads(response)
while response['Status'] != 302:
self.assertEqual(301, response['Status'])
response = self._get_url(response['Location']).json()
content = self._get_url_content(response['Location'])
self._validate_fastq_content(content)
def _get_entities(self, catalog: CatalogName, entity_type):
entities = []
size = 100
params = dict(catalog=catalog, size=str(size))
url = furl(url=config.service_endpoint(),
path=('index', entity_type),
query_params=params).url
while True:
response = self._get_url(url)
body = response.json()
hits = body['hits']
entities.extend(hits)
url = body['pagination']['next']
if url is None:
break
return entities
def test_manifest_status_running(self, step_function_helper):
"""
A running manifest job should return a 301 status and a url to retry checking the job status
"""
execution_id = 'd4ee1bed-0bd7-4c11-9c86-372e07801536'
execution_running_output = {
'executionArn':
StepFunctionHelper().execution_arn(
config.manifest_state_machine_name, execution_id),
'stateMachineArn':
StepFunctionHelper().state_machine_arn(
config.manifest_state_machine_name),
'name':
execution_id,
'status':
'RUNNING',
'startDate':
datetime.datetime(2018, 11, 15, 18, 30, 44, 896000),
'input':
'{"filters": {}}'
}
step_function_helper.describe_execution.return_value = execution_running_output
manifest_service = AsyncManifestService()
token = manifest_service.encode_token({'execution_id': execution_id})
retry_url = config.service_endpoint() + '/manifest/files'
format_ = ManifestFormat.compact
filters = manifest_service.parse_filters('{}')
wait_time, manifest = manifest_service.start_or_inspect_manifest_generation(
self_url=retry_url,
format_=format_,
catalog=self.catalog,
filters=filters,
token=token)
self.assertEqual(type(wait_time), int)
self.assertEqual(wait_time, 1)
expected_token = manifest_service.encode_token({
'execution_id': execution_id,
'request_index': 1
})
location = furl(retry_url, args={'token': expected_token})
expected_obj = Manifest(location=location.url,
was_cached=False,
properties={})
self.assertEqual(expected_obj, manifest)
def _expected_api_endpoints(self, endpoint_states: Mapping[str,
bool]) -> JSON:
return {
'api_endpoints': {
'up':
all(up for endpoint, up in endpoint_states.items()),
**({
config.service_endpoint() + endpoint: {
'up': up
} if up else {
'up':
up,
'error': ("HTTPError('503 Server Error: "
"Service Unavailable for url: "
f"{config.service_endpoint() + endpoint}')")
}
for endpoint, up in endpoint_states.items()
})
}
}
def get_fusillade_login_url(redirect_uri: str = None) -> str:
"""
Get the login URL.
:param redirect_uri: The URI that Fusillade will redirect back to
According to the documentation, client_id is the service's domain name
(azul.config.domain_name). However, specifying client_id will cause an
Auth0 misconfiguration error. This method intentionally excludes
``client_id`` from the request to Fusillade.
"""
if redirect_uri:
if not Authenticator.valid_redirect_uri_pattern.search(redirect_uri):
raise InvalidRedirectUriError(redirect_uri)
else:
redirect_uri = config.service_endpoint() + '/auth/callback'
query = dict(response_type="code",
scope="openid email",
redirect_uri=redirect_uri,
state='')
return '?'.join([Authenticator.get_fusillade_url('authorize'),
urlencode(query)])
def _test_manifest(self, catalog: CatalogName):
for format_, validator, attempts in [
(None, self._check_manifest, 1),
('compact', self._check_manifest, 1),
('full', self._check_manifest, 3),
('terra.bdbag', self._check_terra_bdbag, 1)
]:
with self.subTest('manifest',
catalog=catalog,
format=format_,
attempts=attempts):
assert attempts > 0
params = dict(catalog=catalog)
if format_ is not None:
params['format'] = format_
for attempt in range(attempts):
start = time.time()
response = self._check_endpoint(config.service_endpoint(),
'/manifest/files', params)
log.info('Request %i/%i took %.3fs to execute.',
attempt + 1, attempts,
time.time() - start)
validator(catalog, response)
def _test_dos(self, catalog: CatalogName, file_uuid: str):
with self.subTest('dos', catalog=catalog):
log.info('Resolving file %s with DOS', file_uuid)
response = self._check_endpoint(
config.service_endpoint(),
path=drs.dos_object_url_path(file_uuid),
query=dict(catalog=catalog))
json_data = json.loads(response)['data_object']
file_url = first(json_data['urls'])['url']
while True:
response = self._get_url(file_url, allow_redirects=False)
# We handle redirects ourselves so we can log each request
if response.status_code in (301, 302):
file_url = response.headers['Location']
try:
retry_after = response.headers['Retry-After']
except KeyError:
pass
else:
time.sleep(int(retry_after))
else:
break
self._assertResponseStatus(response)
self._validate_fastq_content(response.content)