def test_get_token(self):
token_client = MagicMock()
token_client.retrieve_token = MagicMock(return_value='token')
token_manager = TokenManager(token_client=token_client)
token = token_manager.get_token()
self.assertTrue(token)
self.assertEqual(token, 'token')
def test_get_token_when_expired(self):
token_client = MagicMock()
token_client.retrieve_token = MagicMock(return_value='token_1')
token_manager = TokenManager(token_client=token_client)
token_manager.get_token()
token_manager.token.is_expired = MagicMock(return_value=True)
token_client.retrieve_token = MagicMock(return_value='token_2')
new_token = token_manager.get_token()
self.assertEqual(new_token, 'token_2')
def __init__(self):
"""This class controls the authentication actions with Ingest Service, including retrieving the token,
store the token and make authenticated headers. Note:
"""
self.s2s_token_client = S2STokenClient()
gcp_credentials_file = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS')
self.s2s_token_client.setup_from_file(gcp_credentials_file)
self.token_manager = TokenManager(token_client=self.s2s_token_client)
def setUp(self):
self.deployment = os.environ.get('DEPLOYMENT_ENV', None)
if self.deployment not in DEPLOYMENTS:
raise RuntimeError(f'DEPLOYMENT_ENV environment variable must be one of {DEPLOYMENTS}')
self.ingest_client_api = IngestApi(url=f"https://api.ingest.{self.deployment}.data.humancellatlas.org")
self.s2s_token_client = S2STokenClient()
gcp_credentials_file = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS')
self.s2s_token_client.setup_from_file(gcp_credentials_file)
self.token_manager = TokenManager(self.s2s_token_client)
self.ingest_broker = IngestUIAgent(self.deployment)
self.ingest_api = IngestApiAgent(deployment=self.deployment)
class IngestAuthAgent:
def __init__(self):
"""This class controls the authentication actions with Ingest Service, including retrieving the token,
store the token and make authenticated headers. Note:
"""
self.s2s_token_client = S2STokenClient()
gcp_credentials_file = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS')
self.s2s_token_client.setup_from_file(gcp_credentials_file)
self.token_manager = TokenManager(token_client=self.s2s_token_client)
def _get_auth_token(self):
"""Generate self-issued JWT token
:return string auth_token: OAuth0 JWT token
"""
auth_token = self.token_manager.get_token()
return auth_token
def make_auth_header(self):
"""Make the authorization headers to communicate with endpoints which implement Auth0 authentication API.
:return dict headers: A header with necessary token information to talk to Auth0 authentication required endpoints.
"""
headers = {"Authorization": f"Bearer {self._get_auth_token()}"}
return headers
def configure_ingest_client(self):
gcp_credentials_file = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS')
self.s2s_token_client = S2STokenClient(
ServiceCredential.from_file(gcp_credentials_file),
INGEST_API_JWT_AUDIENCE)
self.token_manager = TokenManager(self.s2s_token_client)
self.ingest_api = IngestApi(url=INGEST_API,
token_manager=self.token_manager)
def __init__(self, connection, validation_queue, graph, test_path):
self.connection = connection
self.validation_queue = validation_queue
self._graph = graph
self._test_path = test_path
if Config["INGEST_API"] == "http://localhost:8080" or not (
Config["GOOGLE_APPLICATION_CREDENTIALS"]
and Config["INGEST_JWT_AUDIENCE"]):
self._ingest_api = IngestApi(Config['INGEST_API'])
else:
s2s_token_client = S2STokenClient(
credential=ServiceCredential.from_file(
Config['GOOGLE_APPLICATION_CREDENTIALS']),
audience=Config['INGEST_JWT_AUDIENCE'])
token_manager = TokenManager(s2s_token_client)
self._ingest_api = IngestApi(Config['INGEST_API'],
token_manager=token_manager)
self._logger = logging.getLogger(__name__)
class TestIngest(unittest.TestCase):
def setUp(self):
self.deployment = os.environ.get('DEPLOYMENT_ENV', None)
if self.deployment not in DEPLOYMENTS:
raise RuntimeError(f'DEPLOYMENT_ENV environment variable must be one of {DEPLOYMENTS}')
self.ingest_client_api = IngestApi(url=f"https://api.ingest.{self.deployment}.data.humancellatlas.org")
self.s2s_token_client = S2STokenClient()
gcp_credentials_file = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS')
self.s2s_token_client.setup_from_file(gcp_credentials_file)
self.token_manager = TokenManager(self.s2s_token_client)
self.ingest_broker = IngestUIAgent(self.deployment)
self.ingest_api = IngestApiAgent(deployment=self.deployment)
def ingest_and_upload_only(self, dataset_name):
dataset_fixture = DatasetFixture(dataset_name, self.deployment)
runner = DatasetRunner(self.deployment)
runner.valid_run(dataset_fixture)
return runner
def ingest(self, dataset_name):
dataset_fixture = DatasetFixture(dataset_name, self.deployment)
runner = DatasetRunner(self.deployment)
runner.complete_run(dataset_fixture)
return runner
def _create_submission_envelope(self):
token = self.token_manager.get_token()
self.ingest_client_api.set_token(f'Bearer {token}')
submission = self.ingest_client_api.create_submission()
submission_url = submission["_links"]["self"]["href"]
submission_envelope = self.ingest_api.envelope(envelope_id=None, url=submission_url)
return submission_envelope
# TODO move this to ingest client api
def _get_entities(self, url, entity_type):
r = requests.get(url, headers={'Content-type': 'application/json'})
r.raise_for_status()
response = r.json()
if response.get('_embedded') and response['_embedded'].get(entity_type):
return response['_embedded'][entity_type]
else:
return []
def ingest_analysis(self, dataset_name):
analysis_fixture = AnalysisSubmissionFixture()
runner = AnalysisSubmissionRunner(self.deployment, self.ingest_broker, self.ingest_api, self.token_manager,
self.ingest_client_api)
dataset_fixture = DatasetFixture(dataset_name, self.deployment)
runner.run(dataset_fixture, analysis_fixture)
self.assertTrue(runner.bundle_manifest_uuid,
'The analysis process should be attached to an input bundle manifest')
derived_files_url = runner.analysis_process['_links']['derivedFiles'][
'href']
derived_files = self._get_entities(derived_files_url, 'files')
analysis_files = runner.analysis_submission.get_files()
derived_file_uuids = [file['uuid']['uuid'] for file in derived_files]
analysis_file_uuids = [file['uuid']['uuid'] for file in analysis_files]
self.assertTrue(derived_file_uuids, 'There must be files in the analysis submission')
self.assertEqual(derived_file_uuids, analysis_file_uuids,
'The analyses files must be linked to the analyses process.')
input_files_url = runner.analysis_process['_links']['inputFiles'][
'href']
input_files = self._get_entities(input_files_url, 'files')
primary_submission_files = runner.primary_submission.get_files()
input_file_uuids = [file['uuid']['uuid'] for file in input_files]
primary_submission_file_uuids = [file['uuid']['uuid'] for file in primary_submission_files]
self.assertTrue(input_file_uuids, 'There must be files from the primary submission')
self.assertEqual(input_file_uuids, primary_submission_file_uuids,
'The primary submission files must be linked to the analyses process.')
input_bundle_manifest_url = \
runner.analysis_process['_links']['inputBundleManifests']['href']
attached_bundle_manifests = self._get_entities(
input_bundle_manifest_url, 'bundleManifests')
self.assertEqual(len(attached_bundle_manifests), 1,
'There should only be one input bundle manifest for the analyses process')
self.assertEqual(attached_bundle_manifests[0]['bundleUuid'],
runner.bundle_manifest_uuid,
'The input bundle manifest for the analyses process is incorrect')
return runner
def ingest_big_submission(self):
metadata_fixture = MetadataFixture()
runner = BigSubmissionRunner(self.deployment, self.ingest_client_api, self.token_manager)
runner.run(metadata_fixture)
def ingest_updates(self):
runner = UpdateSubmissionRunner(self.deployment, self.ingest_broker, self.ingest_api, self.ingest_client_api)
runner.run()
self.assertEqual(len(runner.updated_bundle_fqids), 1, "There should be 1 bundle updated.")
#!/usr/bin/env python
import sys
from ingest.utils.s2s_token_client import S2STokenClient
from ingest.utils.token_manager import TokenManager
if __name__ == '__main__':
key_file = sys.argv[1]
client = S2STokenClient()
client.setup_from_file(key_file)
token_manager = TokenManager(client)
token = token_manager.get_token()
print(f'Bearer {token}')