def test_fetch_resource(self):
# given:
ingest_client = Mock(name='ingest_client')
uuid = '301636f7-f97b-4379-bf77-c5dcd9f17bcb'
raw_metadata = {'type': 'Biomaterial',
'uuid': {'uuid': uuid},
'content': {'describedBy': "http://some-schema/1.2.3",
'some': {'content': ['we', 'are', 'agnostic', 'of']}},
'dcpVersion': '2019-12-02T13:40:50.520Z',
'submissionDate': 'a submission date',
'updateDate': 'an update date'
}
ingest_client.get_entity_by_callback_link = Mock(return_value=raw_metadata)
# and:
metadata_service = MetadataService(ingest_client)
# when:
metadata_resource = metadata_service.fetch_resource(
'hca.domain.com/api/cellsuspensions/301636f7-f97b-4379-bf77-c5dcd9f17bcb')
# then:
self.assertEqual('biomaterial', metadata_resource.metadata_type)
self.assertEqual(uuid, metadata_resource.uuid)
self.assertEqual(raw_metadata['content'], metadata_resource.metadata_json)
self.assertEqual(utils.to_dcp_version(raw_metadata['dcpVersion']), metadata_resource.dcp_version)
self.assertEqual(raw_metadata['submissionDate'], metadata_resource.provenance.submission_date)
self.assertEqual(raw_metadata['updateDate'], metadata_resource.provenance.update_date)
def test_generate_complete_experiment_graph(self):
# given
ingest_client = self.mock_ingest
crawler = GraphCrawler(MetadataService(ingest_client))
test_assay_process = MetadataResource.from_dict(
self.mock_files.get_entity('processes', 'mock-assay-process'))
test_project = MetadataResource.from_dict(
self.mock_files.get_entity('projects', 'mock-project'))
# when
experiment_graph = crawler.generate_complete_experiment_graph(
test_assay_process, test_project)
# then
expected_links = self.mock_files.get_links_json()
nodes = self._get_nodes(expected_links)
self.assertEqual(
set([node.uuid for node in experiment_graph.nodes.get_nodes()]),
nodes)
self.assertEqual(len(experiment_graph.links.get_links()),
len(expected_links.get('links', [])))
self.assertEqual(experiment_graph.links.to_dict(), expected_links)
def setup_manifest_receiver() -> Thread:
ingest_client = IngestApi()
with Connection(DEFAULT_RABBIT_URL) as conn:
bundle_exchange = Exchange(EXCHANGE, type=EXCHANGE_TYPE)
bundle_queues = [
Queue(ASSAY_QUEUE_MANIFEST,
bundle_exchange,
routing_key=ASSAY_ROUTING_KEY)
]
conf = {
'exchange': EXCHANGE,
'routing_key': ASSAY_COMPLETED_ROUTING_KEY,
'retry': True,
'retry_policy': RETRY_POLICY
}
manifest_generator = ManifestGenerator(
ingest_client, GraphCrawler(MetadataService(ingest_client)))
exporter = ManifestExporter(ingest_api=ingest_client,
manifest_generator=manifest_generator)
manifest_receiver = ManifestReceiver(conn,
bundle_queues,
exporter=exporter,
publish_config=conf)
manifest_process = Thread(target=manifest_receiver.run)
manifest_process.start()
return manifest_process
def test_generate_manifest(self):
# given:
generator = ManifestGenerator(ingest_client=self.ingest,
graph_crawler=GraphCrawler(
MetadataService(self.ingest)))
example_manifest = self.files.get_entity('bundleManifests',
'example-assay-manifest')
# when:
actual_manifest = generator.generate_manifest(
process_uuid='mock-assay-process',
submission_uuid='mock-submission')
# then:
self.assertEqual(example_manifest, actual_manifest.__dict__)
def test_generate_experiment_process_graph(self):
# given
ingest_client = self.mock_ingest
crawler = GraphCrawler(MetadataService(ingest_client))
test_assay_process = MetadataResource.from_dict(
self.mock_files.get_entity('processes', 'mock-assay-process'))
# when
experiment_graph = crawler.generate_experiment_graph(
test_assay_process)
# then
self.assertEqual(len(experiment_graph.nodes.get_nodes()), 15)
self.assertEqual(len(experiment_graph.links.get_links()), 4)
def test_generate_supplementary_files_graph(self):
# given
ingest_client = self.mock_ingest
crawler = GraphCrawler(MetadataService(ingest_client))
test_project = MetadataResource.from_dict(
self.mock_files.get_entity('projects', 'mock-project'))
# when
experiment_graph = crawler.generate_supplementary_files_graph(
test_project)
# then
self.assertEqual(len(experiment_graph.nodes.get_nodes()), 3)
self.assertEqual(len(experiment_graph.links.get_links()),
1) # project, and 2 supplementary files
def setup_terra_exporter() -> Thread:
ingest_api_url = os.environ.get('INGEST_API', 'localhost:8080')
aws_access_key_id = os.environ['AWS_ACCESS_KEY_ID']
aws_access_key_secret = os.environ['AWS_ACCESS_KEY_SECRET']
gcs_svc_credentials_path = os.environ['GCP_SVC_ACCOUNT_KEY_PATH']
gcp_project = os.environ['GCP_PROJECT']
terra_bucket_name = os.environ['TERRA_BUCKET_NAME']
terra_bucket_prefix = os.environ['TERRA_BUCKET_PREFIX']
ingest_client = IngestApi(ingest_api_url)
metadata_service = MetadataService(ingest_client)
schema_service = SchemaService(ingest_client)
graph_crawler = GraphCrawler(metadata_service)
dcp_staging_client = (DcpStagingClient.Builder().with_ingest_client(
ingest_client).with_schema_service(schema_service).with_gcs_info(
gcs_svc_credentials_path, gcp_project, terra_bucket_name,
terra_bucket_prefix).with_gcs_xfer(gcs_svc_credentials_path,
gcp_project, terra_bucket_name,
terra_bucket_prefix,
aws_access_key_id,
aws_access_key_secret).build())
terra_job_service = TerraExportJobService(ingest_client)
terra_exporter = TerraExporter(ingest_client, metadata_service,
graph_crawler, dcp_staging_client,
terra_job_service)
rabbit_host = os.environ.get('RABBIT_HOST', 'localhost')
rabbit_port = int(os.environ.get('RABBIT_PORT', '5672'))
amqp_conn_config = AmqpConnConfig(rabbit_host, rabbit_port)
experiment_queue_config = QueueConfig(EXPERIMENT_QUEUE_TERRA,
EXPERIMENT_ROUTING_KEY, EXCHANGE,
EXCHANGE_TYPE, False, None)
publish_queue_config = QueueConfig(None, EXPERIMENT_COMPLETED_ROUTING_KEY,
EXCHANGE, EXCHANGE_TYPE, True,
RETRY_POLICY)
terra_listener = TerraListener(amqp_conn_config, terra_exporter,
terra_job_service, experiment_queue_config,
publish_queue_config)
terra_exporter_listener_process = Thread(
target=lambda: terra_listener.run())
terra_exporter_listener_process.start()
return terra_exporter_listener_process