def get_table_item(self,
table: DynamoTable,
key: typing.Union[str, int] = ""):
"""Retrieves dynamobdb item corresponding with primary key in the specified table.
Input:
table: (DynamoTable) enum
key: (str, int) primary key in table
Output:
item: dynamodb item
"""
dynamo_table = self._get_dynamo_table_resource_from_enum(table)
try:
table_key = {
self._get_dynamo_table_primary_key_from_enum(table): key
}
item = dynamo_table.get_item(Key=table_key,
ConsistentRead=True)['Item']
except KeyError:
raise MatrixException(status=requests.codes.not_found,
title=f"Unable to find table item with key "
f"{key} from DynamoDb Table {table.value}.")
return item
def test_run__with_one_message_in_queue_and_fails(self, mock_load_obj,
mock_transaction,
mock_complete_subtask,
mock_request_format,
mock_log_error):
request_id = str(uuid.uuid4())
payload = {
'request_id': request_id,
's3_obj_key': "test_s3_obj_key",
'type': "test_type"
}
self.sqs_handler.add_message_to_queue("test_query_job_q_name", payload)
mock_complete_subtask.side_effect = MatrixException(
status=requests.codes.not_found, title=f"Unable to find")
self.query_runner.run(max_loops=1)
mock_log_error.assert_called_once()
query_queue_messages = self.sqs_handler.receive_messages_from_queue(
"test_query_job_q_name", 1)
self.assertEqual(query_queue_messages, None)
deadletter_queue_messages = self.sqs_handler.receive_messages_from_queue(
"test_deadletter_query_job_q_name", 1)
self.assertEqual(len(deadletter_queue_messages), 1)
message_body = json.loads(deadletter_queue_messages[0]['Body'])
self.assertEqual(message_body['request_id'], request_id)
self.assertEqual(message_body['s3_obj_key'], "test_s3_obj_key")
def test_get_matrix_not_found(self, mock_get_table_item):
request_id = str(uuid.uuid4())
mock_get_table_item.side_effect = MatrixException(status=requests.codes.not_found, title=f"Unable to find")
response = get_matrix(request_id)
self.assertEqual(response[1], 404)
self.assertTrue(request_id in response[0]['message'])
def test_get_matrix_processing__driver_not_initialized(self, mock_is_request_complete, mock_get_table_item,
mock_initialized):
request_id = str(uuid.uuid4())
mock_initialized.return_value = True
mock_get_table_item.side_effect = MatrixException(status=requests.codes.not_found, title=f"Unable to find")
response = get_matrix(request_id)
self.assertEqual(response[1], requests.codes.ok)
self.assertEqual(response[0]['status'], MatrixRequestStatus.IN_PROGRESS.value)
def add_message_to_queue(self, queue_url: str, payload: dict):
response = self.sqs.meta.client.send_message(
QueueUrl=queue_url, MessageBody=json.dumps(payload))
status = response['ResponseMetadata']['HTTPStatusCode']
if status != 200:
raise MatrixException(
status=500,
title="Internal error",
detail=f"Adding message for {payload} "
f"was unsuccessful to SQS {queue_url} with status {status})")
def delete_message_from_queue(self, queue_url: str, receipt_handle: str):
response = self.sqs.meta.client.delete_message(
QueueUrl=queue_url, ReceiptHandle=receipt_handle)
status = response['ResponseMetadata']['HTTPStatusCode']
if status != 200:
raise MatrixException(
status=500,
title="Internal error",
detail=
f"Deleting message with receipt handle {receipt_handle} from {queue_url} "
f"was unsuccessful with status {status})")
def _parse_from_metadatas(self, bundle_dir, bundle_manifest_path):
protocol_id = json.load(
open(os.path.join(bundle_dir, "analysis_protocol_0.json")))['protocol_core']['protocol_id']
if protocol_id.startswith("smartseq2"):
cell_lines, expression_lines = self._parse_ss2_bundle(bundle_dir, bundle_manifest_path)
elif protocol_id.startswith("optimus"):
cell_lines, expression_lines = self._parse_optimus_bundle(bundle_dir, bundle_manifest_path)
elif protocol_id.startswith("cellranger"):
cell_lines, expression_lines = self._parse_cellranger_bundle(bundle_dir, bundle_manifest_path)
else:
raise MatrixException(400, f"Failed to parse cell and expression metadata. "
f"Unsupported analysis protocol {protocol_id}.")
return (TableName.CELL, cell_lines, bundle_dir), (TableName.EXPRESSION, expression_lines, bundle_dir)
def receive_messages_from_queue(self,
queue_url: str,
wait_time=15,
num_messages=1):
response = self.sqs.meta.client.receive_message(
QueueUrl=queue_url,
MaxNumberOfMessages=num_messages,
WaitTimeSeconds=wait_time)
status = response['ResponseMetadata']['HTTPStatusCode']
if status != 200:
raise MatrixException(
status=500,
title="Internal error",
detail=f"Retrieving message from {queue_url} "
f"was unsuccessful with status {status})")
return response.get('Messages')
