def test_create_direct_ingest_process_job_task_secondary(
self, mock_client: mock.MagicMock,
mock_uuid: mock.MagicMock) -> None:
# Arrange
file_path = to_normalized_unprocessed_file_path(
"bucket/ingest_view_name.csv",
file_type=GcsfsDirectIngestFileType.INGEST_VIEW,
)
ingest_args = GcsfsIngestArgs(
datetime.datetime(year=2019, month=7, day=20),
file_path=GcsfsFilePath.from_absolute_path(file_path),
)
body = {
"cloud_task_args": ingest_args.to_serializable(),
"args_type": "GcsfsIngestArgs",
}
body_encoded = json.dumps(body).encode()
uuid = "random-uuid"
mock_uuid.uuid4.return_value = uuid
date = "2019-07-20"
queue_path = "us-xx-process-queue-path"
queue_name = "direct-ingest-state-us-xx-process-job-queue"
task_name = "{}/{}-{}-{}".format(
DIRECT_INGEST_STATE_PROCESS_JOB_QUEUE_V2, _REGION.region_code,
date, uuid)
url_params = {"region": _REGION.region_code, "file_path": file_path}
task = tasks_v2.types.task_pb2.Task(
name=task_name,
app_engine_http_request={
"http_method": "POST",
"relative_uri": f"/direct/process_job?{urlencode(url_params)}",
"body": body_encoded,
},
)
mock_client.return_value.task_path.return_value = task_name
mock_client.return_value.queue_path.return_value = queue_path
# Act
DirectIngestCloudTaskManagerImpl(
).create_direct_ingest_process_job_task(_REGION,
DirectIngestInstance.SECONDARY,
ingest_args)
# Assert
mock_client.return_value.queue_path.assert_called_with(
self.mock_project_id,
QUEUES_REGION,
queue_name,
)
mock_client.return_value.create_task.assert_called_with(
parent=queue_path, task=task)
def test_create_direct_ingest_process_job_task_gcsfs_args(
self, mock_client: MagicMock, mock_uuid: MagicMock,
mock_datetime: MagicMock) -> None:
# Arrange
file_path = to_normalized_unprocessed_file_path(
"bucket/file_path.csv", GcsfsDirectIngestFileType.INGEST_VIEW)
ingest_args = GcsfsIngestArgs(
ingest_time=datetime.datetime(year=2019, month=7, day=20),
file_path=GcsfsFilePath.from_absolute_path(file_path),
)
body = {
"cloud_task_args": ingest_args.to_serializable(),
"args_type": "GcsfsIngestArgs",
}
body_encoded = json.dumps(body).encode()
uuid = "random-uuid"
mock_uuid.uuid4.return_value = uuid
date = "2019-07-20"
mock_datetime.date.today.return_value = date
queue_path = f"{_REGION.shared_queue}-path"
task_name = _REGION.get_queue_name() + "/{}-{}-{}".format(
_REGION.region_code, date, uuid)
task = tasks_v2.types.task_pb2.Task(
name=task_name,
app_engine_http_request={
"http_method": "POST",
"relative_uri":
f"/direct/process_job?region={_REGION.region_code}",
"body": body_encoded,
},
)
mock_client.return_value.task_path.return_value = task_name
mock_client.return_value.queue_path.return_value = queue_path
# Act
DirectIngestCloudTaskManagerImpl(
).create_direct_ingest_process_job_task(_REGION, ingest_args)
# Assert
mock_client.return_value.queue_path.assert_called_with(
self.mock_project_id, QUEUES_REGION, _REGION.shared_queue)
mock_client.return_value.create_task.assert_called_with(
parent=queue_path, task=task)
def test_create_direct_ingest_process_job_task_gcsfs_args(
self, mock_client, mock_uuid, mock_datetime):
# Arrange
project_id = 'recidiviz-456'
file_path = to_normalized_unprocessed_file_path(
'bucket/file_path.csv', GcsfsDirectIngestFileType.INGEST_VIEW)
ingest_args = \
GcsfsIngestArgs(
ingest_time=datetime.datetime(year=2019, month=7, day=20),
file_path=GcsfsFilePath.from_absolute_path(file_path))
body = {
'cloud_task_args': ingest_args.to_serializable(),
'args_type': 'GcsfsIngestArgs'
}
body_encoded = json.dumps(body).encode()
uuid = 'random-uuid'
mock_uuid.uuid4.return_value = uuid
date = '2019-07-20'
mock_datetime.date.today.return_value = date
queue_path = _REGION.shared_queue + '-path'
task_name = _REGION.shared_queue + '/{}-{}-{}'.format(
_REGION.region_code, date, uuid)
task = tasks_v2.types.task_pb2.Task(
name=task_name,
app_engine_http_request={
'http_method': 'POST',
'relative_uri':
f'/direct/process_job?region={_REGION.region_code}',
'body': body_encoded
})
mock_client.return_value.task_path.return_value = task_name
mock_client.return_value.queue_path.return_value = queue_path
# Act
DirectIngestCloudTaskManagerImpl(project_id=project_id).\
create_direct_ingest_process_job_task(_REGION, ingest_args)
# Assert
mock_client.return_value.queue_path.assert_called_with(
project_id, QUEUES_REGION, _REGION.shared_queue)
mock_client.return_value.create_task.assert_called_with(
parent=queue_path, task=task)
def test_create_direct_ingest_process_job_task_gcsfs_args(
self, mock_client, mock_uuid, mock_datetime):
# Arrange
ingest_args = GcsfsIngestArgs(
datetime.datetime(year=2019, month=7, day=20),
file_path=to_normalized_unprocessed_file_path('file_path.csv'))
body = {
'ingest_args': ingest_args.to_serializable(),
'args_type': 'GcsfsIngestArgs'
}
body_encoded = json.dumps(body).encode()
uuid = 'random-uuid'
mock_uuid.uuid4.return_value = uuid
date = '2019-07-20'
mock_datetime.date.today.return_value = date
queue_path = _REGION.shared_queue + '-path'
task_name = _REGION.shared_queue + '/{}-{}-{}'.format(
_REGION.region_code, date, uuid)
task = tasks.types.Task(
name=task_name,
app_engine_http_request={
'relative_uri':
f'/direct/process_job?region={_REGION.region_code}',
'body': body_encoded
})
mock_client.return_value.task_path.return_value = task_name
mock_client.return_value.queue_path.return_value = queue_path
# Act
DirectIngestCloudTaskManagerImpl().\
create_direct_ingest_process_job_task(_REGION, ingest_args)
# Assert
mock_client.return_value.queue_path.assert_called_with(
metadata.project_id(), metadata.region(), _REGION.shared_queue)
mock_client.return_value.create_task.assert_called_with(
queue_path, task)