def test_ingest_view_export(self, mock_supported, mock_region,
mock_environment):
mock_supported.return_value = ['us_xx']
region_code = 'us_xx'
mock_environment.return_value = 'staging'
mock_controller = create_autospec(GcsfsDirectIngestController)
mock_region.return_value = fake_region(region_code=region_code,
environment='staging',
ingestor=mock_controller)
export_args = GcsfsIngestViewExportArgs(
ingest_view_name='my_ingest_view',
upper_bound_datetime_prev=datetime.datetime(2020, 4, 29),
upper_bound_datetime_to_export=datetime.datetime(2020, 4, 30))
request_args = {
'region': region_code,
}
body = {
'cloud_task_args': export_args.to_serializable(),
'args_type': 'GcsfsIngestViewExportArgs',
}
body_encoded = json.dumps(body).encode()
headers = {'X-Appengine-Cron': 'test-cron'}
response = self.client.post('/ingest_view_export',
query_string=request_args,
headers=headers,
data=body_encoded)
self.assertEqual(200, response.status_code)
mock_controller.do_ingest_view_export.assert_called_with(export_args)
def test_create_direct_ingest_ingest_view_export_task(
self, mock_client: mock.MagicMock,
mock_uuid: mock.MagicMock) -> None:
# Arrange
export_args = GcsfsIngestViewExportArgs(
ingest_view_name="my_ingest_view",
output_bucket_name="my_ingest_bucket",
upper_bound_datetime_prev=datetime.datetime(2020, 4, 29),
upper_bound_datetime_to_export=datetime.datetime(2020, 4, 30),
)
body = {
"cloud_task_args": export_args.to_serializable(),
"args_type": "GcsfsIngestViewExportArgs",
}
body_encoded = json.dumps(body).encode()
uuid = "random-uuid"
mock_uuid.uuid4.return_value = uuid
date = "2019-07-20"
queue_path = f"{DIRECT_INGEST_BQ_IMPORT_EXPORT_QUEUE_V2}-path"
task_name = DIRECT_INGEST_BQ_IMPORT_EXPORT_QUEUE_V2 + "/{}-{}-{}".format(
_REGION.region_code, date, uuid)
url_params = {
"region": _REGION.region_code,
"output_bucket": "my_ingest_bucket",
}
task = tasks_v2.types.task_pb2.Task(
name=task_name,
app_engine_http_request={
"http_method": "POST",
"relative_uri":
f"/direct/ingest_view_export?{urlencode(url_params)}",
"body": body_encoded,
},
)
mock_client.return_value.task_path.return_value = task_name
mock_client.return_value.queue_path.return_value = queue_path
# Act
DirectIngestCloudTaskManagerImpl(
).create_direct_ingest_ingest_view_export_task(
_REGION, DirectIngestInstance.PRIMARY, export_args)
# Assert
mock_client.return_value.queue_path.assert_called_with(
self.mock_project_id, QUEUES_REGION,
DIRECT_INGEST_BQ_IMPORT_EXPORT_QUEUE_V2)
mock_client.return_value.create_task.assert_called_with(
parent=queue_path, task=task)
def test_create_direct_ingest_ingest_view_export_task(
self, mock_client, mock_uuid):
# Arrange
project_id = 'recidiviz-456'
export_args = GcsfsIngestViewExportArgs(
ingest_view_name='my_ingest_view',
upper_bound_datetime_prev=datetime.datetime(2020, 4, 29),
upper_bound_datetime_to_export=datetime.datetime(2020, 4, 30))
body = {
'cloud_task_args': export_args.to_serializable(),
'args_type': 'GcsfsIngestViewExportArgs'
}
body_encoded = json.dumps(body).encode()
uuid = 'random-uuid'
mock_uuid.uuid4.return_value = uuid
date = '2019-07-20'
queue_path = _REGION.shared_queue + '-path'
task_name = DIRECT_INGEST_BQ_IMPORT_EXPORT_QUEUE_V2 + '/{}-{}-{}'.format(
_REGION.region_code, date, uuid)
task = tasks_v2.types.task_pb2.Task(
name=task_name,
app_engine_http_request={
'http_method': 'POST',
'relative_uri':
f'/direct/ingest_view_export?region={_REGION.region_code}',
'body': body_encoded
})
mock_client.return_value.task_path.return_value = task_name
mock_client.return_value.queue_path.return_value = queue_path
# Act
DirectIngestCloudTaskManagerImpl(
project_id=project_id
).create_direct_ingest_ingest_view_export_task(_REGION, export_args)
# Assert
mock_client.return_value.queue_path.assert_called_with(
project_id, QUEUES_REGION, DIRECT_INGEST_BQ_IMPORT_EXPORT_QUEUE_V2)
mock_client.return_value.create_task.assert_called_with(
parent=queue_path, task=task)
def test_ingest_view_export(
self,
mock_supported: mock.MagicMock,
mock_region: mock.MagicMock,
mock_environment: mock.MagicMock,
) -> None:
mock_supported.return_value = ["us_xx"]
region_code = "us_xx"
mock_environment.return_value = "staging"
mock_controller = create_autospec(GcsfsDirectIngestController)
mock_region.return_value = fake_region(region_code=region_code,
environment="staging",
ingestor=mock_controller)
export_args = GcsfsIngestViewExportArgs(
ingest_view_name="my_ingest_view",
upper_bound_datetime_prev=datetime.datetime(2020, 4, 29),
upper_bound_datetime_to_export=datetime.datetime(2020, 4, 30),
)
request_args = {
"region": region_code,
}
body = {
"cloud_task_args": export_args.to_serializable(),
"args_type": "GcsfsIngestViewExportArgs",
}
body_encoded = json.dumps(body).encode()
headers = {"X-Appengine-Cron": "test-cron"}
response = self.client.post(
"/ingest_view_export",
query_string=request_args,
headers=headers,
data=body_encoded,
)
self.assertEqual(200, response.status_code)
mock_controller.do_ingest_view_export.assert_called_with(export_args)
