def test_gcp_convert_ndjson_to_json(self):
publisher = JsonPublisher(
self.mock_client,
self.mock_storage_client,
self.project_id,
str(self.incremental_sql_path),
self.api_version,
self.test_bucket,
["submission_date:DATE:2020-03-15"],
)
mock_out = MagicMock(side_effect=[['{"a": 1}', '{"b": "cc"}'], None])
mock_out.__iter__ = Mock(
return_value=iter(['{"a": 1}', '{"b": "cc"}']))
file_handler = MagicMock()
file_handler.__enter__.return_value = mock_out
smart_open.open = MagicMock(return_value=file_handler)
publisher._gcp_convert_ndjson_to_json("test_path")
smart_open.open.assert_has_calls([
call("gs://test-bucket/blob_path/000000000000.ndjson"),
call("gs://test-bucket/blob_path/000000000000.json.tmp.gz", "w"),
])
file_handler.write.assert_has_calls([
call("["),
call('{"a": 1}'),
call(","),
call('{"b": "cc"}'),
call("]")
])
def test_write_results_to_temp_table(self):
publisher = JsonPublisher(
self.mock_client,
self.mock_storage_client,
self.project_id,
str(self.incremental_sql_path),
self.api_version,
self.test_bucket,
["submission_date:DATE:2020-03-15"],
)
publisher._write_results_to_temp_table()
assert publisher.temp_table.startswith(self.temp_table)
self.mock_client.query.assert_called_once()
def test_no_date_parameter_incremental(self):
with pytest.raises(SystemExit) as e:
publisher = JsonPublisher(
self.mock_client,
self.mock_storage_client,
self.project_id,
str(self.incremental_sql_path),
self.api_version,
self.test_bucket,
None,
)
publisher.publish_json()
assert e.type == SystemExit
assert e.value.code == 1
def test_parameter_date(self):
publisher = JsonPublisher(
self.mock_client,
self.mock_storage_client,
self.project_id,
str(self.incremental_sql_path),
self.api_version,
self.test_bucket,
["submission_date:DATE:2020-11-02"],
)
assert publisher.date == "2020-11-02"
def test_invalid_query_file(self):
with pytest.raises(FileNotFoundError) as e:
JsonPublisher(
self.mock_client,
self.mock_storage_client,
self.project_id,
"invalid_path",
self.api_version,
self.test_bucket,
)
assert e.type == FileNotFoundError
def test_dataset_table_version_splitting(self):
publisher = JsonPublisher(
self.mock_client,
self.mock_storage_client,
self.project_id,
str(self.non_incremental_sql_path),
self.api_version,
self.test_bucket,
)
assert publisher.dataset == "test"
assert publisher.table == "non_incremental_query"
assert publisher.version == "v1"
def test_publish_last_updated_to_gcs(self):
publisher = JsonPublisher(
self.mock_client,
self.mock_storage_client,
self.project_id,
str(self.incremental_sql_path),
self.api_version,
self.test_bucket,
["submission_date:DATE:2020-03-15"],
)
mock_out = MagicMock(side_effect=[['{"a": 1}', '{"b": "cc"}'], None])
mock_out.__iter__ = Mock(
return_value=iter(['{"a": 1}', '{"b": "cc"}']))
file_handler = MagicMock()
file_handler.__enter__.return_value = mock_out
smart_open.open = MagicMock(return_value=file_handler)
publisher.publish_json()
assert publisher.last_updated is not None
mock_out.write.assert_called_with(
json.dumps(publisher.last_updated.strftime("%Y-%m-%d %H:%M:%S")))
