def test_socrata_smoke(domain, dataset_id, local_engine_empty):
# This relies on the Socrata API being available, but good to smoke test some popular datasets
# to make sure the mounting works end-to-end.
try:
mount_socrata(
"socrata_mount", None, None, None, None, domain=domain, tables={"data": dataset_id}
)
result = local_engine_empty.run_sql("SELECT * FROM socrata_mount.data LIMIT 10")
assert len(result) == 10
finally:
local_engine_empty.delete_schema("socrata_mount")
def test_socrata_mounting_slug(local_engine_empty):
with open(os.path.join(INGESTION_RESOURCES, "socrata/find_datasets.json"), "r") as f:
socrata_meta = json.load(f)
socrata = MagicMock(spec=Socrata)
socrata.datasets.return_value = socrata_meta
with mock.patch("sodapy.Socrata", return_value=socrata):
mount_socrata(
"test/pg_mount", None, None, None, None, "example.com", None, "some_token",
)
assert local_engine_empty.get_all_tables("test/pg_mount") == [
"current_employee_names_salaries_and_position_xzkq_xp2w"
]
def test_socrata_mounting_error():
socrata = MagicMock(spec=Socrata)
socrata.datasets.side_effect = Exception("Unknown response format: text/html; charset=utf-8")
with mock.patch("sodapy.Socrata", return_value=socrata):
with pytest.raises(RepositoryNotFoundError):
mount_socrata(
"test/pg_mount",
None,
None,
None,
None,
"example.com",
{"some_table": "xzkq-xp2w"},
"some_token",
)
def test_socrata_mounting_missing_tables():
with open(os.path.join(INGESTION_RESOURCES, "socrata/find_datasets.json"), "r") as f:
socrata_meta = json.load(f)
socrata = MagicMock(spec=Socrata)
socrata.datasets.return_value = socrata_meta
with mock.patch("sodapy.Socrata", return_value=socrata):
with pytest.raises(ValueError) as e:
mount_socrata(
"test/pg_mount",
None,
None,
None,
None,
"example.com",
{"some_table": "wrong_id"},
"some_token",
)
assert "Some Socrata tables couldn't be found! Missing tables: xzkq-xp2w" in str(e.value)
def test_socrata_mounting(local_engine_empty):
with open(os.path.join(INGESTION_RESOURCES, "socrata/find_datasets.json"), "r") as f:
socrata_meta = json.load(f)
socrata = MagicMock(spec=Socrata)
socrata.datasets.return_value = socrata_meta
with mock.patch("sodapy.Socrata", return_value=socrata):
mount_socrata(
"test/pg_mount",
None,
None,
None,
None,
"example.com",
{"some_table": "xzkq-xp2w"},
"some_token",
)
assert local_engine_empty.get_full_table_schema("test/pg_mount", "some_table") == [
TableColumn(
ordinal=1, name=":id", pg_type="text", is_pk=False, comment="Socrata column ID"
),
TableColumn(
ordinal=2,
name="full_or_part_time",
pg_type="text",
is_pk=False,
comment="Whether the employee was employed full- (F) or part-time (P).",
),
TableColumn(
ordinal=3, name="hourly_rate", pg_type="numeric", is_pk=False, comment=mock.ANY
),
TableColumn(
ordinal=4, name="salary_or_hourly", pg_type="text", is_pk=False, comment=mock.ANY
),
TableColumn(
ordinal=5,
name="job_titles",
pg_type="text",
is_pk=False,
comment="Title of employee at the time when the data was updated.",
),
TableColumn(
ordinal=6, name="typical_hours", pg_type="numeric", is_pk=False, comment=mock.ANY
),
TableColumn(
ordinal=7, name="annual_salary", pg_type="numeric", is_pk=False, comment=mock.ANY
),
TableColumn(
ordinal=8, name=_long_name_col_sg, pg_type="text", is_pk=False, comment=mock.ANY
),
TableColumn(
ordinal=9,
name="department",
pg_type="text",
is_pk=False,
comment="Department where employee worked.",
),
]
assert local_engine_empty.run_sql(
"SELECT option_value FROM information_schema.foreign_table_options "
"WHERE foreign_table_name = 'some_table' "
"AND foreign_table_schema = 'test/pg_mount' "
"AND option_name = 'column_map'"
) == [(f'{{"{_long_name_col_sg}": "{_long_name_col}"}}',)]