def test_backward_compatibility_missing_part_spec_list(ops, missing_spec_list):
as_json = to_json_without_spec_list(missing_spec_list)
metadata = TableMetadataParser.from_json(ops, None, as_json)
assert metadata.location == missing_spec_list.location
assert metadata.last_column_id == missing_spec_list.last_column_id
assert metadata.schema.as_struct() == missing_spec_list.schema.as_struct()
assert str(metadata.spec) == str(missing_spec_list.spec)
"""
def test_from_json_sorts_snapshot_log(ops, expected_metadata_sorting):
current_snapshot = expected_metadata_sorting.current_snapshot()
previous_snapshot_id = expected_metadata_sorting.current_snapshot(
).parent_id
previous_snapshot = expected_metadata_sorting.snapshot(
previous_snapshot_id)
expected_log = [
SnapshotLogEntry(previous_snapshot.timestamp_millis,
previous_snapshot.snapshot_id),
SnapshotLogEntry(current_snapshot.timestamp_millis,
current_snapshot.snapshot_id)
]
as_json = TableMetadataParser.to_json(expected_metadata_sorting)
metadata = TableMetadataParser.from_json(ops, None, as_json)
assert expected_log == metadata.snapshot_log
def test_compression_property(expected, prop):
config = {ConfigProperties.COMPRESS_METADATA: prop}
output_file = FileSystemOutputFile.from_path(TableMetadataParser.get_file_extension(config), dict)
TableMetadataParser.write(expected, output_file)
assert prop == is_compressed(TableMetadataParser.get_file_extension(config))
read = TableMetadataParser.read(None,
FileSystemInputFile.from_location(TableMetadataParser.get_file_extension(config), None))
verify_metadata(read, expected)
def test_json_conversion(ops, expected_metadata):
current_snapshot_id = expected_metadata.current_snapshot_id
previous_snapshot_id = expected_metadata.current_snapshot().parent_id
previous_snapshot = expected_metadata.snapshot(previous_snapshot_id)
as_json = TableMetadataParser.to_json(expected_metadata)
metadata = TableMetadataParser.from_json(ops, None, as_json)
assert metadata.location == expected_metadata.location
assert metadata.last_column_id == expected_metadata.last_column_id
assert metadata.schema.as_struct() == expected_metadata.schema.as_struct()
assert str(metadata.spec) == str(expected_metadata.spec)
assert metadata.default_spec_id == expected_metadata.default_spec_id
assert metadata.specs == expected_metadata.specs
assert metadata.properties == expected_metadata.properties
assert metadata.snapshot_log == expected_metadata.snapshot_log
assert metadata.current_snapshot().snapshot_id == current_snapshot_id
assert metadata.current_snapshot().parent_id == previous_snapshot_id
assert metadata.current_snapshot(
).manifests == expected_metadata.current_snapshot().manifests
assert metadata.snapshot(
previous_snapshot_id).snapshot_id == previous_snapshot_id
assert metadata.snapshot(
previous_snapshot_id).manifests == previous_snapshot.manifests
def prop(request):
config = {ConfigProperties.COMPRESS_METADATA: request.param}
yield request.param
if os.path.exists(TableMetadataParser.get_file_extension(config)):
os.remove(TableMetadataParser.get_file_extension(config))