def test_lookup_default(legacy_ncs_lookup: NcsLookup, mock_assets_config,
threads_count):
tags = [
SensorTag("Ásgarðr", "asset"),
SensorTag("tag1", "asset"),
SensorTag("tag2", "asset"),
SensorTag("tag4", "asset"),
SensorTag("tag5", "asset1"),
]
result = list(
legacy_ncs_lookup.lookup(
mock_assets_config,
tags,
[YearPartition(2019), YearPartition(2020)],
threads_count=threads_count,
))
assert reduce_tag_locations(result) == {
("Ásgarðr", YearPartition(2019)): (
"path/%C3%81sgar%C3%B0r/%C3%81sgar%C3%B0r_2019.csv",
CsvFileType,
),
("tag2", YearPartition(2020)): (
"path/tag2/parquet/tag2_2020.parquet",
ParquetFileType,
),
("tag5", YearPartition(2020)): (
"path1/tag5/parquet/tag5_2020.parquet",
ParquetFileType,
),
}
def test_files_lookup_asgard(legacy_ncs_lookup: NcsLookup):
tag = SensorTag("Ásgarðr", "asset")
partitions = [YearPartition(2019), YearPartition(2020)]
result = reduce_tag_locations([
legacy_ncs_lookup.files_lookup("path/%C3%81sgar%C3%B0r", tag,
partitions)
])
assert result == {
("Ásgarðr", YearPartition(2019)): (
"path/%C3%81sgar%C3%B0r/%C3%81sgar%C3%B0r_2019.csv",
CsvFileType,
),
}
def test_files_lookup_tag3(legacy_ncs_lookup: NcsLookup):
tag = SensorTag("tag3", "asset")
result = reduce_tag_locations([
legacy_ncs_lookup.files_lookup(
"path/tag3", tag,
[YearPartition(2019), YearPartition(2020)])
])
assert result == {
("tag3", YearPartition(2020)): (
"path/tag3/parquet/tag3_2020.parquet",
ParquetFileType,
),
}
def test_lookup_exceptions(legacy_ncs_lookup: NcsLookup, mock_assets_config,
threads_count):
tags = [
SensorTag("Ásgarðr", "asset"),
SensorTag("tag1", "asset"),
]
with pytest.raises(ConfigException):
list(
legacy_ncs_lookup.lookup(
mock_assets_config,
tags,
[YearPartition(2019), YearPartition(2020)],
threads_count=threads_count,
))
def test_ncs_yearly_parquet_file_type(mock_file_system):
ncs_file_type = NcsYearlyParquetFileType()
assert type(ncs_file_type.file_type) is ParquetFileType
assert ncs_file_type.partition_type is YearPartition
partition = YearPartition(2020)
paths = ncs_file_type.paths(mock_file_system, "tag1", [partition])
assert list(paths) == [(partition, "parquet/tag1_2020.parquet")]
def test_ncs_csv_file_type(mock_file_system):
ncs_file_type = NcsCsvFileType()
assert type(ncs_file_type.file_type) is CsvFileType
assert ncs_file_type.partition_type is YearPartition
partition = YearPartition(2020)
paths = list(ncs_file_type.paths(mock_file_system, "tag1", [partition]))
assert list(paths) == [(partition, "tag1_2020.csv")]
def test_tag_locations(parquet_file_type):
tag = SensorTag("tag1", "asset")
location_2020 = Location("path/2020.parquet", parquet_file_type)
locations = {
YearPartition(2020): location_2020,
YearPartition(2018): Location("path/2018.parquet", parquet_file_type),
}
tag_locations = TagLocations(tag, locations)
assert tag_locations.available()
assert tag_locations.partitions() == [
YearPartition(2018), YearPartition(2020)
]
assert tag_locations.get_location(2020) is location_2020
assert tag_locations.get_location(2019) is None
result = list(tag_locations)
assert result == [
(
SensorTag(name="tag1", asset="asset"),
YearPartition(2018),
Location(path="path/2018.parquet", file_type=parquet_file_type),
),
(
SensorTag(name="tag1", asset="asset"),
YearPartition(2020),
Location(path="path/2020.parquet", file_type=parquet_file_type),
),
]
def test_max_file_size(legacy_ncs_lookup: NcsLookup):
tag = SensorTag("tag10", "asset")
partitions = [YearPartition(2020)]
result = reduce_tag_locations(
[legacy_ncs_lookup.files_lookup("path3/tag10", tag, partitions)])
assert not result
def test_year_partition():
assert YearPartition(2020) < YearPartition(2021)
assert not YearPartition(2021) < YearPartition(2020)
def test_split_by_partitions_validation_error():
with pytest.raises(ValueError):
list(
split_by_partitions(PartitionBy.YEAR, datetime(2021, 12, 1),
datetime(2021, 11, 1)))
@pytest.mark.parametrize(
"partition_by, start_period, end_period, result",
[
[
PartitionBy.YEAR,
datetime(2020, 1, 1),
datetime(2021, 12, 1),
[
YearPartition(2020),
YearPartition(2021),
],
],
[
PartitionBy.YEAR,
datetime(2020, 1, 1),
datetime(2020, 1, 1),
[YearPartition(2020)],
],
[
PartitionBy.MONTH,
datetime(2020, 1, 1),
datetime(2020, 3, 1),
[
MonthPartition(2020, 1),