def monotone(parameter_name: str, filename: str, source_marker: str) -> bool:
data = parsed_data(source_marker, filename)
uts = data.get(parameter_name, transposed=True)[0]
for idx in range(1, len(uts)):
if uts[idx] <= uts[idx - 1]:
return False
return True
def unintersected_files(source_marker: str) -> FileList:
"""
List files which does not intersect in UT.
:param source_marker: identificator of a data source.
:return list of files not intersecting in UT.
"""
deduplicated_files_list = deduplicated_files(source_marker)
path, parser_class, selector, features_extractor = resolve_data_source(source_marker)
unitersected: List[Any] = []
previous_key = None
for filename in sorted(deduplicated_files_list):
data = parsed_data(source_marker, filename)
features = features_extractor(basename(filename))
key = '{year}.{day}'.format(**features)
ut = data.get('ut', transposed=True)[0]
if key != previous_key or len(unitersected) == 0:
previous_key = key
unitersected.append((filename, ut[0], ut[-1]))
continue
if ut[0] <= unitersected[-1][1] <= ut[-1] or ut[0] <= unitersected[-1][2] <= ut[-1]:
unitersected.pop()
else:
unitersected.append((filename, ut[0], ut[-1]))
return [filename for filename, start, end in unitersected]
def test_make_continuity_filter_empty() -> None:
path, _, _, _ = resolve_data_source(DE2_SOURCE_NACS)
data = parsed_data(DE2_SOURCE_NACS,
join(path, '1981295T001140_0_DE2_NACS_1S_V01.ASC'))
is_continuous = make_continuity_filter(data, tuple())
assert is_continuous(3 - 3) is True
assert is_continuous(4 - 3) is True
assert is_continuous(5 - 3) is True
assert is_continuous(6 - 3) is True
assert is_continuous(7 - 3) is True
assert is_continuous(8 - 3) is True
def test_make_continuity_filter_o_dens_he_dens() -> None:
path, _, _, _ = resolve_data_source(DE2_SOURCE_NACS)
data = parsed_data(DE2_SOURCE_NACS,
join(path, '1981295T001140_0_DE2_NACS_1S_V01.ASC'))
is_continuous = make_continuity_filter(data, ('o_dens', 'he_dens'))
assert is_continuous(1698 - 3) is False
assert is_continuous(1699 - 3) is True
assert is_continuous(1700 - 3) is False
assert is_continuous(1701 - 3) is False
assert is_continuous(1702 - 3) is True
assert is_continuous(1703 - 3) is False
assert is_continuous(1704 - 3) is False
def sampled_day(source_marker: str, year: int, day: int, sampling: int, *continuity_params: str) -> List[FileParserWindow]:
"""
Computes continuous data chunks inside of one day.
:param source_marker: identificator of a data source.
:param year: integer value representing year.
:param day: integer value representing day.
:param sampling: a sampling in seconds which is used to count continuity in UT.
:param continuity_params: list of params continuity of which must be preserved.
:return list of sampled data chunks.
"""
if continuity_params is None:
continuity_params = []
files_list = files_of_the_day(source_marker, year, day)
sampled_chunks: List[FileParserWindow] = []
for file in sorted(files_list):
logger.error(file)
data = parsed_data(source_marker, file)
sampled_chunks = [*sampled_chunks, *sample(data, sampling, continuity_params)]
return sampled_chunks