def test_clean_file_str_input(_bad_file_str: str, _clean_file: str):
copyfile(_bad_file_str, _clean_file)
with pytest.raises(TfsFormatError):
read_tfs(_bad_file_str)
remove_header_comments_from_files([_clean_file])
df = read_tfs(_clean_file)
assert df.isna().any().any()
remove_nan_from_files([_clean_file])
df = read_tfs(_clean_file + ".dropna")
assert len(df) > 0
assert not df.isna().any().any()
def test_filled_write(_output_dir: str, _filled_tfs: MyTfs):
df = _filled_tfs(plane="X", directory=_output_dir)
df.write()
assert pathlib.Path(df.get_filename()).is_file()
df_read = read_tfs(df.get_filename(), index="NAME")
assert_frame_equal(df, df_read)
assert_dict_equal(df.headers, df_read.headers, compare_keys=True)
def test_empty_write(_output_dir: str):
df = MyTfs(plane="X", directory=_output_dir)
df.write()
assert pathlib.Path(df.get_filename()).is_file()
df_read = read_tfs(df.get_filename(), index="NAME")
assert_frame_equal(df, df_read,
check_exact=False) # float precision can be an issue
assert_dict_equal(df.headers, df_read.headers, compare_keys=True)
def read_tfs(self, filename: str) -> TfsDataFrame:
"""Actually reads the TFS file from self.directory with filename.
This function can be ovewriten to use something instead of tfs
to load the files.
Arguments:
filename: The name of the file to load.
Returns:
A tfs instance of the requested file.
"""
tfs_data = read_tfs(os.path.join(self.directory, filename))
if "NAME" in tfs_data:
tfs_data = tfs_data.set_index("NAME", drop=False)
return tfs_data
def remove_nan_from_files(list_of_files: list, replace: bool = False):
""" Remove NAN-Entries from files in list_of_files.
If replace=False a new file with .dropna in it's name is created, otherwise the file is
overwritten.
"""
for filepath in list_of_files:
try:
df = read_tfs(filepath)
LOG.info(f"Read file {filepath:s}")
except (IOError, TfsFormatError):
LOG.info(f"Skipped file {filepath:s}")
else:
df = df.dropna(axis='index')
if not replace:
filepath += ".dropna"
write_tfs(filepath, df)
def remove_nan_from_files(list_of_files: list, replace: bool = None) -> None:
""" Remove NAN-Entries from files in list_of_files.
If replace=False a new file with .dropna appended to its name is created, otherwise the file is
overwritten.
"""
replace = False if replace is None else replace
for filepath in list_of_files:
try:
tfs_data_frame = read_tfs(filepath)
LOG.info(f"Read file {filepath:s}")
except (IOError, TfsFormatError):
LOG.info(f"Skipped file {filepath:s}")
else:
tfs_data_frame = tfs_data_frame.dropna(axis="index")
if not replace:
filepath += ".dropna"
write_tfs(filepath, tfs_data_frame)
def read(self) -> 'FixedTfs':
return type(self)(self._plane, self._directory,
read_tfs(self._filename, index=self.index.name))
def test_filled_write(_output_dir, _filled_tfs):
df = _filled_tfs(plane="X", directory=_output_dir)
df.write()
assert os.path.isfile(df.get_filename())
df_read = read_tfs(df.get_filename(), index="NAME")
compare_dataframes(df, df_read)