def test_merge_and_collapse_gz_output(self):
merge_and_collapse_pattern("data/*.txt", "tmp_output.gz",
"output file tmp_output.gz does not exist")
self.assertTrue(os.path.isfile("tmp_output.gz"))
with gzip.open("tmp_output.gz", "rt") as fin:
self._check_expected_values(fin)
os.remove("tmp_output.gz")
def test_merge_and_collapse_gz_input_and_output(self):
merge_and_collapse_pattern("data/*.gz", "tmp_output2.gz")
self.assertTrue(os.path.isfile("tmp_output2.gz"),
"output file tmp_output2.gz does not exist")
with gzip.open("tmp_output2.gz", "rt") as fin:
self._check_expected_values(fin)
os.remove("tmp_output2.gz")
#check that input files were not removed
self.assertTrue(os.path.isfile("data/a.gz"), "input file was removed")
self.assertTrue(os.path.isfile("data/b.gz"), "input file was removed")
self.assertTrue(os.path.isfile("data/c.gz"), "input file was removed")
def test_different_tmp_dir(self):
os.makedirs("./test/different/tempdir")
output_path = merge_and_collapse_pattern(
"data/*.txt", batch=2, tmpdir="./test/different/tempdir")
with gzip.open(output_path, "rt") as fin:
self._check_expected_values(fin)
os.remove(output_path)
shutil.rmtree("./test")
def test_input_removing(self):
# copy the input data into another directory
shutil.copytree("data", "test_data")
output_path = merge_and_collapse_pattern("test_data/*.gz",
delete_input=True)
with gzip.open(output_path, "rt") as fin:
self._check_expected_values(fin)
os.remove(output_path)
#check that input files were removed
self.assertFalse(os.path.exists("test_data/a.gz"),
"input file was not removed")
self.assertFalse(os.path.exists("test_data/b.gz"),
"input file was not removed")
self.assertFalse(os.path.exists("test_data/c.gz"),
"input file was not removed")
shutil.rmtree("test_data")
def test_merge_and_collapse_gz_input_automatic_output_file_naming(self):
output_path = merge_and_collapse_pattern("data/*.gz")
with gzip.open(output_path, "rt") as fin:
self._check_expected_values(fin)
os.remove(output_path)
def test_merge_and_collapse_mix_txt_and_gz(self):
merge_and_collapse_pattern("data/*", "tmp_output")
with gzip.open("tmp_output", "rt") as fin:
self._check_expected_values(fin, 'd')
os.remove("tmp_output")
def test_merge_and_collapse_threshold(self):
merge_and_collapse_pattern("data/*.txt", "tmp_output", threshold=16)
with gzip.open("tmp_output", "rt") as fin:
self._check_expected_values(fin, 't')
os.remove("tmp_output")