def test_should_download_zipped_csv(self):
os = OSFS("./tests/test_integration/resources/")
file_name = "test_csv_zipped"
test_zip_file = 'http://localhost:8001/local_data/base_train.zip'
test_ds_zip = DataSet(os, file_name, "test_id", test_zip_file,
"test dataset", "zip")
test_ds_zip.download()
test_ds_zip.unzip_file()
df = pd.read_csv(test_ds_zip.uri)
self.assertEqual((2, 2), df.shape)
os.remove(file_name + "/train.csv")
os.removedir(file_name)
## only download
os = OSFS("./tests/test_integration/resources/")
file_name = "train.csv"
test_file = 'http://localhost:8001/local_data/train.csv'
test_ds = DataSet(os, file_name, "test_id", test_file, "test dataset")
test_ds.download()
test_ds.unzip_file()
df = pd.read_csv(test_ds.uri)
self.assertEqual((2, 2), df.shape)
os.remove(file_name)
def test_unzip_local_data(self):
os = OSFS(".")
os_remove = os.remove
os.remove = mock.Mock(return_value=None)
os.copy("./tests/resources/local_data/base_train.zip",
"./tests/resources/local_data/train.zip")
test_local = DataSet(os, "/local/path", "train",
"./tests/resources/local_data/train.zip",
"test dataset", "zip")
test_local.unzip_file()
result = os.exists("./tests/resources/local_data/train/train.csv")
os.remove = os_remove
os.remove("./tests/resources/local_data/train/train.csv")
os.remove("./tests/resources/local_data/train.zip")
os.removedir("./tests/resources/local_data/train")
self.assertTrue(result)
def test_prepare_dataset(self):
os = mock.Mock()
test_ds = DataSet(os, "/local/path/test_id2", "test_id2",
"http://source/to/file", "test dataset")
test_ds.download = mock.Mock()
test_ds.unzip_file = mock.Mock()
test_ds.prepare()
test_ds.download.assert_called_once_with()
test_ds.unzip_file.assert_called_once_with()