def test_list_file(self):
"""Test list files method."""
db = FileSystemFilestore(SERVER_DIR)
db.upload_file(CSV_FILE)
db.upload_file(GZIP_CSV_FILE)
db.upload_file(TSV_FILE)
db.upload_file(GZIP_TSV_FILE)
files = db.list_files()
self.assertEqual(len(files), 4)
db.upload_file(CSV_FILE)
db.upload_file(GZIP_CSV_FILE)
db.upload_file(TSV_FILE)
db.upload_file(GZIP_TSV_FILE)
files = db.list_files()
self.assertEqual(len(files), 8)
def test_download_dataset(self):
"""Test loading a dataset from Url. Note that this test depends on the
accessed web service to be running. It will fail otherwise."""
# Skip test if DOWNLOAD_URL is None
if DOWNLOAD_URL is None:
print('Skipping download test')
return
store = FileSystemDatastore(STORE_DIR)
ds = store.download_dataset(url=DOWNLOAD_URL)
dataset_dir = os.path.join(STORE_DIR, ds.identifier)
self.assertTrue(os.path.isdir(dataset_dir))
self.assertTrue(os.path.isfile(os.path.join(dataset_dir, DATA_FILE)))
self.assertTrue(os.path.isfile(os.path.join(dataset_dir, DESCRIPTOR_FILE)))
self.assertFalse(os.path.isfile(os.path.join(dataset_dir, METADATA_FILE)))
self.validate_class_size_dataset(ds)
# Download file into a given filestore
fs = FileSystemFilestore(FSSTORE_DIR)
ds, fh = store.download_dataset(
url=DOWNLOAD_URL,
filestore=fs
)
self.validate_class_size_dataset(ds)
self.assertEqual(len(fs.list_files()), 1)
self.assertIsNotNone(fh)
self.assertIsNotNone(fs.get_file(fh.identifier))
def test_upload_file(self):
"""Test file upload."""
db = FileSystemFilestore(SERVER_DIR)
fh = db.upload_file(CSV_FILE)
self.assertEqual(fh.file_name, os.path.basename(CSV_FILE))
self.assertEqual(fh.mimetype, fs.FORMAT_CSV)
self.assertEqual(fh.identifier, db.get_file(fh.identifier).identifier)
self.assertTrue(
os.path.isfile(
os.path.join(SERVER_DIR, fh.identifier, METADATA_FILENAME)))
self.assertTrue(os.path.isfile(fh.filepath))
self.assertTrue(fh.is_tabular)
# Re-load the repository
db = FileSystemFilestore(SERVER_DIR)
fh = db.get_file(fh.identifier)
self.assertEqual(fh.file_name, os.path.basename(CSV_FILE))
self.assertEqual(fh.mimetype, fs.FORMAT_CSV)
self.assertEqual(fh.identifier, db.get_file(fh.identifier).identifier)
# Add files with other valid suffixes
fh = db.upload_file(CSV_FILE)
self.assertFalse(fh.compressed)
self.assertEqual(fh.delimiter, ',')
fh = db.upload_file(GZIP_CSV_FILE)
self.assertTrue(fh.compressed)
self.assertEqual(fh.delimiter, ',')
fh = db.upload_file(TSV_FILE)
self.assertFalse(fh.compressed)
self.assertEqual(fh.delimiter, '\t')
fh = db.upload_file(GZIP_TSV_FILE)
self.assertTrue(fh.compressed)
self.assertEqual(fh.delimiter, '\t')
# Re-load the repository
db = FileSystemFilestore(SERVER_DIR)
self.assertEqual(len(db.list_files()), 5)
fh = db.upload_file(TEXT_FILE)
self.assertFalse(fh.is_tabular)