def setUp(self):
"""Create instances of the default datastore and filestore."""
# Drop directory if it exists
if os.path.isdir(SERVER_DIR):
shutil.rmtree(SERVER_DIR)
os.makedirs(SERVER_DIR)
self.datastore = FileSystemDatastore(DATASTORE_DIR)
self.filestore = FileSystemFilestore(FILESTORE_DIR)
def test_delete_file(self):
"""Test delete file method."""
db = FileSystemFilestore(SERVER_DIR)
f = db.upload_file(CSV_FILE)
f = db.get_file(f.identifier)
self.assertIsNotNone(f)
self.assertTrue(db.delete_file(f.identifier))
f = db.get_file(f.identifier)
self.assertIsNone(f)
def test_upload_stream(self):
"""Test file upload from an open file object."""
db = FileSystemFilestore(SERVER_DIR)
file = FileStorage(filename=CSV_FILE)
fh = db.upload_stream(file=file, file_name=os.path.basename(CSV_FILE))
self.assertEqual(fh.file_name, os.path.basename(CSV_FILE))
self.assertEqual(fh.mimetype, fs.FORMAT_CSV)
self.assertTrue(os.path.isfile(fh.filepath))
self.assertEqual(fh.identifier, db.get_file(fh.identifier).identifier)
def setUp(self):
"""Create an instance of the Mimir processor for an empty server
directory.
"""
# Drop directory if it exists
if os.path.isdir(SERVER_DIR):
shutil.rmtree(SERVER_DIR)
os.makedirs(SERVER_DIR)
self.datastore = MimirDatastore(DATASTORE_DIR)
self.filestore = FileSystemFilestore(FILESTORE_DIR)
def setUp(self):
"""Create an instance of the default vizier processor for an empty server
directory.
"""
# Drop directory if it exists
if os.path.isdir(SERVER_DIR):
shutil.rmtree(SERVER_DIR)
os.makedirs(SERVER_DIR)
self.processor = VizualTaskProcessor(api=DefaultVizualApi())
self.datastore = FileSystemDatastore(DATASTORE_DIR)
self.filestore = FileSystemFilestore(FILESTORE_DIR)
def set_up(self, engine):
"""Create an empty file server repository."""
if os.path.isdir(SERVER_DIR):
shutil.rmtree(SERVER_DIR)
os.mkdir(SERVER_DIR)
# Setup file server
self.fs = FileSystemFilestore(FILESERVER_DIR)
# Setup the respective datastore and Vizual engine
if engine == ENGINEENV_DEFAULT:
self.datastore = FileSystemDatastore(DATASTORE_DIR)
elif engine == ENGINEENV_MIMIR:
self.datastore = MimirDatastore(DATASTORE_DIR)
def setUp(self):
"""Create an instance of the Mimir processor for an empty server
directory.
"""
# Drop directory if it exists
if os.path.isdir(SERVER_DIR):
shutil.rmtree(SERVER_DIR)
os.makedirs(SERVER_DIR)
self.processor = MimirProcessor()
self.datastore = MimirDatastore(DATASTORE_DIR)
self.filestore = FileSystemFilestore(FILESTORE_DIR)
self.available_lenses = set(mimir.getAvailableLensTypes())
def test_get_file(self):
"""Test file get method."""
db = FileSystemFilestore(SERVER_DIR)
fh1 = db.upload_file(CSV_FILE)
fh2 = db.get_file(fh1.identifier)
self.assertEqual(fh1.identifier, fh2.identifier)
self.assertEqual(fh1.filepath, fh2.filepath)
self.assertEqual(fh1.mimetype, fh2.mimetype)
# Ensure that the file parses as a CSV file
with fh1.open() as csvfile:
rows = 0
for row in csv.reader(csvfile, delimiter=fh1.delimiter):
rows += 1
self.assertEqual(rows, 3)
def get_filestore(self, identifier):
"""Get the filestore instance for the project with the given identifier.
Paramaters
----------
identifier: string
Unique identifier for filestore
Returns
-------
vizier.filestore.fs.base.FileSystemFilestore
"""
filestore_dir = os.path.join(self.base_path, identifier)
return FileSystemFilestore(filestore_dir)
def test_upload_file(self):
"""Test file upload."""
db = FileSystemFilestore(SERVER_DIR)
fh = db.upload_file(CSV_FILE)
self.assertEqual(fh.file_name, os.path.basename(CSV_FILE))
self.assertEqual(fh.mimetype, fs.FORMAT_CSV)
self.assertEqual(fh.identifier, db.get_file(fh.identifier).identifier)
self.assertTrue(
os.path.isfile(
os.path.join(SERVER_DIR, fh.identifier, METADATA_FILENAME)))
self.assertTrue(os.path.isfile(fh.filepath))
self.assertTrue(fh.is_tabular)
# Re-load the repository
db = FileSystemFilestore(SERVER_DIR)
fh = db.get_file(fh.identifier)
self.assertEqual(fh.file_name, os.path.basename(CSV_FILE))
self.assertEqual(fh.mimetype, fs.FORMAT_CSV)
self.assertEqual(fh.identifier, db.get_file(fh.identifier).identifier)
# Add files with other valid suffixes
fh = db.upload_file(CSV_FILE)
self.assertFalse(fh.compressed)
self.assertEqual(fh.delimiter, ',')
fh = db.upload_file(GZIP_CSV_FILE)
self.assertTrue(fh.compressed)
self.assertEqual(fh.delimiter, ',')
fh = db.upload_file(TSV_FILE)
self.assertFalse(fh.compressed)
self.assertEqual(fh.delimiter, '\t')
fh = db.upload_file(GZIP_TSV_FILE)
self.assertTrue(fh.compressed)
self.assertEqual(fh.delimiter, '\t')
# Re-load the repository
db = FileSystemFilestore(SERVER_DIR)
self.assertEqual(len(db.list_files()), 5)
fh = db.upload_file(TEXT_FILE)
self.assertFalse(fh.is_tabular)
def test_list_file(self):
"""Test list files method."""
db = FileSystemFilestore(SERVER_DIR)
db.upload_file(CSV_FILE)
db.upload_file(GZIP_CSV_FILE)
db.upload_file(TSV_FILE)
db.upload_file(GZIP_TSV_FILE)
files = db.list_files()
self.assertEqual(len(files), 4)
db.upload_file(CSV_FILE)
db.upload_file(GZIP_CSV_FILE)
db.upload_file(TSV_FILE)
db.upload_file(GZIP_TSV_FILE)
files = db.list_files()
self.assertEqual(len(files), 8)
def test_load_dataset(self):
"""Test validation of load dataset command."""
db = FileSystemFilestore(SERVER_DIR)
fh = db.upload_file(CSV_FILE)
cmd = load_dataset(dataset_name='ds',
file={
pckg.FILE_ID: fh.identifier,
pckg.FILE_NAME: fh.file_name
},
validate=True).to_external_form(command=PACKAGE.get(
vizual.VIZUAL_LOAD),
datasets=DATASETS)
self.assertEqual(cmd, 'LOAD DATASET ds FROM ' + fh.file_name)
cmd = load_dataset(dataset_name='ds',
file={
pckg.FILE_URL: 'http://some.file.url'
},
validate=True).to_external_form(command=PACKAGE.get(
vizual.VIZUAL_LOAD),
datasets=DATASETS)
self.assertEqual(cmd, 'LOAD DATASET ds FROM http://some.file.url')
cmd = load_dataset(dataset_name='ds',
file={
pckg.FILE_ID: fh.identifier,
pckg.FILE_URL: 'http://some.file.url'
},
validate=True).to_external_form(command=PACKAGE.get(
vizual.VIZUAL_LOAD),
datasets=DATASETS)
self.assertEqual(cmd, 'LOAD DATASET ds FROM http://some.file.url')
cmd = load_dataset(dataset_name='ds',
file={
pckg.FILE_ID: 'Some File'
},
validate=True).to_external_form(command=PACKAGE.get(
vizual.VIZUAL_LOAD),
datasets=DATASETS)
self.assertEqual(cmd, 'LOAD DATASET ds FROM \'Some File\'')
def setup_fileserver(self):
"""Create a fresh file server."""
if os.path.isdir(FILESERVER_DIR):
shutil.rmtree(FILESERVER_DIR)
os.mkdir(FILESERVER_DIR)
self.fileserver = FileSystemFilestore(FILESERVER_DIR)
