def test_delete_file(self): """Test delete file method.""" db = FileSystemFilestore(SERVER_DIR) f = db.upload_file(CSV_FILE) f = db.get_file(f.identifier) self.assertIsNotNone(f) self.assertTrue(db.delete_file(f.identifier)) f = db.get_file(f.identifier) self.assertIsNone(f)
def test_download_dataset(self): """Test loading a dataset from Url. Note that this test depends on the accessed web service to be running. It will fail otherwise.""" # Skip test if DOWNLOAD_URL is None if DOWNLOAD_URL is None: print('Skipping download test') return store = FileSystemDatastore(STORE_DIR) ds = store.download_dataset(url=DOWNLOAD_URL) dataset_dir = os.path.join(STORE_DIR, ds.identifier) self.assertTrue(os.path.isdir(dataset_dir)) self.assertTrue(os.path.isfile(os.path.join(dataset_dir, DATA_FILE))) self.assertTrue(os.path.isfile(os.path.join(dataset_dir, DESCRIPTOR_FILE))) self.assertFalse(os.path.isfile(os.path.join(dataset_dir, METADATA_FILE))) self.validate_class_size_dataset(ds) # Download file into a given filestore fs = FileSystemFilestore(FSSTORE_DIR) ds, fh = store.download_dataset( url=DOWNLOAD_URL, filestore=fs ) self.validate_class_size_dataset(ds) self.assertEqual(len(fs.list_files()), 1) self.assertIsNotNone(fh) self.assertIsNotNone(fs.get_file(fh.identifier))
def test_upload_stream(self): """Test file upload from an open file object.""" db = FileSystemFilestore(SERVER_DIR) file = FileStorage(filename=CSV_FILE) fh = db.upload_stream(file=file, file_name=os.path.basename(CSV_FILE)) self.assertEqual(fh.file_name, os.path.basename(CSV_FILE)) self.assertEqual(fh.mimetype, fs.FORMAT_CSV) self.assertTrue(os.path.isfile(fh.filepath)) self.assertEqual(fh.identifier, db.get_file(fh.identifier).identifier)
def test_get_file(self): """Test file get method.""" db = FileSystemFilestore(SERVER_DIR) fh1 = db.upload_file(CSV_FILE) fh2 = db.get_file(fh1.identifier) self.assertEqual(fh1.identifier, fh2.identifier) self.assertEqual(fh1.filepath, fh2.filepath) self.assertEqual(fh1.mimetype, fh2.mimetype) # Ensure that the file parses as a CSV file with fh1.open() as csvfile: rows = 0 for row in csv.reader(csvfile, delimiter=fh1.delimiter): rows += 1 self.assertEqual(rows, 3)
def test_upload_file(self): """Test file upload.""" db = FileSystemFilestore(SERVER_DIR) fh = db.upload_file(CSV_FILE) self.assertEqual(fh.file_name, os.path.basename(CSV_FILE)) self.assertEqual(fh.mimetype, fs.FORMAT_CSV) self.assertEqual(fh.identifier, db.get_file(fh.identifier).identifier) self.assertTrue( os.path.isfile( os.path.join(SERVER_DIR, fh.identifier, METADATA_FILENAME))) self.assertTrue(os.path.isfile(fh.filepath)) self.assertTrue(fh.is_tabular) # Re-load the repository db = FileSystemFilestore(SERVER_DIR) fh = db.get_file(fh.identifier) self.assertEqual(fh.file_name, os.path.basename(CSV_FILE)) self.assertEqual(fh.mimetype, fs.FORMAT_CSV) self.assertEqual(fh.identifier, db.get_file(fh.identifier).identifier) # Add files with other valid suffixes fh = db.upload_file(CSV_FILE) self.assertFalse(fh.compressed) self.assertEqual(fh.delimiter, ',') fh = db.upload_file(GZIP_CSV_FILE) self.assertTrue(fh.compressed) self.assertEqual(fh.delimiter, ',') fh = db.upload_file(TSV_FILE) self.assertFalse(fh.compressed) self.assertEqual(fh.delimiter, '\t') fh = db.upload_file(GZIP_TSV_FILE) self.assertTrue(fh.compressed) self.assertEqual(fh.delimiter, '\t') # Re-load the repository db = FileSystemFilestore(SERVER_DIR) self.assertEqual(len(db.list_files()), 5) fh = db.upload_file(TEXT_FILE) self.assertFalse(fh.is_tabular)