def test_md5(self):
"""
"""
test_file = os.path.join(os.path.dirname(__file__), "fixtures",
"links.md")
# Local
local_file = StorageFile(test_file)
local_md5 = local_file.md5_hex
self.assertIsNotNone(local_md5)
# GCP
gs_path = "gs://aries_test/links.md"
local_file.copy(gs_path)
gs_file = StorageFile(gs_path)
self.assertEqual(local_md5, gs_file.md5_hex)
gs_file.delete()
# AWS
if os.environ.get("AWS_SECRET_ACCESS_KEY") and os.environ.get(
"AWS_ACCESS_KEY_ID"):
s3_path = "s3://davelab-test/links.md"
local_file.copy(s3_path)
s3_file = StorageFile(s3_path)
self.assertEqual(local_md5, s3_file.md5_hex)
s3_file.delete()
def test_gs_file(self):
"""Tests accessing a Google Cloud Storage file.
"""
# Test the blob property
# File exists
gs_file_exists = StorageFile("gs://aries_test/file_in_root.txt")
self.assertFalse(gs_file_exists.is_gz())
self.assertTrue(gs_file_exists.blob.exists())
self.assertEqual(gs_file_exists.size, 34)
# File does not exists
gs_file_null = StorageFile("gs://aries_test/abc.txt")
self.assertFalse(gs_file_null.blob.exists())
# Test the read() method
self.assertEqual(gs_file_exists.read(), b'This is a file in the bucket root.')
with self.assertRaises(Exception):
gs_file_null.read()
# Test write into a new file
with gs_file_null('w+b') as f:
f.write(b"abc")
f.seek(0)
self.assertEqual(f.read(), b"abc")
# File will be uploaded to bucket after closed.
# Test reading from the bucket
self.assertEqual(gs_file_null.read(), b"abc")
gs_file_null.delete()
def test_create_and_move_blob(self):
gs_file = StorageFile("gs://aries_test/new_file.txt")
self.assertFalse(gs_file.blob.exists())
gs_file.create()
self.assertTrue(gs_file.blob.exists())
dest = "gs://aries_test/moved_file.txt"
gs_file.move(dest)
self.assertFalse(gs_file.exists())
dest_file = StorageFile(dest)
self.assertTrue(dest_file.exists())
dest_file.delete()
def test_upload_from_file(self):
gs_file = StorageFile("gs://aries_test/local_upload.txt")
# Try to upload a file that does not exist.
local_file_non_exist = os.path.join(os.path.dirname(__file__), "abc.txt")
with self.assertRaises(FileNotFoundError):
gs_file.upload_from_file(local_file_non_exist)
# Upload a file and check the content.
local_file = os.path.join(os.path.dirname(__file__), "fixtures", "test_file.txt")
gs_file.upload_from_file(local_file)
self.assertEqual(gs_file.read(), b'This is a local test file.\n')
gs_file.delete()
def test_binary_read_write(self):
# File does not exist, a new one will be created
file_uri = os.path.join(self.TEST_ROOT, "test.txt")
storage_file = StorageFile(file_uri).open("wb")
self.assertEqual(storage_file.scheme, self.SCHEME)
self.assertTrue(storage_file.seekable())
self.assertFalse(storage_file.readable())
self.assertEqual(storage_file.write(b"abc"), 3)
self.assertEqual(storage_file.tell(), 3)
self.assertEqual(storage_file.write(b"def"), 3)
self.assertEqual(storage_file.tell(), 6)
storage_file.close()
self.assertTrue(storage_file.exists())
storage_file.open('rb')
self.assertEqual(storage_file.read(), b"abcdef")
storage_file.close()
storage_file.delete()
self.assertFalse(storage_file.exists())