def test_md5(self):
"""
"""
test_file = os.path.join(os.path.dirname(__file__), "fixtures",
"links.md")
# Local
local_file = StorageFile(test_file)
local_md5 = local_file.md5_hex
self.assertIsNotNone(local_md5)
# GCP
gs_path = "gs://aries_test/links.md"
local_file.copy(gs_path)
gs_file = StorageFile(gs_path)
self.assertEqual(local_md5, gs_file.md5_hex)
gs_file.delete()
# AWS
if os.environ.get("AWS_SECRET_ACCESS_KEY") and os.environ.get(
"AWS_ACCESS_KEY_ID"):
s3_path = "s3://davelab-test/links.md"
local_file.copy(s3_path)
s3_file = StorageFile(s3_path)
self.assertEqual(local_md5, s3_file.md5_hex)
s3_file.delete()
def test_http(self):
"""
"""
# URL does not exist
storage_obj = StorageFile("http://example.com/abc/")
self.assertFalse(storage_obj.exists())
# URL exists
storage_obj = StorageFile("https://www.google.com")
self.assertTrue(storage_obj.exists())
# Download. Copy to local file.
storage_obj = StorageFile("https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf")
local_file_path = os.path.join(self.test_folder_path, "test.pdf")
if os.path.exists(local_file_path):
os.remove(local_file_path)
storage_obj.copy(local_file_path)
self.assertTrue(os.path.exists(local_file_path))
self.assertGreater(StorageFile(local_file_path).size, 0)
StorageFile(local_file_path).delete()
def test_create_copy_and_delete_file(self):
new_folder_uri = os.path.join(self.TEST_ROOT, "new_folder")
with TempFolder(new_folder_uri) as folder:
self.assertTrue(folder.is_empty())
# Create a sub folder inside the new folder
sub_folder_uri = os.path.join(new_folder_uri, "sub_folder")
logger.debug(sub_folder_uri)
sub_folder = StorageFolder(sub_folder_uri).create()
self.assertTrue(sub_folder.exists())
# Copy an empty file
src_file_path = os.path.join(self.TEST_ROOT, "test_folder_0",
"empty_file")
dst_file_path = os.path.join(new_folder_uri, "copied_file")
f = StorageFile(src_file_path)
logger.debug(f.exists())
time.sleep(2)
f.copy(dst_file_path)
self.assertTrue(StorageFile(dst_file_path).exists())
# Copy a file with content and replace the empty file
src_file_path = os.path.join(self.TEST_ROOT, "test_folder_0",
"abc.txt")
dst_file_path = os.path.join(new_folder_uri, "copied_file")
f = StorageFile(src_file_path)
f.copy(dst_file_path)
dst_file = StorageFile(dst_file_path)
self.assertTrue(dst_file.exists())
# Use the shortcut to read file, the content will be binary.
self.assertEqual(dst_file.read(), b"abc\ncba\n")
# Empty the folder. This should delete file and sub folder only
folder.empty()
self.assertTrue(folder.exists())
self.assertTrue(folder.is_empty())
self.assertFalse(sub_folder.exists())
self.assertFalse(dst_file.exists())
def copy_from_http(self):
storage_obj = StorageFile("https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf")
gs_path = "gs://davelab_temp/qq6/test.pdf"
storage_obj.copy("gs://davelab_temp/qq6/test.pdf")
self.assertTrue(StorageFile(gs_path).exists())
StorageFile(gs_path).delete()