def test_mount_blobs(self):
testbucketname = os.getenv('MLGIT_TEST_BUCKET', 'ml-git-datasets')
hfspath = os.path.join(self.tmp_dir, 'objectsfs')
s3 = boto3.resource(
's3',
region_name='eu-west-1',
aws_access_key_id='fake_access_key',
aws_secret_access_key='fake_secret_key',
)
keypath = 'zdj7We7Je5MRECsZUF7uptseHHPY29zGoqFsVHw6sbgv1MbWS'
file = os.path.join('hdata', keypath)
with open(file, 'rb') as f:
s3.Bucket(testbucketname).Object(keypath).put(file, Body=f)
c = yaml_load('hdata/config.yaml')
r = LocalRepository(c, hfspath)
s3store = S3Store(testbucketname, bucket)
links = {'Links': [{'Hash': keypath, 'Size': 16822}]}
with open(file, 'rb') as f:
self.assertEqual(f.read(), r._mount_blobs(s3store, links))
def test_get_ipld(self):
testbucketname = os.getenv('MLGIT_TEST_BUCKET', 'ml-git-datasets')
hfspath = os.path.join(self.tmp_dir, 'objectsfs')
s3 = boto3.resource(
's3',
region_name='eu-west-1',
aws_access_key_id='fake_access_key',
aws_secret_access_key='fake_secret_key',
)
keypath = 'zdj7WdjnTVfz5AhTavcpsDT62WiQo4AeQy6s4UC1BSEZYx4NP'
file = os.path.join('hdata', keypath)
with open(file, 'rb') as f:
s3.Bucket(testbucketname).Object(keypath).put(file, Body=f)
c = yaml_load('hdata/config.yaml')
r = LocalRepository(c, hfspath)
s3store = S3Store(testbucketname, bucket)
links = {
'Links': [{
'Hash': 'zdj7WVyQ8wTdnDXsbg8wxwwFkt2Bzp95Tncsfg8PCgKXeLTye',
'Size': 16822
}]
}
self.assertEqual(links, r._get_ipld(s3store, keypath))
def test_put_object(self):
s3store = S3Store(bucketname, bucket)
f = 'data/think-hires.jpg'
k = 'path/think-hires.jpg'
self.assertFalse(s3store.key_exists(k))
with open(f, 'rb') as file:
s3store.put_object(k, file.read())
self.assertTrue(s3store.key_exists(k))
def test_list_files_from_path(self):
s3store = S3Store(bucketname, bucket)
k = 'path/think-hires.jpg'
f = 'data/think-hires.jpg'
s3store.put(k, f)
self.assertTrue(s3store.key_exists('path/think-hires.jpg'))
files = s3store.list_files_from_path('path')
self.assertEqual(files[0], 'path/think-hires.jpg')
files = s3store.list_files_from_path(None)
self.assertEqual(files[0], 'path/think-hires.jpg')
def test_get_object(self):
s3store = S3Store(bucketname, bucket)
k = 'path/think-hires.jpg'
f = 'data/think-hires.jpg'
self.assertFalse(s3store.key_exists(k))
s3store.put(k, f)
img = s3store.get_object(k)
with open(f, 'rb') as file:
img2 = file.read()
self.assertEqual(img, img2)
s3store._delete(k)
self.assertRaises(Exception, s3store.get_object, k)