def test_delete_file(self):
bucket = s3.S3Bucket(config_filename=self.config_filename,
secret_filename=self.secret_filename)
def delete_object(Bucket=None, Key=None):
return None
bucket.client = mock.Mock(delete_object=delete_object)
bucket.delete_file('key')
def test_get_file_properties(self):
bucket = s3.S3Bucket(config_filename=self.config_filename,
secret_filename=self.secret_filename)
def get_object(Bucket=None, Key=None):
return {}
bucket.client = mock.Mock(get_object=get_object)
self.assertEqual(bucket.get_file_properties('key'), {})
def test_download_file(self):
bucket = s3.S3Bucket(config_filename=self.config_filename,
secret_filename=self.secret_filename)
def download_file(Key=None, Filename=None):
return None
bucket.bucket = mock.Mock(download_file=download_file)
bucket.download_file('key', 'filename')
def test_upload_file(self):
bucket = s3.S3Bucket(config_filename=self.config_filename,
secret_filename=self.secret_filename)
def upload_file(Filename=None, Key=None, ExtraArgs=None):
return None
bucket.bucket = mock.Mock(upload_file=upload_file)
self.assertEqual(
bucket.upload_file('filename', 'key', False),
bucket.endpoint + '/' + bucket.bucket_name + '/' + 'key')
self.assertEqual(
bucket.upload_file('filename', 'key', True),
bucket.endpoint + '/' + bucket.bucket_name + '/' + 'key')
def test_list_files(self):
bucket = s3.S3Bucket(config_filename=self.config_filename,
secret_filename=self.secret_filename)
def list_objects(Bucket=None, Prefix=None, Marker=None, MaxKeys=1000):
bucket._counter += 1
if bucket._counter <= 1:
return {
'Contents': [
{
'Key': 'a',
'LastModified': datetime.datetime(2021, 1, 1)
},
{
'Key': 'b',
'LastModified': datetime.datetime(2022, 1, 1)
},
{
'Key': 'c',
'LastModified': datetime.datetime(2020, 1, 1)
},
],
'IsTruncated':
True,
}
else:
return {
'Contents': [],
'IsTruncated': False,
}
bucket.client = mock.Mock(list_objects=list_objects)
bucket._counter = 0
self.assertEqual(list(bucket.list_files('key')), ['a', 'b', 'c'])
bucket._counter = 0
self.assertEqual(
list(
bucket.list_files('key',
max_last_modified=datetime.datetime(
2021, 1, 1))), ['a', 'c'])
def test_is_file(self):
bucket = s3.S3Bucket(config_filename=self.config_filename,
secret_filename=self.secret_filename)
def get_object(Bucket=None, Key=None):
if Key == 'key1':
return {}
elif Key == 'key2':
raise ClientError({'Error': {
'Code': 'NoSuchKey'
}}, 'get_object')
elif Key == 'key3':
raise ClientError({'Error': {
'Code': 'ValueError'
}}, 'get_object')
bucket.client = mock.Mock(get_object=get_object)
self.assertTrue(bucket.is_file('key1'))
self.assertFalse(bucket.is_file('key2'))
with self.assertRaises(ClientError):
bucket.is_file('key3')
from PIL import Image
from src import s3
import json
import requests
deployment = 'prod'
bucket = s3.S3Bucket()
run_ids = set()
with open(
'apps/dispatch/src/app/components/simulations/browse/example-simulations.{}.json'
.format(deployment), 'r') as file:
for run in json.load(file):
run_ids.add(run['id'])
response = requests.get('https://api.biosimulations.{}/projects'.format(
'org' if deployment == 'prod' else 'dev'))
response.raise_for_status()
for run in response.json():
run_ids.add(run['simulationRun'])
run_ids = sorted(run_ids)
for i_run, run_id in enumerate(run_ids):
print('{} of {}'.format(i_run + 1, len(run_ids)))
response = requests.get('https://api.biosimulations.{}/files/{}'.format(
'org' if deployment == 'prod' else 'dev', run_id))
response.raise_for_status()
for file in response.json():
def test_delete_files_with_prefix(self):
bucket = s3.S3Bucket(config_filename=self.config_filename,
secret_filename=self.secret_filename)
bucket._counter = 0
def list_objects(Bucket=None, Prefix=None, Marker=None, MaxKeys=1000):
bucket._counter += 1
if bucket._counter <= 1:
return {
'Contents': [
{
'Key': 'a',
'LastModified': datetime.datetime(2021, 1, 1)
},
{
'Key': 'b',
'LastModified': datetime.datetime(2022, 1, 1)
},
{
'Key': 'c',
'LastModified': datetime.datetime(2020, 1, 1)
},
],
'NextMarker':
'x',
'IsTruncated':
True,
}
elif bucket._counter <= 2:
return {
'Contents': [
{
'Key': 'a',
'LastModified': datetime.datetime(2021, 1, 1)
},
{
'Key': 'b',
'LastModified': datetime.datetime(2022, 1, 1)
},
{
'Key': 'c',
'LastModified': datetime.datetime(2020, 1, 1)
},
],
'NextMarker':
None,
'IsTruncated':
True,
}
else:
return {
'Contents': [],
'NextMarker': None,
'IsTruncated': False,
}
def delete_objects(Bucket=None, Delete=None):
return None
bucket.client = mock.Mock(list_objects=list_objects,
delete_objects=delete_objects)
bucket.delete_files_with_prefix('key')