def test_ensure_glacier_exists(self, mock_download):
version = FileVersionFactory()
version.metadata['archive'] = 'foo'
version.save()
self.ctx.vault = mock.Mock()
ensure_glacier(self.ctx, version, dry_run=False)
assert_false(self.ctx.vault.upload_archive.called)
def test_download(self, mock_container, mock_exists):
files_audit.audit_temp_path = os.path.join(
storage_settings.AUDIT_TEMP_PATH)
file_contents = ['fake', 'file', 'content']
mock_obj = mock.Mock()
mock_obj.fetch.return_value = iter(file_contents)
mock_container.get_object.return_value = mock_obj
mock_exists.return_value = False
version = FileVersionFactory()
version.metadata = {
'sha256':
'dff32002043d7a4da7173d2034cb2f6856d10549bd8cc6d7e16d62f1304681f8'
} # fakefilecontent
mock_open = mock.mock_open()
with mock.patch('scripts.osfstorage.files_audit.open',
mock_open,
create=True):
download_from_cloudfiles(version)
mock_container.get_object.assert_called_with(
version.location['object'])
mock_open.assert_called_once_with(
os.path.join(os.path.join(storage_settings.AUDIT_TEMP_PATH),
version.location['object']), 'wb')
handle = mock_open()
assert_equal(handle.write.call_count, 3)
for content in file_contents:
handle.write.assert_any_call(content)
def test_ensure_glacier_exists(self, mock_download):
version = FileVersionFactory()
version.metadata['archive'] = 'foo'
version.save()
self.ctx.vault = mock.Mock()
ensure_glacier(self.ctx, version, dry_run=False)
assert_false(self.ctx.vault.upload_archive.called)
def test_ensure_glacier(self, mock_vault, mock_download):
glacier_id = 'iamarchived'
version = FileVersionFactory()
file_path = os.path.join(storage_settings.AUDIT_TEMP_PATH, version.location['object'])
mock_download.return_value = file_path
mock_vault.upload_archive.return_value = glacier_id
ensure_glacier(version, dry_run=False)
mock_vault.upload_archive.assert_called_with(file_path, description=version.location['object'])
version.reload()
assert_equal(version.metadata['archive'], glacier_id)
def test_ensure_glacier(self, mock_vault, mock_download):
glacier_id = 'iamarchived'
version = FileVersionFactory()
file_path = os.path.join(storage_settings.AUDIT_TEMP_PATH,
version.location['object'])
mock_download.return_value = file_path
mock_vault.upload_archive.return_value = glacier_id
ensure_glacier(version, dry_run=False)
mock_vault.upload_archive.assert_called_with(
file_path, description=version.location['object'])
version.reload()
assert_equal(version.metadata['archive'], glacier_id)
def test_ensure_parity(self, mock_container, mock_download,
mock_create_parity, mock_remove):
mock_container.list_all.return_value = []
mock_create_parity.return_value = ['hi'] * 8
version = FileVersionFactory()
ensure_parity(version, dry_run=False)
assert_equal(len(mock_container.create.call_args_list), 8)
def test_get_targets(self):
versions = [FileVersionFactory() for _ in range(5)]
versions[0].location['container'] = TEST_CONTAINER_NAME
versions[0].save()
targets = get_targets()
assert_equal(len(targets), 1)
assert_equal(targets[0], versions[0])
def test_ensure_parity_exists(self, mock_download, mock_create_parity):
self.ctx.container_parity = mock.Mock()
self.ctx.container_parity.list_all.side_effect = [['hi'], ['hi'] * 4]
version = FileVersionFactory()
ensure_parity(self.ctx, version, dry_run=False)
assert_false(mock_download.called)
assert_false(self.ctx.container_parity.create.called)
def test_ensure_parity_exists(self, mock_container, mock_download,
mock_create_parity):
mock_container.list_all.side_effect = [['hi'], ['hi'] * 4]
version = FileVersionFactory()
ensure_parity(version, dry_run=False)
assert_false(mock_download.called)
assert_false(mock_container.create.called)
def test_download(self, mock_exists):
files_audit.audit_temp_path = os.path.join(storage_settings.AUDIT_TEMP_PATH)
file_contents = ['fake', 'file', 'content']
self.ctx.container_primary = mock.Mock()
self.ctx.container_primary.get_object.return_value.fetch.return_value = iter(file_contents)
version = FileVersionFactory()
version.metadata = {'sha256': 'dff32002043d7a4da7173d2034cb2f6856d10549bd8cc6d7e16d62f1304681f8'} # fakefilecontent
mock_open = mock.mock_open()
with mock.patch('scripts.osfstorage.files_audit.open', mock_open, create=True):
download_from_cloudfiles(self.ctx, version)
self.ctx.container_primary.get_object.assert_called_with(version.location['object'])
mock_open.assert_called_once_with(os.path.join(os.path.join(storage_settings.AUDIT_TEMP_PATH), version.location['object']), 'wb')
handle = mock_open()
assert_equal(handle.write.call_count, 3)
for content in file_contents:
handle.write.assert_any_call(content)
def test_ensure_glacier_not_multipart(self, mock_getsize, mock_download):
glacier_id = 'iamarchived'
version = FileVersionFactory()
file_path = os.path.join(storage_settings.AUDIT_TEMP_PATH, version.location['object'])
mock_download.return_value = file_path
self.ctx.vault = mock.Mock()
self.ctx.vault.upload_archive.return_value.id = glacier_id
with mock.patch('scripts.osfstorage.files_audit.open', mock.mock_open(read_data='hello world'), create=True) as mfp:
ensure_glacier(self.ctx, version, dry_run=False)
mfp.assert_called_once_with(file_path, 'rb')
self.ctx.vault.upload_archive.assert_called_with(
vaultName=storage_settings.GLACIER_VAULT_NAME,
archiveDescription=version.location['object'],
body=mfp(),
)
version.reload()
assert_equal(version.metadata['archive'], glacier_id)
def test_inventory(self):
version = FileVersionFactory(
size=24601,
metadata={'archive': '123456'},
location={
'service': 'cloud',
'container': 'cloud',
'object': 'abcdef',
},
)
glacier_audit.check_glacier_version(version, mock_inventory)
def test_inventory_wrong_size(self):
version = FileVersionFactory(
size=24602,
metadata={'archive': '123456'},
location={
'service': 'cloud',
'container': 'cloud',
'object': 'abcdef',
},
)
with assert_raises(glacier_audit.BadSize):
glacier_audit.check_glacier_version(version, mock_inventory)
def test_inventory_not_found(self):
version = FileVersionFactory(
size=24601,
metadata={'archive': '123456'},
location={
'service': 'cloud',
'container': 'cloud',
'object': 'abcdefg',
},
)
with assert_raises(glacier_audit.NotFound):
glacier_audit.check_glacier_version(version, mock_inventory)
def test_ensure_glacier_not_multipart(self, mock_getsize, mock_download):
glacier_id = 'iamarchived'
version = FileVersionFactory()
file_path = os.path.join(storage_settings.AUDIT_TEMP_PATH,
version.location['object'])
mock_download.return_value = file_path
self.ctx.vault = mock.Mock()
self.ctx.vault.upload_archive.return_value.id = glacier_id
with mock.patch('scripts.osfstorage.files_audit.open',
mock.mock_open(read_data='hello world'),
create=True) as mfp:
ensure_glacier(self.ctx, version, dry_run=False)
mfp.assert_called_once_with(file_path, 'rb')
self.ctx.vault.upload_archive.assert_called_with(
vaultName=storage_settings.GLACIER_VAULT_NAME,
archiveDescription=version.location['object'],
body=mfp(),
)
version.reload()
assert_equal(version.metadata['archive'], glacier_id)
def test_migrate_version(self, mock_test_container, mock_prod_container):
mock_test_object = mock.Mock()
mock_test_container.get_object.return_value = mock_test_object
version = FileVersionFactory()
version.location['container'] = TEST_CONTAINER_NAME
version.save()
migrate_version(version)
mock_test_container.get_object.assert_called_with(
version.location['object'])
mock_test_object.copy.assert_called_with(mock_prod_container)
version.reload()
assert_equal(version.location['container'], PROD_CONTAINER_NAME)
assert_equal(len(get_targets()), 0)
def setUp(self):
super(TestMigrateDates, self).setUp()
self.path = 'old-pizza'
self.project = ProjectFactory()
self.node_settings = self.project.get_addon('osfstorage')
self.node_file = NodeFile(path=self.path)
self.node_file.save()
self.node_file.reload()
self.date = self.node_file.date_modified
self.project.files_versions['old_pizza'] = [self.node_file._id]
self.project.save()
self.version = FileVersionFactory(date_modified=timezone.now())
self.record, _ = OsfStorageFileRecord.get_or_create(
self.node_file.path, self.node_settings)
self.record.versions = [self.version]
self.record.save()
def test_ensure_glacier_is_multipart(self, mock_getsize,
mock_multipart_upload, mock_download):
glacier_id = 'omgitsanid'
version = FileVersionFactory()
file_path = os.path.join(storage_settings.AUDIT_TEMP_PATH,
version.location['object'])
mock_download.return_value = file_path
mock_multipart_upload.return_value = glacier_id
ensure_glacier(self.ctx, version, dry_run=False)
mock_multipart_upload.assert_called_with(
self.ctx,
version,
file_path,
files_audit.GLACIER_SINGLE_OPERATION_THRESHOLD +
1 # 100 MB + 1 Byte
)
assert_equal(version.metadata['archive'], glacier_id)
def test_migrate_version(self, mock_test_container, mock_prod_container):
mock_test_object = mock.Mock()
mock_test_container.get_object.return_value = mock_test_object
version = FileVersionFactory()
version.location['container'] = TEST_CONTAINER_NAME
version.save()
migrate_version(version)
mock_test_container.get_object.assert_called_with(version.location['object'])
mock_test_object.copy.assert_called_with(mock_prod_container)
version.reload()
assert_equal(version.location['container'], PROD_CONTAINER_NAME)
assert_equal(len(get_targets()), 0)
def test_dry_run(self):
versions = [FileVersionFactory() for _ in range(5)]
versions[0].location['container'] = TEST_CONTAINER_NAME
versions[0].save()
main(dry_run=True)
assert_equal(len(get_targets()), 1)
