def test_migrate_containers(files_to_migrate, as_admin, migrate_storage):
"""Testing collection migration"""
# get file stored by hash in legacy storage
(_, _, url_1, file_path_1) = files_to_migrate[0]
# get file stored by uuid in legacy storage
(_, _, url_2, file_path_2) = files_to_migrate[1]
# get file stored by uuid in legacy/v1 storage
(_, _, url_3, file_path_3) = files_to_migrate[2]
# get the ticket
r = as_admin.get(url_1, params={'ticket': ''})
assert r.ok
ticket = r.json()['ticket']
# download the file
assert as_admin.get(url_1, params={'ticket': ticket}).ok
# get the ticket
r = as_admin.get(url_2, params={'ticket': ''})
assert r.ok
ticket = r.json()['ticket']
# download the file
assert as_admin.get(url_2, params={'ticket': ticket}).ok
# get the ticket
r = as_admin.get(url_3, params={'ticket': ''})
assert r.ok
ticket = r.json()['ticket']
# download the file
assert as_admin.get(url_3, params={'ticket': ticket}).ok
# run the migration
migrate_storage.main('--containers')
# delete files from the legacy storage
config.local_fs.remove(file_path_1)
config.local_fs.remove(file_path_2)
config.local_fs2.remove(file_path_3)
# get the files from the new filesystem
# get the ticket
r = as_admin.get(url_1, params={'ticket': ''})
assert r.ok
ticket = r.json()['ticket']
# download the file
assert as_admin.get(url_1, params={'ticket': ticket}).ok
# get the ticket
r = as_admin.get(url_2, params={'ticket': ''})
assert r.ok
ticket = r.json()['ticket']
# download the file
assert as_admin.get(url_2, params={'ticket': ticket}).ok
# get the ticket
r = as_admin.get(url_3, params={'ticket': ''})
assert r.ok
ticket = r.json()['ticket']
# download the file
assert as_admin.get(url_3, params={'ticket': ticket}).ok
def test_migrate_gears_error(gears_to_migrate, migrate_storage,
second_storage_provider):
"""Testing that the migration script throws an exception if it couldn't migrate a file"""
# get the other file, so we can clean up
(_, gear_file_id_2, gear_file_provider_id_2) = gears_to_migrate[0]
# delete the file
source_fs = get_provider(gear_file_provider_id_2).storage_plugin
source_fs.remove_file(gear_file_id_2, None)
with pytest.raises(Exception):
migrate_storage.main('--gears', '--destination',
second_storage_provider)
def test_migrate_gears_error(gears_to_migrate, migrate_storage):
"""Testing that the migration script throws an exception if it couldn't migrate a file"""
# get file storing by hash in legacy storage
(gear_id, gear_file_path_1) = gears_to_migrate[0]
# get the other file, so we can clean up
(_, gear_file_path_2) = gears_to_migrate[1]
# delete the file
config.local_fs.remove_file(None, gear_file_path_1)
with pytest.raises(Exception):
migrate_storage.main('--gears')
# clean up
config.local_fs.remove_file(None, gear_file_path_2)
def test_migrate_gears(gears_to_migrate, as_admin, migrate_storage,
with_site_settings, second_storage_provider):
"""Testing collection migration"""
(gear_id_1, gear_file_id_1, gear_file_provider_id_1) = gears_to_migrate[0]
# get gears before migration
assert as_admin.get('/gears/temp/' + gear_id_1).ok
# run migration
migrate_storage.main('--gears', '--destination', second_storage_provider)
#delete files from the source storage
source_fs = get_provider(gear_file_provider_id_1).storage_plugin
source_fs.remove_file(gear_file_id_1, None)
# get the files that will now be served from the new provider
assert as_admin.get('/gears/temp/' + gear_id_1).ok
def test_migrate_containers_error(files_to_migrate, migrate_storage,
second_storage_provider):
"""Testing that the migration script throws an exception if it couldn't migrate a file"""
#For now we will get the only provider in the system which is osfs for local file minipulation
storage_service = StorageProviderService()
local_fs = storage_service.determine_provider(
None, None, force_site_provider=True).storage_plugin
# get the file to migrate
(_, _, _, file_path_1, provider_id, file_id) = files_to_migrate[0]
# delete the file from the filesystem but its still associated with the acquisition
local_fs.remove_file(file_id, None)
# Migrating with missing data files will error
with pytest.raises(Exception):
migrate_storage.main('--containers', '--destination',
second_storage_provider)
def test_migrate_gears(gears_to_migrate, as_admin, migrate_storage):
"""Testing collection migration"""
(gear_id_1, gear_file_path_1) = gears_to_migrate[0]
(gear_id_2, gear_file_path_2) = gears_to_migrate[1]
# get gears before migration
assert as_admin.get('/gears/temp/' + gear_id_1).ok
assert as_admin.get('/gears/temp/' + gear_id_2).ok
# run migration
migrate_storage.main('--gears')
# delete files from the legacy storage
config.local_fs.remove_file(None, gear_file_path_1)
config.local_fs.remove_file(None, gear_file_path_2)
# get the files from the new filesystem
assert as_admin.get('/gears/temp/' + gear_id_1).ok
assert as_admin.get('/gears/temp/' + gear_id_2).ok
def test_file_replaced_handling(files_to_migrate, migrate_storage, as_admin,
file_form, api_db, mocker, caplog):
origin_find_one_and_update = pymongo.collection.Collection.find_one_and_update
def mocked(*args, **kwargs):
self = args[0]
filter = args[1]
update = args[2]
as_admin.post('/sessions/' + session_id + '/files',
files=file_form((file_name_1, 'new_content')))
return origin_find_one_and_update(self, filter, update)
with mocker.mock_module.patch.object(pymongo.collection.Collection,
'find_one_and_update', mocked):
# get file storing by hash in legacy storage
(session_id, file_name_1, url_1, file_path_1) = files_to_migrate[0]
# get ile storing by uuid in legacy storage
(_, file_name_2, url_2, file_path_2) = files_to_migrate[1]
# run the migration
migrate_storage.main('--containers')
file_1_id = api_db['sessions'].find_one({'files.name': file_name_1
})['files'][0]['_id']
file_2_id = api_db['sessions'].find_one({'files.name': file_name_2
})['files'][1]['_id']
assert config.primary_storage.get_file_info(
file_1_id, util.path_from_uuid(file_1_id)) is not None
assert config.primary_storage.get_file_info(
file_2_id, util.path_from_uuid(file_2_id)) is not None
assert any(
log.message ==
'Probably the following file has been updated during the migration and its hash is changed, cleaning up from the new filesystem'
for log in caplog.records)
def test_migrate_containers(files_to_migrate, as_admin, migrate_storage,
second_storage_provider):
"""Testing collection migration"""
# get file stored by uuid in storage
(_, _, url_2, file_path_2, src_provider_id,
file_id_2) = files_to_migrate[0]
source_fs = get_provider(src_provider_id).storage_plugin
dest_fs = get_provider(second_storage_provider).storage_plugin
# get the ticket
r = as_admin.get(url_2, params={'ticket': ''})
assert r.ok
ticket = r.json()['ticket']
# download the file
assert as_admin.get(url_2, params={'ticket': ticket}).ok
# run the migration
migrate_storage.main(
'--containers',
'--source',
src_provider_id, # Filters on id to avoid migrating other test data in DB
'--destination',
second_storage_provider)
# Verify source file is not deleted
assert source_fs.get_file_info(file_id_2, None) is not None
# delete files from the source storage to clean up
source_fs.remove_file(file_id_2, None)
# Verify file was moved to destination
assert dest_fs.get_file_info(file_id_2, None) is not None
# get the files from the new filesystem
# get the ticket
r = as_admin.get(url_2, params={'ticket': ''})
assert r.ok
ticket = r.json()['ticket']
# download the file
assert as_admin.get(url_2, params={'ticket': ticket}).ok
def test_migrate_analysis(files_to_migrate, as_admin, migrate_storage,
default_payload, data_builder, file_form):
"""Testing analysis migration"""
# get file storing by hash in legacy storage
(session_id, file_name_1, url_1, file_path_1) = files_to_migrate[0]
# get ile storing by uuid in legacy storage
(_, _, url_2, file_path_2) = files_to_migrate[1]
gear_doc = default_payload['gear']['gear']
gear_doc['inputs'] = {'csv': {'base': 'file'}}
gear = data_builder.create_gear(gear=gear_doc)
# create project analysis (job) using project's file as input
r = as_admin.post('/sessions/' + session_id + '/analyses',
json={
'label': 'test analysis job',
'job': {
'gear_id': gear,
'inputs': {
'csv': {
'type': 'session',
'id': session_id,
'name': file_name_1
}
},
'tags': ['example']
}
})
assert r.ok
analysis_id1 = r.json()['_id']
r = as_admin.get('/sessions/' + session_id + '/analyses/' + analysis_id1)
assert r.ok
analysis_files1 = '/sessions/' + session_id + '/analyses/' + analysis_id1 + '/files'
# run the migration
migrate_storage.main('--containers')
# delete files from the legacy storage
config.local_fs.remove_file(None, file_path_1)
config.local_fs.remove_file(None, file_path_2)
# get the files from the new filesystem
# get the ticket
r = as_admin.get(url_1, params={'ticket': ''})
assert r.ok
ticket = r.json()['ticket']
# download the file
assert as_admin.get(url_1, params={'ticket': ticket}).ok
# get the ticket
r = as_admin.get(url_2, params={'ticket': ''})
assert r.ok
ticket = r.json()['ticket']
# download the file
assert as_admin.get(url_2, params={'ticket': ticket}).ok
# get analysis download ticket for single file
r = as_admin.get(analysis_files1 + '/' + file_name_1,
params={'ticket': ''})
assert r.ok
ticket = r.json()['ticket']
# download single analysis file w/ ticket
r = as_admin.get(analysis_files1 + '/' + file_name_1,
params={'ticket': ticket})
assert r.ok
r = as_admin.get('/sessions/' + session_id + '/analyses/' + analysis_id1)
assert r.ok
input_file_id = r.json()['inputs'][0]['_id']
r = as_admin.get('/sessions/' + session_id)
assert r.ok
project_file_id = r.json()['files'][0]['_id']
assert input_file_id == project_file_id
def test_migrate_analysis(files_to_migrate, as_admin, migrate_storage,
default_payload, data_builder, file_form,
second_storage_provider):
"""Testing analysis migration"""
# get file storing by uuid in storage
(session_id, file_name_2, url_2, file_path_2, provider_id_2,
file_id_2) = files_to_migrate[0]
gear_doc = default_payload['gear']['gear']
gear_doc['inputs'] = {'csv': {'base': 'file'}}
gear = data_builder.create_gear(gear=gear_doc)
# create project analysis (job) using project's file as input
r = as_admin.post('/sessions/' + session_id + '/analyses',
json={
'label': 'test analysis job',
'job': {
'gear_id': gear,
'inputs': {
'csv': {
'type': 'session',
'id': session_id,
'name': file_name_2
}
},
'tags': ['example']
}
})
assert r.ok
analysis_id1 = r.json()['_id']
r = as_admin.get('/sessions/' + session_id + '/analyses/' + analysis_id1)
assert r.ok
analysis_files1 = '/sessions/' + session_id + '/analyses/' + analysis_id1 + '/files'
# run the migration
migrate_storage.main('--containers', '--destination',
second_storage_provider)
# delete files from the local storage
source_fs = get_provider(provider_id_2).storage_plugin
source_fs.remove_file(file_id_2, None)
# get the ticket
r = as_admin.get(url_2, params={'ticket': ''})
assert r.ok
ticket = r.json()['ticket']
# download the file from the new provider
assert as_admin.get(url_2, params={'ticket': ticket}).ok
# get analysis download ticket for single file
r = as_admin.get(analysis_files1 + '/' + file_name_2,
params={'ticket': ''})
assert r.ok
ticket = r.json()['ticket']
# download single analysis file w/ ticket
r = as_admin.get(analysis_files1 + '/' + file_name_2,
params={'ticket': ticket})
assert r.ok
r = as_admin.get('/sessions/' + session_id + '/analyses/' + analysis_id1)
assert r.ok
input_file_id = r.json()['inputs'][0]['_id']
r = as_admin.get('/sessions/' + session_id)
assert r.ok
project_file_id = r.json()['files'][0]['_id']
assert input_file_id == project_file_id
#clean up the file on the disk but its still in the db so this will break future tests
dest_fs = get_provider(second_storage_provider).storage_plugin
dest_fs.remove_file(file_id_2, None)