def test_removeAll(self):
b = Bank('local')
b.update()
b.removeAll()
self.assertFalse(os.path.exists(b.get_data_dir()))
bdb = b.banks.find_one({'name': b.name})
self.assertTrue(bdb is None)
def biomaj_status_ko(options, config):
'''
Get failed banks
'''
banks = Bank.list()
banks_list = []
headers = ["Name", "Type(s)", "Release", "Visibility", "Last run"]
if not options.json:
banks_list.append(headers)
for bank in sorted(banks, key=lambda k: k['name']):
try:
bank = Bank(bank['name'], options=options, no_log=True)
bank.load_session(UpdateWorkflow.FLOW)
if bank.session is not None:
if bank.use_last_session and not bank.session.get_status(
Workflow.FLOW_OVER):
wf_status = bank.session.get('workflow_status')
if wf_status is None or not wf_status:
banks_list.append(bank.get_bank_release_info()['info'])
except Exception as e:
return (False, str(e))
if options.json:
msg = {'headers': headers, 'banks': banks_list}
return (True, msg)
return (True, tabulate(banks_list, headers="firstrow", tablefmt="psql"))
def test_directhttp_release(self):
b = Bank('directhttp')
res = b.update()
self.assertTrue(b.session.get('update'))
self.assertTrue(
os.path.exists(b.session.get_full_release_directory() +
'/flat/debian/README.html'))
def test_mix_stop_from_task(self):
"""
Get a first release, then fromscratch --stop-after, then restart from-task
"""
b = Bank('local')
b.update()
rel = b.session.get('release')
b2 = Bank('local')
b2.options.stop_after = 'download'
b2.options.fromscratch = True
res = b2.update()
self.assertTrue(b2.session.get('release') == rel + '__1')
b3 = Bank('local')
res = b3.update()
self.assertTrue(b3.session.get('release') == rel + '__1')
self.assertTrue(res)
def test_delete_old_dirs(self):
'''
Try updating 3 times, oldest dir should be removed
'''
b = Bank('local')
b.removeAll(True)
b = Bank('local')
b.update()
self.assertTrue(b.session.get('update'))
b.options.fromscratch = True
b.update()
self.assertTrue(b.session.get('update'))
self.assertTrue(len(b.bank['production']) == 2)
b.update()
self.assertTrue(b.session.get('update'))
# one new dir, but olders must be deleted
self.assertTrue(len(b.bank['production']) == 2)
def test_new_session(self):
'''
Checks an empty session is created
'''
b = Bank('alu')
b.load_session(UpdateWorkflow.FLOW)
for key in b.session._session['status'].keys():
self.assertFalse(b.session.get_status(key))
def test_extract_release_from_file_content(self):
b = Bank('local')
b.load_session(UpdateWorkflow.FLOW)
b.session.config.set('release.file', 'test_100\.txt')
b.session.config.set('release.regexp', 'Release\s*(\d+)')
w = UpdateWorkflow(b)
w.wf_release()
self.assertTrue(b.session.get('release') == '103')
def test_remoterelease_check(self):
b = Bank('local')
b.load_session(ReleaseCheckWorkflow.FLOW)
b.session.config.set('release.file', 'test_(\d+)\.txt')
b.session.config.set('release.regexp', '')
workflow = ReleaseCheckWorkflow(b)
res = workflow.start()
remoterelease = b.session.get('remoterelease')
self.assertTrue(remoterelease == '100')
def test_stats(self):
b = Bank('local')
b.update()
rel = b.session.get('release')
stats = Bank.get_banks_disk_usage()
self.assertTrue(stats[0]['size'] > 0)
for release in stats[0]['releases']:
if release['name'] == rel:
self.assertTrue(release['size'] > 0)
def biomaj_owner(options):
'''
Bank ownership management
'''
if not options.bank:
return (False, "Bank option is missing")
bank = Bank(options.bank, options=options, no_log=True)
bank.set_owner(options.owner)
return (True, None)
def test_update_hardlinks(self):
"""
Update a bank twice with hard links. Files copied from previous release
must be links.
"""
b = Bank('local')
b.config.set('keep.old.version', '3')
b.config.set('use_hardlinks', '1')
# Create a file in bank dir (which is the source dir) so we can manipulate
# it. The pattern is taken into account by the bank configuration.
# Note that this file is created in the source tree so we remove it after
# or if this test fails in between.
tmp_remote_file = b.config.get('remote.dir') + 'test.safe_to_del'
if os.path.exists(tmp_remote_file):
os.remove(tmp_remote_file)
open(tmp_remote_file, "w")
# First update
b.update()
self.assertTrue(b.session.get('update'))
old_release = b.session.get_full_release_directory()
# Touch tmp_remote_file to force update. We set the date to tomorrow so we
# are sure that a new release will be detected.
tomorrow = time.time(
) + 3660 * 24 # 3660s for safety (leap second, etc.)
os.utime(tmp_remote_file, (tomorrow, tomorrow))
# Second update
try:
b.update()
self.assertTrue(b.session.get('update'))
new_release = b.session.get_full_release_directory()
# Test that files in both releases are links to the the same file.
# We can't use tmp_remote_file because it's the source of update and we
# can't use test.fasta.gz because it is uncompressed and then not the
# same file.
for f in ['test2.fasta', 'test_100.txt']:
file_old_release = os.path.join(old_release, 'flat', f)
file_new_release = os.path.join(new_release, 'flat', f)
try:
self.assertTrue(
os.path.samefile(file_old_release, file_new_release))
except AssertionError:
msg = "In %s: copy worked but hardlinks were not used." % self.id(
)
logging.info(msg)
# Test that no links are done for tmp_remote_file
file_old_release = os.path.join(old_release, 'flat',
'test.safe_to_del')
file_new_release = os.path.join(new_release, 'flat',
'test.safe_to_del')
self.assertFalse(
os.path.samefile(file_old_release, file_new_release))
except Exception:
raise
finally:
# Remove file
if os.path.exists(tmp_remote_file):
os.remove(tmp_remote_file)
def test_remove_session(self):
b = Bank('alu')
for i in range(1, 5):
s = Session('alu', self.config, UpdateWorkflow.FLOW)
s._session['status'][Workflow.FLOW_INIT] = True
b.session = s
b.save_session()
self.assertTrue(len(b.bank['sessions']) == 4)
b.remove_session(b.session.get('id'))
self.assertTrue(len(b.bank['sessions']) == 3)
def biomaj_check(options):
'''
Check bank properties
'''
if not options.bank:
return (False, "Bank name is missing")
bank = Bank(options.bank, options=options, no_log=True)
msg = options.bank + " check: " + str(bank.check()) + "\n"
return (True, msg)
def test_delete_old_dirs_with_freeze(self):
"""
Try updating 3 times, oldest dir should be removed but not freezed releases
"""
b = Bank('local')
b.removeAll(True)
b = Bank('local')
b.update()
b.freeze(b.session.get('release'))
self.assertTrue(b.session.get('update'))
b.options.fromscratch = True
b.update()
b.freeze(b.session.get('release'))
self.assertTrue(b.session.get('update'))
self.assertTrue(len(b.bank['production']) == 2)
b.update()
self.assertTrue(b.session.get('update'))
# one new dir, but olders must be deleted
self.assertTrue(len(b.bank['production']) == 3)
def test_no_update(self):
'''
Try updating twice, at second time, bank should not be updated
'''
b = Bank('local')
b.update()
self.assertTrue(b.session.get('update'))
b.update()
self.assertFalse(b.session.get('update'))
self.assertFalse(b.session.get_status(Workflow.FLOW_POSTPROCESS))
def test_get_release(self):
'''
Get release
'''
b = Bank('alu')
b.load_session(UpdateWorkflow.FLOW)
res = b.update()
self.assertTrue(b.session.get('update'))
self.assertTrue(res)
self.assertTrue(b.session._session['release'] is not None)
def test_computed_nofile(self):
b = Bank('computed2')
b.load_session(UpdateWorkflow.FLOW)
b.session.config.set('protocol', 'none')
b.session.config.set('sub1.files.move', 'flat/test_.*')
res = b.update(True)
self.assertTrue(res)
self.assertTrue(
os.path.exists(b.session.get_full_release_directory() +
'/sub1/flat/test_100.txt'))
def test_search(self):
b = Bank('localprocess')
b.update()
search_res = Bank.search(['blast'], [])
self.assertTrue(len(search_res) == 1)
search_res = Bank.search([], ['nucleic'])
self.assertTrue(len(search_res) == 1)
search_res = Bank.search(['blast'], ['nucleic'])
self.assertTrue(len(search_res) == 1)
search_res = Bank.search(['blast'], ['proteic'])
self.assertTrue(len(search_res) == 0)
def biomaj_unpublish(options):
'''
Unpublish a bank
'''
if not options.bank:
return (False, "Bank name is missing")
bmaj = Bank(options.bank, options=options, no_log=True)
bmaj.load_session()
bmaj.unpublish()
return (True, None)
def biomaj_visibility(options):
'''
Bank visibility management
'''
if not options.bank:
return (False, "Bank option is missing")
if options.visibility not in ['public', 'private']:
return (False, "Valid values are public|private")
bank = Bank(options.bank, options=options, no_log=True)
bank.set_visibility(options.visibility)
return (True, "Do not forget to update accordingly the visibility.default parameter in the configuration file")
def test_fromscratch_update(self):
'''
Try updating twice, at second time, bank should be updated (force with fromscratc)
'''
b = Bank('local')
b.update()
self.assertTrue(b.session.get('update'))
sess = b.session.get('release')
b.options.fromscratch = True
b.update()
self.assertTrue(b.session.get('update'))
self.assertEqual(b.session.get('release'), sess + '__1')
def biomaj_daemon_bank_isowner(bank):
'''
Checks that logged user is admin or owner of the bank
'''
(http_code, options, error) = daemon_api_auth(request)
if error:
abort(http_code, error)
bank_log = Bank(bank, options=options, no_log=True)
if bank_log.is_owner():
return jsonify({'is_admin_or_owner': True})
else:
return jsonify({'is_admin_or_owner': False})
def biomaj_show(options, config):
'''
show bank details
'''
if not options.bank:
return (False, "Bank option is required")
bank = Bank(options.bank, options=options, no_log=True)
results = []
headers = ["Name", "Release", "Format(s)", "Type(s)", "Tag(s)", "File(s)"]
if not options.json:
results.append(headers)
current = None
fformat = None
if 'current' in bank.bank and bank.bank['current']:
current = bank.bank['current']
for prod in bank.bank['production']:
include = True
release = prod['release']
if current == prod['session']:
release += ' (current)'
if options.release and (prod['release'] != options.release
and prod['prod_dir'] != options.release):
include = False
if include:
session = bank.get_session_from_release(prod['release'])
formats = session['formats']
afiles = []
atags = []
atypes = []
for fformat in list(formats.keys()):
for elt in formats[fformat]:
if options.json:
atypes.append(elt['types'])
else:
atypes.append(','.join(elt['types']))
for tag in list(elt['tags'].keys()):
atags.append(elt['tags'][tag])
for eltfile in elt['files']:
afiles.append(eltfile)
if options.json:
results.append([
bank.bank['name'], release, fformat, atypes, atags, afiles
])
msg = {'details': results, 'headers': headers}
else:
results.append([
bank.bank['name'], release, fformat, ','.join(atypes),
','.join(atags), ','.join(afiles)
])
msg = tabulate(results, headers="firstrow", tablefmt="grid")
return (True, msg)
def biomaj_freeze(options):
'''
freeze a bank
'''
if not options.bank:
return (False, "Bank name is missing")
if not options.release:
return (False, "Bank release is missing")
bmaj = Bank(options.bank, options=options)
res = bmaj.freeze(options.release)
if not res:
return (False, 'Failed to freeze the bank release')
return (True, None)
def test_publish(self):
'''
Update a bank, then publish it
'''
b = Bank('local')
b.update()
current_link = os.path.join(b.config.get('data.dir'),
b.config.get('dir.version'), 'current')
self.assertFalse(os.path.exists(current_link))
self.assertTrue(b.bank['current'] is None)
b.publish()
self.assertTrue(os.path.exists(current_link))
self.assertTrue(b.bank['current'] == b.session._session['id'])
def test_freeze(self):
b = Bank('local')
b.update()
rel = b.session.get('release')
b.freeze(rel)
prod = b.get_production(rel)
self.assertTrue(prod['freeze'] == True)
res = b.remove(rel)
self.assertTrue(res == False)
b.unfreeze(rel)
prod = b.get_production(rel)
self.assertTrue(prod['freeze'] == False)
res = b.remove(rel)
self.assertTrue(res == True)
def test_multi(self):
b = Bank('multi')
res = b.update()
with open(
os.path.join(b.session.get_full_release_directory(),
'flat/test1.json'), 'r') as content_file:
content = content_file.read()
my_json = json.loads(content)
self.assertTrue(my_json['args']['key1'] == 'value1')
with open(
os.path.join(b.session.get_full_release_directory(),
'flat/test2.json'), 'r') as content_file:
content = content_file.read()
my_json = json.loads(content)
self.assertTrue(my_json['form']['key1'] == 'value1')
def setUp(self):
self.utils = UtilsForTest()
curdir = os.path.dirname(os.path.realpath(__file__))
BiomajConfig.load_config(self.utils.global_properties,
allow_user_config=False)
#Delete all banks
b = Bank('local')
b.banks.remove({})
self.config = BiomajConfig('local')
data_dir = self.config.get('data.dir')
lock_file = os.path.join(data_dir, 'local.lock')
if os.path.exists(lock_file):
os.remove(lock_file)
def test_postprocesses_exec_again(self):
'''
Execute once, set a status to false, check that False processes are executed
'''
b = Bank('localprocess')
pfactory = PostProcessFactory(b)
pfactory.run()
self.assertTrue(pfactory.blocks['BLOCK1']['META0']['PROC0'])
self.assertTrue(pfactory.blocks['BLOCK2']['META1']['PROC1'])
self.assertTrue(pfactory.blocks['BLOCK2']['META1']['PROC2'])
blocks = copy.deepcopy(pfactory.blocks)
blocks['BLOCK2']['META1']['PROC2'] = False
pfactory2 = PostProcessFactory(b, blocks)
pfactory2.run()
self.assertTrue(pfactory2.blocks['BLOCK2']['META1']['PROC2'])
def biomaj_remove_pending(options):
'''
Remove pending releases
'''
if not options.bank:
return (False, "Bank name is missing")
bmaj = Bank(options.bank, options=options, no_log=True)
if bmaj.is_locked():
return (False, 'Bank is locked due to an other action')
if not options.proxy:
res = bmaj.remove_pending(options.release)
return (res, '')
res = biomaj_remove_pending_request(options)
if not res:
return (False, 'Failed to send removal request')
return (True, 'Request sent')
