def test_removeAll(self):
b = Bank('local')
b.update()
b.removeAll()
self.assertFalse(os.path.exists(b.get_data_dir()))
bdb = b.banks.find_one({'name': b.name})
self.assertTrue(bdb is None)
def test_computed_ref_release(self):
b = Bank('computed2')
res = b.update(True)
self.assertTrue(b.session.get('update'))
b2 = Bank('computed2')
res = b2.update(True)
self.assertFalse(b2.session.get('update'))
def test_computed_ref_release(self):
b = Bank('computed2')
res = b.update(True)
self.assertTrue(b.session.get('update'))
b2 = Bank('computed2')
res = b2.update(True)
self.assertFalse(b2.session.get('update'))
def test_removeAll(self):
b = Bank('local')
b.update()
b.removeAll()
self.assertFalse(os.path.exists(b.get_data_dir()))
bdb = b.banks.find_one({'name': b.name})
self.assertTrue(bdb is None)
def test_update_hardlinks(self):
"""
Update a bank twice with hard links^. Files copied from previous release
must be links.
"""
b = Bank('local')
b.config.set('keep.old.version', '3')
b.config.set('use_hardlinks', '1')
# First update
b.update()
self.assertTrue(b.session.get('update'))
old_release = b.session.get_full_release_directory()
# Update test.fasta to force update (not that this file is modified in the
# source tree)
remote_file = b.session.config.get('remote.dir') + 'test.fasta.gz'
stat = os.stat(remote_file)
one_day = 3600 * 24
os.utime(remote_file, (stat.st_atime + one_day, stat.st_atime + one_day))
# Second update
b.update()
self.assertTrue(b.session.get('update'))
new_release = b.session.get_full_release_directory()
# Test that test2.fasta in both release are the same file (we can't use
# test.fasta because it is uncompressed and then not the same file)
file_old_release = os.path.join(old_release, 'flat', 'test2.fasta')
file_new_release = os.path.join(new_release, 'flat', 'test2.fasta')
try:
self.assertTrue(os.path.samefile(file_old_release, file_new_release))
except AssertionError:
msg = "In %s: copy worked but hardlinks were not used." % self.id()
logging.info(msg)
# Restore date (otherwise repeated tests fail)
os.utime(remote_file, (stat.st_atime, stat.st_atime))
def test_update_hardlinks(self):
"""
Update a bank twice with hard links. Files copied from previous release
must be links.
"""
b = Bank('local')
b.config.set('keep.old.version', '3')
b.config.set('use_hardlinks', '1')
# Create a file in bank dir (which is the source dir) so we can manipulate
# it. The pattern is taken into account by the bank configuration.
# Note that this file is created in the source tree so we remove it after
# or if this test fails in between.
tmp_remote_file = b.config.get('remote.dir') + 'test.safe_to_del'
if os.path.exists(tmp_remote_file):
os.remove(tmp_remote_file)
open(tmp_remote_file, "w")
# First update
b.update()
self.assertTrue(b.session.get('update'))
old_release = b.session.get_full_release_directory()
# Touch tmp_remote_file to force update. We set the date to tomorrow so we
# are sure that a new release will be detected.
tomorrow = time.time(
) + 3660 * 24 # 3660s for safety (leap second, etc.)
os.utime(tmp_remote_file, (tomorrow, tomorrow))
# Second update
try:
b.update()
self.assertTrue(b.session.get('update'))
new_release = b.session.get_full_release_directory()
# Test that files in both releases are links to the the same file.
# We can't use tmp_remote_file because it's the source of update and we
# can't use test.fasta.gz because it is uncompressed and then not the
# same file.
for f in ['test2.fasta', 'test_100.txt']:
file_old_release = os.path.join(old_release, 'flat', f)
file_new_release = os.path.join(new_release, 'flat', f)
try:
self.assertTrue(
os.path.samefile(file_old_release, file_new_release))
except AssertionError:
msg = "In %s: copy worked but hardlinks were not used." % self.id(
)
logging.info(msg)
# Test that no links are done for tmp_remote_file
file_old_release = os.path.join(old_release, 'flat',
'test.safe_to_del')
file_new_release = os.path.join(new_release, 'flat',
'test.safe_to_del')
self.assertFalse(
os.path.samefile(file_old_release, file_new_release))
except Exception:
raise
finally:
# Remove file
if os.path.exists(tmp_remote_file):
os.remove(tmp_remote_file)
def test_stats(self):
b = Bank('local')
b.update()
rel = b.session.get('release')
stats = Bank.get_banks_disk_usage()
self.assertTrue(stats[0]['size']>0)
for release in stats[0]['releases']:
if release['name'] == rel:
self.assertTrue(release['size']>0)
def test_stats(self):
b = Bank('local')
b.update()
rel = b.session.get('release')
stats = Bank.get_banks_disk_usage()
self.assertTrue(stats[0]['size'] > 0)
for release in stats[0]['releases']:
if release['name'] == rel:
self.assertTrue(release['size'] > 0)
def test_stats(self):
b = Bank("local")
b.update()
rel = b.session.get("release")
stats = Bank.get_banks_disk_usage()
self.assertTrue(stats[0]["size"] > 0)
for release in stats[0]["releases"]:
if release["name"] == rel:
self.assertTrue(release["size"] > 0)
def test_reupdate_from_task_wrong_release(self):
b = Bank("local")
b.options.stop_after = "download"
b.update()
self.assertFalse(b.session.get_status("postprocess"))
b2 = Bank("local")
b2.options.from_task = "postprocess"
b2.options.release = "wrongrelease"
res = b2.update()
self.assertFalse(res)
def test_reupdate_from_task_error(self):
b = Bank("local")
b.options.stop_after = "check"
b.update()
self.assertFalse(b.session.get_status("postprocess"))
b2 = Bank("local")
b2.options.from_task = "postprocess"
b2.options.release = b.session.get("release")
res = b2.update()
self.assertFalse(res)
def test_no_update(self):
"""
Try updating twice, at second time, bank should not be updated
"""
b = Bank("local")
b.update()
self.assertTrue(b.session.get("update"))
b.update()
self.assertFalse(b.session.get("update"))
self.assertFalse(b.session.get_status(Workflow.FLOW_POSTPROCESS))
def test_reupdate_from_task_wrong_release(self):
b = Bank('local')
b.options.stop_after = 'download'
b.update()
self.assertFalse(b.session.get_status('postprocess'))
b2 = Bank('local')
b2.options.from_task = 'postprocess'
b2.options.release = 'wrongrelease'
res = b2.update()
self.assertFalse(res)
def test_computed(self):
b = Bank('computed')
res = b.update(True)
self.assertTrue(res)
self.assertTrue(os.path.exists(b.session.get_full_release_directory()+'/sub1/flat/test_100.txt'))
self.assertTrue(b.session.get('update'))
# Check that, with depends non updated, bank is not updated itself
nextb = Bank('computed')
res = nextb.update(True)
self.assertFalse(nextb.session.get('update'))
def test_reupdate_from_task_error(self):
b = Bank('local')
b.options.stop_after = 'check'
b.update()
self.assertFalse(b.session.get_status('postprocess'))
b2 = Bank('local')
b2.options.from_task = 'postprocess'
b2.options.release = b.session.get('release')
res = b2.update()
self.assertFalse(res)
def test_no_update(self):
'''
Try updating twice, at second time, bank should not be updated
'''
b = Bank('local')
b.update()
self.assertTrue(b.session.get('update'))
b.update()
self.assertFalse(b.session.get('update'))
self.assertFalse(b.session.get_status(Workflow.FLOW_POSTPROCESS))
def test_reupdate_from_task_error(self):
b = Bank('local')
b.options.stop_after = 'check'
b.update()
self.assertFalse(b.session.get_status('postprocess'))
b2 = Bank('local')
b2.options.from_task = 'postprocess'
b2.options.release = b.session.get('release')
res = b2.update()
self.assertFalse(res)
def test_no_update(self):
'''
Try updating twice, at second time, bank should not be updated
'''
b = Bank('local')
b.update()
self.assertTrue(b.session.get('update'))
b.update()
self.assertFalse(b.session.get('update'))
self.assertFalse(b.session.get_status(Workflow.FLOW_POSTPROCESS))
def test_reupdate_from_task_wrong_release(self):
b = Bank('local')
b.options.stop_after = 'download'
b.update()
self.assertFalse(b.session.get_status('postprocess'))
b2 = Bank('local')
b2.options.from_task = 'postprocess'
b2.options.release = 'wrongrelease'
res = b2.update()
self.assertFalse(res)
def test_reupdate_from_task(self):
b = Bank("local")
b.options.stop_after = "download"
b.update()
self.assertFalse(b.session.get_status("postprocess"))
b2 = Bank("local")
b2.options.from_task = "postprocess"
b2.options.release = b.session.get("release")
b2.update()
self.assertTrue(b2.session.get_status("postprocess"))
self.assertEqual(b.session.get_full_release_directory(), b2.session.get_full_release_directory())
def test_search(self):
b = Bank('localprocess')
b.update()
search_res = Bank.search(['blast'],[])
self.assertTrue(len(search_res)==1)
search_res = Bank.search([],['nucleic'])
self.assertTrue(len(search_res)==1)
search_res = Bank.search(['blast'],['nucleic'])
self.assertTrue(len(search_res)==1)
search_res = Bank.search(['blast'],['proteic'])
self.assertTrue(len(search_res)==0)
def test_search(self):
b = Bank("localprocess")
b.update()
search_res = Bank.search(["blast"], [])
self.assertTrue(len(search_res) == 1)
search_res = Bank.search([], ["nucleic"])
self.assertTrue(len(search_res) == 1)
search_res = Bank.search(["blast"], ["nucleic"])
self.assertTrue(len(search_res) == 1)
search_res = Bank.search(["blast"], ["proteic"])
self.assertTrue(len(search_res) == 0)
def test_search(self):
b = Bank('localprocess')
b.update()
search_res = Bank.search(['blast'], [])
self.assertTrue(len(search_res) == 1)
search_res = Bank.search([], ['nucleic'])
self.assertTrue(len(search_res) == 1)
search_res = Bank.search(['blast'], ['nucleic'])
self.assertTrue(len(search_res) == 1)
search_res = Bank.search(['blast'], ['proteic'])
self.assertTrue(len(search_res) == 0)
def test_reupdate_from_task(self):
b = Bank('local')
b.options.stop_after = 'download'
b.update()
self.assertFalse(b.session.get_status('postprocess'))
b2 = Bank('local')
b2.options.from_task = 'postprocess'
b2.options.release = b.session.get('release')
b2.update()
self.assertTrue(b2.session.get_status('postprocess'))
self.assertEqual(b.session.get_full_release_directory(), b2.session.get_full_release_directory())
def test_reupdate_from_task(self):
b = Bank('local')
b.options.stop_after = 'download'
b.update()
self.assertFalse(b.session.get_status('postprocess'))
b2 = Bank('local')
b2.options.from_task = 'postprocess'
b2.options.release = b.session.get('release')
b2.update()
self.assertTrue(b2.session.get_status('postprocess'))
self.assertEqual(b.session.get_full_release_directory(),
b2.session.get_full_release_directory())
def test_remove(self):
'''
test removal of a production dir
'''
b = Bank('local')
b.update()
self.assertTrue(os.path.exists(b.session.get_full_release_directory()))
self.assertTrue(len(b.bank['production'])==1)
b.remove(b.session.get('release'))
self.assertFalse(os.path.exists(b.session.get_full_release_directory()))
b = Bank('local')
self.assertTrue(len(b.bank['production'])==0)
def test_fromscratch_update(self):
"""
Try updating twice, at second time, bank should be updated (force with fromscratc)
"""
b = Bank("local")
b.update()
self.assertTrue(b.session.get("update"))
sess = b.session.get("release")
b.options.fromscratch = True
b.update()
self.assertTrue(b.session.get("update"))
self.assertEqual(b.session.get("release"), sess + "__1")
def test_fromscratch_update(self):
'''
Try updating twice, at second time, bank should be updated (force with fromscratc)
'''
b = Bank('local')
b.update()
self.assertTrue(b.session.get('update'))
sess = b.session.get('release')
b.options.fromscratch = True
b.update()
self.assertTrue(b.session.get('update'))
self.assertEqual(b.session.get('release'), sess+'__1')
def test_computed(self):
b = Bank('computed')
res = b.update(True)
self.assertTrue(res)
self.assertTrue(
os.path.exists(b.session.get_full_release_directory() +
'/sub1/flat/test_100.txt'))
self.assertTrue(b.session.get('update'))
# Check that, with depends non updated, bank is not updated itself
nextb = Bank('computed')
res = nextb.update(True)
self.assertFalse(nextb.session.get('update'))
def test_publish(self):
"""
Update a bank, then publish it
"""
b = Bank("local")
b.update()
current_link = os.path.join(b.config.get("data.dir"), b.config.get("dir.version"), "current")
self.assertFalse(os.path.exists(current_link))
self.assertTrue(b.bank["current"] is None)
b.publish()
self.assertTrue(os.path.exists(current_link))
self.assertTrue(b.bank["current"] == b.session._session["id"])
def test_remove(self):
"""
test removal of a production dir
"""
b = Bank("local")
b.update()
self.assertTrue(os.path.exists(b.session.get_full_release_directory()))
self.assertTrue(len(b.bank["production"]) == 1)
b.remove(b.session.get("release"))
self.assertFalse(os.path.exists(b.session.get_full_release_directory()))
b = Bank("local")
self.assertTrue(len(b.bank["production"]) == 0)
def test_fromscratch_update(self):
'''
Try updating twice, at second time, bank should be updated (force with fromscratc)
'''
b = Bank('local')
b.update()
self.assertTrue(b.session.get('update'))
sess = b.session.get('release')
b.options.fromscratch = True
b.update()
self.assertTrue(b.session.get('update'))
self.assertEqual(b.session.get('release'), sess + '__1')
def test_postprocess_wrong_process_name(self):
"""If a wrong process name is given, update returns False and prints an error message"""
b = Bank('local')
b.options.stop_after = 'download'
b.update()
self.assertFalse(b.session.get_status('postprocess'))
b2 = Bank('local')
b2.options.from_task = 'postprocess'
b2.options.release = b.session.get('release')
b2.options.process = 'fake'
self.assertFalse(b2.update())
self.assertFalse(b2.session.get_status('postprocess'))
self.assertEqual(b.session.get_full_release_directory(), b2.session.get_full_release_directory())
def test_remove(self):
'''
test removal of a production dir
'''
b = Bank('local')
b.update()
self.assertTrue(os.path.exists(b.session.get_full_release_directory()))
self.assertTrue(len(b.bank['production']) == 1)
b.remove(b.session.get('release'))
self.assertFalse(os.path.exists(
b.session.get_full_release_directory()))
b = Bank('local')
self.assertTrue(len(b.bank['production']) == 0)
def test_publish(self):
'''
Update a bank, then publish it
'''
b = Bank('local')
b.update()
current_link = os.path.join(b.config.get('data.dir'),
b.config.get('dir.version'), 'current')
self.assertFalse(os.path.exists(current_link))
self.assertTrue(b.bank['current'] is None)
b.publish()
self.assertTrue(os.path.exists(current_link))
self.assertTrue(b.bank['current'] == b.session._session['id'])
def test_clean_old_sessions(self):
"""
Checks a session is used if present
"""
b = Bank("local")
for i in range(1, 5):
s = Session("alu", self.config, UpdateWorkflow.FLOW)
s._session["status"][Workflow.FLOW_INIT] = True
b.session = s
b.save_session()
b2 = Bank("local")
b2.update()
b2.clean_old_sessions()
self.assertTrue(len(b2.bank["sessions"]) == 1)
def test_postprocess_wrong_process_name(self):
"""If a wrong process name is given, update returns False and prints an error message"""
b = Bank('local')
b.options.stop_after = 'download'
b.update()
self.assertFalse(b.session.get_status('postprocess'))
b2 = Bank('local')
b2.options.from_task = 'postprocess'
b2.options.release = b.session.get('release')
b2.options.process = 'fake'
self.assertFalse(b2.update())
self.assertFalse(b2.session.get_status('postprocess'))
self.assertEqual(b.session.get_full_release_directory(),
b2.session.get_full_release_directory())
def test_freeze(self):
b = Bank('local')
b.update()
rel = b.session.get('release')
b.freeze(rel)
prod = b.get_production(rel)
self.assertTrue(prod['freeze'] == True)
res = b.remove(rel)
self.assertTrue(res == False)
b.unfreeze(rel)
prod = b.get_production(rel)
self.assertTrue(prod['freeze'] == False)
res = b.remove(rel)
self.assertTrue(res == True)
def test_owner(self):
'''
test ACL with owner
'''
b = Bank('local')
res = b.update()
self.assertTrue(res)
b.set_owner('sample')
b2 = Bank('local')
try:
res = b2.update()
self.fail('not owner, should not be allowed')
except Exception as e:
pass
def test_clean_old_sessions(self):
'''
Checks a session is used if present
'''
b = Bank('local')
for i in range(1,5):
s = Session('alu', self.config, UpdateWorkflow.FLOW)
s._session['status'][Workflow.FLOW_INIT] = True
b.session = s
b.save_session()
b2 = Bank('local')
b2.update()
b2.clean_old_sessions()
self.assertTrue(len(b2.bank['sessions']) == 1)
def test_publish(self):
'''
Update a bank, then publish it
'''
b = Bank('local')
b.update()
current_link = os.path.join(b.config.get('data.dir'),
b.config.get('dir.version'),
'current')
self.assertFalse(os.path.exists(current_link))
self.assertTrue(b.bank['current'] is None)
b.publish()
self.assertTrue(os.path.exists(current_link))
self.assertTrue(b.bank['current'] == b.session._session['id'])
def test_freeze(self):
b = Bank("local")
b.update()
rel = b.session.get("release")
b.freeze(rel)
prod = b.get_production(rel)
self.assertTrue(prod["freeze"] == True)
res = b.remove(rel)
self.assertTrue(res == False)
b.unfreeze(rel)
prod = b.get_production(rel)
self.assertTrue(prod["freeze"] == False)
res = b.remove(rel)
self.assertTrue(res == True)
def test_freeze(self):
b = Bank('local')
b.update()
rel = b.session.get('release')
b.freeze(rel)
prod = b.get_production(rel)
self.assertTrue(prod['freeze'] == True)
res = b.remove(rel)
self.assertTrue(res == False)
b.unfreeze(rel)
prod = b.get_production(rel)
self.assertTrue(prod['freeze'] == False)
res = b.remove(rel)
self.assertTrue(res == True)
def test_clean_old_sessions(self):
'''
Checks a session is used if present
'''
b = Bank('local')
for i in range(1, 5):
s = Session('alu', self.config, UpdateWorkflow.FLOW)
s._session['status'][Workflow.FLOW_INIT] = True
b.session = s
b.save_session()
b2 = Bank('local')
b2.update()
b2.clean_old_sessions()
self.assertTrue(len(b2.bank['sessions']) == 1)
def test_owner(self):
'''
test ACL with owner
'''
b = Bank('local')
res = b.update()
self.assertTrue(res)
b.set_owner('sample')
b2 = Bank('local')
try:
res = b2.update()
self.fail('not owner, should not be allowed')
except Exception as e:
pass
def test_owner(self):
"""
test ACL with owner
"""
b = Bank("local")
res = b.update()
self.assertTrue(res)
b.set_owner("sample")
b2 = Bank("local")
try:
res = b2.update()
self.fail("not owner, should not be allowed")
except Exception as e:
pass
def test_mix_stop_from_task4(self):
'''
Get a first release, then fromscratch --stop-after, then restart from-task
'''
b = Bank('local')
b.update()
rel = b.session.get('release')
b2 = Bank('local')
b2.options.stop_before = 'download'
b2.options.fromscratch = True
res = b2.update()
b3 = Bank('local')
b3.options.from_task = 'postprocess'
res = b3.update()
self.assertFalse(res)
def test_mix_stop_from_task4(self):
'''
Get a first release, then fromscratch --stop-after, then restart from-task
'''
b = Bank('local')
b.update()
rel = b.session.get('release')
b2 = Bank('local')
b2.options.stop_before = 'download'
b2.options.fromscratch = True
res = b2.update()
b3 = Bank('local')
b3.options.from_task = 'postprocess'
res = b3.update()
self.assertFalse(res)
def execute(self, options):
'''
Execute update or remove command
'''
start_time = datetime.datetime.now()
start_time = time.mktime(start_time.timetuple())
is_ok = None
is_updated = False
action = None
try:
options.no_log = False
if options.update:
action = 'update'
self.__start_action(options.bank, action)
bmaj = Bank(options.bank, options)
self.logger.debug('Log file: ' + bmaj.config.log_file)
is_ok = bmaj.update(depends=True)
is_updated = bmaj.session.get('update')
Notify.notifyBankAction(bmaj)
self.__end_action()
elif options.remove or options.removeall:
action = 'remove'
self.__start_action(options.bank, action)
if options.removeall:
bmaj = Bank(options.bank, options, no_log=True)
print('Log file: ' + bmaj.config.log_file)
is_ok = bmaj.removeAll(options.force)
else:
bmaj = Bank(options.bank, options)
self.logger.debug('Log file: ' + bmaj.config.log_file)
is_ok = bmaj.remove(options.release)
Notify.notifyBankAction(bmaj)
self.__end_action()
elif options.removepending:
bmaj = Bank(options.bank, options, no_log=True)
bmaj.remove_pending(options.release)
elif options.repair:
action = 'repair'
self.__start_action(options.bank, action)
bmaj = Bank(options.bank, options)
self.logger.debug('Log file: ' + bmaj.config.log_file)
is_ok = bmaj.repair()
is_updated = bmaj.session.get('update')
Notify.notifyBankAction(bmaj)
self.__end_action()
except Exception as e:
self.logger.exception('Exception: ' + str(e))
is_ok = False
end_time = datetime.datetime.now()
end_time = time.mktime(end_time.timetuple())
execution_time = end_time - start_time
return {
'error': not is_ok,
'execution_time': execution_time,
'action': action,
'updated': is_updated
}
def biomaj_bank_update(options):
'''
Update a bank
'''
if not options.bank:
return (False, "Bank name is missing")
banks = options.bank.split(',')
gres = True
msg = ''
for bank in banks:
options.bank = bank
bmaj = Bank(bank, options=options, no_log=True)
if bmaj.is_locked():
return (False, 'Bank is locked due to an other action')
check_status = bmaj.check()
if not check_status:
msg += 'Skip bank ' + options.bank + ': wrong config\n'
gres = False
continue
else:
msg += 'Bank update request sent for ' + options.bank + '\n'
if not options.proxy:
res = bmaj.update(depends=True)
return (res, '')
res = biomaj_bank_update_request(options)
if not res:
msg += 'Failed to send update request for ' + options.bank + '\n'
if not gres:
return (False, msg)
return (True, msg)
def test_mix_stop_from_task(self):
'''
Get a first release, then fromscratch --stop-after, then restart from-task
'''
b = Bank('local')
b.update()
rel = b.session.get('release')
b2 = Bank('local')
b2.options.stop_after = 'download'
b2.options.fromscratch = True
res = b2.update()
self.assertTrue(b2.session.get('release') == rel + '__1')
b3 = Bank('local')
res = b3.update()
self.assertTrue(b3.session.get('release') == rel + '__1')
self.assertTrue(res)
def test_download_from_list(self):
"""
Use remote.list to define a list of files to download
"""
b = Bank('local')
fd, file_path = tempfile.mkstemp()
try:
b.config.set('remote.list', file_path)
with os.fdopen(fd, 'w') as tmp:
tmp.write('[{"name": "test_100.txt", "root": "' +
b.config.get('remote.dir') + '"}]')
b.update()
self.assertTrue(b.session.get('update'))
finally:
#os.remove(file_path)
print(file_path)
def test_directhttp_release(self):
b = Bank('directhttp')
res = b.update()
self.assertTrue(b.session.get('update'))
self.assertTrue(
os.path.exists(b.session.get_full_release_directory() +
'/flat/debian/README.html'))
def execute(self, options):
'''
List remote content
'''
start_time = datetime.datetime.now()
start_time = time.mktime(start_time.timetuple())
error = None
try:
options.no_log = False
if options.update:
bmaj = Bank(options.bank, options)
self.logger.debug('Log file: ' + bmaj.config.log_file)
error = bmaj.update(depends=True)
Notify.notifyBankAction(bmaj)
elif options.remove or options.removeall:
if options.removeall:
bmaj = Bank(options.bank, options, no_log=True)
print('Log file: ' + bmaj.config.log_file)
error = bmaj.removeAll(options.force)
else:
bmaj = Bank(options.bank, options)
self.logger.debug('Log file: ' + bmaj.config.log_file)
error = bmaj.remove(options.release)
Notify.notifyBankAction(bmaj)
except Exception as e:
self.logger.exception('Exception: ' + str(e))
error = True
end_time = datetime.datetime.now()
end_time = time.mktime(end_time.timetuple())
execution_time = end_time - start_time
return {'error': error, 'execution_time': execution_time}
def test_computed_nofile(self):
b = Bank('computed2')
b.load_session(UpdateWorkflow.FLOW)
b.session.config.set('protocol', 'none')
b.session.config.set('sub1.files.move', 'flat/test_.*')
res = b.update(True)
self.assertTrue(res)
self.assertTrue(
os.path.exists(b.session.get_full_release_directory() +
'/sub1/flat/test_100.txt'))
def test_computed_ref_release(self):
b = Bank('computed2')
res = b.update(True)
b2 = Bank('sub1')
b2release = b2.bank['production'][len(b2.bank['production']) -
1]['release']
brelease = b.bank['production'][len(b.bank['production']) -
1]['release']
self.assertTrue(res)
self.assertTrue(brelease == b2release)
def test_get_release(self):
'''
Get release
'''
b = Bank('alu')
b.load_session(UpdateWorkflow.FLOW)
res = b.update()
self.assertTrue(b.session.get('update'))
self.assertTrue(res)
self.assertTrue(b.session._session['release'] is not None)
def test_postprocesses_restart_from_proc(self):
b = Bank('localprocess')
b.update()
proc1file = os.path.join(b.session.get_full_release_directory(),
'proc1.txt')
proc2file = os.path.join(b.session.get_full_release_directory(),
'proc2.txt')
self.assertTrue(os.path.exists(proc1file))
self.assertTrue(os.path.exists(proc2file))
os.remove(proc1file)
os.remove(proc2file)
# Restart from postprocess, reexecute all processes
b2 = Bank('localprocess')
b2.options.from_task = 'postprocess'
b2.options.release = b.session.get('release')
b2.update()
self.assertTrue(os.path.exists(proc1file))
self.assertTrue(os.path.exists(proc2file))
os.remove(proc1file)
os.remove(proc2file)
# Restart from postprocess, but at process PROC2 and following
b3 = Bank('localprocess')
b3.options.from_task = 'postprocess'
b3.options.process = 'PROC2'
b3.options.release = b.session.get('release')
b3.update()
#self.assertFalse(os.path.exists(proc1file))
self.assertTrue(os.path.exists(proc2file))
def test_multi(self):
b = Bank('multi')
res = b.update()
with open(
os.path.join(b.session.get_full_release_directory(),
'flat/test1.json'), 'r') as content_file:
content = content_file.read()
my_json = json.loads(content)
self.assertTrue(my_json['args']['key1'] == 'value1')
with open(
os.path.join(b.session.get_full_release_directory(),
'flat/test2.json'), 'r') as content_file:
content = content_file.read()
my_json = json.loads(content)
self.assertTrue(my_json['form']['key1'] == 'value1')
def test_release_control(self):
'''
Try updating twice, at second time, modify one file (same date),
bank should update
'''
b = Bank('local')
b.update()
b.session.config.set('keep.old.version', '3')
self.assertTrue(b.session.get('update'))
remote_file = b.session.config.get('remote.dir') + 'test2.fasta'
os.utime(remote_file, None)
# Update test2.fasta and set release.control
b.session.config.set('release.control', 'true')
b.update()
self.assertTrue(b.session.get('update'))
b.update()
self.assertFalse(b.session.get('update'))
b.session.config.set('remote.files', '^test2.fasta')
b.update()
self.assertTrue(b.session.get('update'))
