def test_removeAll(self): b = Bank('local') b.update() b.removeAll() self.assertFalse(os.path.exists(b.get_data_dir())) bdb = b.banks.find_one({'name': b.name}) self.assertTrue(bdb is None)
def test_computed_ref_release(self): b = Bank('computed2') res = b.update(True) self.assertTrue(b.session.get('update')) b2 = Bank('computed2') res = b2.update(True) self.assertFalse(b2.session.get('update'))
def test_update_hardlinks(self): """ Update a bank twice with hard links^. Files copied from previous release must be links. """ b = Bank('local') b.config.set('keep.old.version', '3') b.config.set('use_hardlinks', '1') # First update b.update() self.assertTrue(b.session.get('update')) old_release = b.session.get_full_release_directory() # Update test.fasta to force update (not that this file is modified in the # source tree) remote_file = b.session.config.get('remote.dir') + 'test.fasta.gz' stat = os.stat(remote_file) one_day = 3600 * 24 os.utime(remote_file, (stat.st_atime + one_day, stat.st_atime + one_day)) # Second update b.update() self.assertTrue(b.session.get('update')) new_release = b.session.get_full_release_directory() # Test that test2.fasta in both release are the same file (we can't use # test.fasta because it is uncompressed and then not the same file) file_old_release = os.path.join(old_release, 'flat', 'test2.fasta') file_new_release = os.path.join(new_release, 'flat', 'test2.fasta') try: self.assertTrue(os.path.samefile(file_old_release, file_new_release)) except AssertionError: msg = "In %s: copy worked but hardlinks were not used." % self.id() logging.info(msg) # Restore date (otherwise repeated tests fail) os.utime(remote_file, (stat.st_atime, stat.st_atime))
def test_update_hardlinks(self): """ Update a bank twice with hard links. Files copied from previous release must be links. """ b = Bank('local') b.config.set('keep.old.version', '3') b.config.set('use_hardlinks', '1') # Create a file in bank dir (which is the source dir) so we can manipulate # it. The pattern is taken into account by the bank configuration. # Note that this file is created in the source tree so we remove it after # or if this test fails in between. tmp_remote_file = b.config.get('remote.dir') + 'test.safe_to_del' if os.path.exists(tmp_remote_file): os.remove(tmp_remote_file) open(tmp_remote_file, "w") # First update b.update() self.assertTrue(b.session.get('update')) old_release = b.session.get_full_release_directory() # Touch tmp_remote_file to force update. We set the date to tomorrow so we # are sure that a new release will be detected. tomorrow = time.time( ) + 3660 * 24 # 3660s for safety (leap second, etc.) os.utime(tmp_remote_file, (tomorrow, tomorrow)) # Second update try: b.update() self.assertTrue(b.session.get('update')) new_release = b.session.get_full_release_directory() # Test that files in both releases are links to the the same file. # We can't use tmp_remote_file because it's the source of update and we # can't use test.fasta.gz because it is uncompressed and then not the # same file. for f in ['test2.fasta', 'test_100.txt']: file_old_release = os.path.join(old_release, 'flat', f) file_new_release = os.path.join(new_release, 'flat', f) try: self.assertTrue( os.path.samefile(file_old_release, file_new_release)) except AssertionError: msg = "In %s: copy worked but hardlinks were not used." % self.id( ) logging.info(msg) # Test that no links are done for tmp_remote_file file_old_release = os.path.join(old_release, 'flat', 'test.safe_to_del') file_new_release = os.path.join(new_release, 'flat', 'test.safe_to_del') self.assertFalse( os.path.samefile(file_old_release, file_new_release)) except Exception: raise finally: # Remove file if os.path.exists(tmp_remote_file): os.remove(tmp_remote_file)
def test_stats(self): b = Bank('local') b.update() rel = b.session.get('release') stats = Bank.get_banks_disk_usage() self.assertTrue(stats[0]['size']>0) for release in stats[0]['releases']: if release['name'] == rel: self.assertTrue(release['size']>0)
def test_stats(self): b = Bank('local') b.update() rel = b.session.get('release') stats = Bank.get_banks_disk_usage() self.assertTrue(stats[0]['size'] > 0) for release in stats[0]['releases']: if release['name'] == rel: self.assertTrue(release['size'] > 0)
def test_stats(self): b = Bank("local") b.update() rel = b.session.get("release") stats = Bank.get_banks_disk_usage() self.assertTrue(stats[0]["size"] > 0) for release in stats[0]["releases"]: if release["name"] == rel: self.assertTrue(release["size"] > 0)
def test_reupdate_from_task_wrong_release(self): b = Bank("local") b.options.stop_after = "download" b.update() self.assertFalse(b.session.get_status("postprocess")) b2 = Bank("local") b2.options.from_task = "postprocess" b2.options.release = "wrongrelease" res = b2.update() self.assertFalse(res)
def test_reupdate_from_task_error(self): b = Bank("local") b.options.stop_after = "check" b.update() self.assertFalse(b.session.get_status("postprocess")) b2 = Bank("local") b2.options.from_task = "postprocess" b2.options.release = b.session.get("release") res = b2.update() self.assertFalse(res)
def test_no_update(self): """ Try updating twice, at second time, bank should not be updated """ b = Bank("local") b.update() self.assertTrue(b.session.get("update")) b.update() self.assertFalse(b.session.get("update")) self.assertFalse(b.session.get_status(Workflow.FLOW_POSTPROCESS))
def test_reupdate_from_task_wrong_release(self): b = Bank('local') b.options.stop_after = 'download' b.update() self.assertFalse(b.session.get_status('postprocess')) b2 = Bank('local') b2.options.from_task = 'postprocess' b2.options.release = 'wrongrelease' res = b2.update() self.assertFalse(res)
def test_computed(self): b = Bank('computed') res = b.update(True) self.assertTrue(res) self.assertTrue(os.path.exists(b.session.get_full_release_directory()+'/sub1/flat/test_100.txt')) self.assertTrue(b.session.get('update')) # Check that, with depends non updated, bank is not updated itself nextb = Bank('computed') res = nextb.update(True) self.assertFalse(nextb.session.get('update'))
def test_reupdate_from_task_error(self): b = Bank('local') b.options.stop_after = 'check' b.update() self.assertFalse(b.session.get_status('postprocess')) b2 = Bank('local') b2.options.from_task = 'postprocess' b2.options.release = b.session.get('release') res = b2.update() self.assertFalse(res)
def test_no_update(self): ''' Try updating twice, at second time, bank should not be updated ''' b = Bank('local') b.update() self.assertTrue(b.session.get('update')) b.update() self.assertFalse(b.session.get('update')) self.assertFalse(b.session.get_status(Workflow.FLOW_POSTPROCESS))
def test_reupdate_from_task(self): b = Bank("local") b.options.stop_after = "download" b.update() self.assertFalse(b.session.get_status("postprocess")) b2 = Bank("local") b2.options.from_task = "postprocess" b2.options.release = b.session.get("release") b2.update() self.assertTrue(b2.session.get_status("postprocess")) self.assertEqual(b.session.get_full_release_directory(), b2.session.get_full_release_directory())
def test_search(self): b = Bank('localprocess') b.update() search_res = Bank.search(['blast'],[]) self.assertTrue(len(search_res)==1) search_res = Bank.search([],['nucleic']) self.assertTrue(len(search_res)==1) search_res = Bank.search(['blast'],['nucleic']) self.assertTrue(len(search_res)==1) search_res = Bank.search(['blast'],['proteic']) self.assertTrue(len(search_res)==0)
def test_search(self): b = Bank("localprocess") b.update() search_res = Bank.search(["blast"], []) self.assertTrue(len(search_res) == 1) search_res = Bank.search([], ["nucleic"]) self.assertTrue(len(search_res) == 1) search_res = Bank.search(["blast"], ["nucleic"]) self.assertTrue(len(search_res) == 1) search_res = Bank.search(["blast"], ["proteic"]) self.assertTrue(len(search_res) == 0)
def test_search(self): b = Bank('localprocess') b.update() search_res = Bank.search(['blast'], []) self.assertTrue(len(search_res) == 1) search_res = Bank.search([], ['nucleic']) self.assertTrue(len(search_res) == 1) search_res = Bank.search(['blast'], ['nucleic']) self.assertTrue(len(search_res) == 1) search_res = Bank.search(['blast'], ['proteic']) self.assertTrue(len(search_res) == 0)
def test_reupdate_from_task(self): b = Bank('local') b.options.stop_after = 'download' b.update() self.assertFalse(b.session.get_status('postprocess')) b2 = Bank('local') b2.options.from_task = 'postprocess' b2.options.release = b.session.get('release') b2.update() self.assertTrue(b2.session.get_status('postprocess')) self.assertEqual(b.session.get_full_release_directory(), b2.session.get_full_release_directory())
def test_remove(self): ''' test removal of a production dir ''' b = Bank('local') b.update() self.assertTrue(os.path.exists(b.session.get_full_release_directory())) self.assertTrue(len(b.bank['production'])==1) b.remove(b.session.get('release')) self.assertFalse(os.path.exists(b.session.get_full_release_directory())) b = Bank('local') self.assertTrue(len(b.bank['production'])==0)
def test_fromscratch_update(self): """ Try updating twice, at second time, bank should be updated (force with fromscratc) """ b = Bank("local") b.update() self.assertTrue(b.session.get("update")) sess = b.session.get("release") b.options.fromscratch = True b.update() self.assertTrue(b.session.get("update")) self.assertEqual(b.session.get("release"), sess + "__1")
def test_fromscratch_update(self): ''' Try updating twice, at second time, bank should be updated (force with fromscratc) ''' b = Bank('local') b.update() self.assertTrue(b.session.get('update')) sess = b.session.get('release') b.options.fromscratch = True b.update() self.assertTrue(b.session.get('update')) self.assertEqual(b.session.get('release'), sess+'__1')
def test_computed(self): b = Bank('computed') res = b.update(True) self.assertTrue(res) self.assertTrue( os.path.exists(b.session.get_full_release_directory() + '/sub1/flat/test_100.txt')) self.assertTrue(b.session.get('update')) # Check that, with depends non updated, bank is not updated itself nextb = Bank('computed') res = nextb.update(True) self.assertFalse(nextb.session.get('update'))
def test_publish(self): """ Update a bank, then publish it """ b = Bank("local") b.update() current_link = os.path.join(b.config.get("data.dir"), b.config.get("dir.version"), "current") self.assertFalse(os.path.exists(current_link)) self.assertTrue(b.bank["current"] is None) b.publish() self.assertTrue(os.path.exists(current_link)) self.assertTrue(b.bank["current"] == b.session._session["id"])
def test_remove(self): """ test removal of a production dir """ b = Bank("local") b.update() self.assertTrue(os.path.exists(b.session.get_full_release_directory())) self.assertTrue(len(b.bank["production"]) == 1) b.remove(b.session.get("release")) self.assertFalse(os.path.exists(b.session.get_full_release_directory())) b = Bank("local") self.assertTrue(len(b.bank["production"]) == 0)
def test_fromscratch_update(self): ''' Try updating twice, at second time, bank should be updated (force with fromscratc) ''' b = Bank('local') b.update() self.assertTrue(b.session.get('update')) sess = b.session.get('release') b.options.fromscratch = True b.update() self.assertTrue(b.session.get('update')) self.assertEqual(b.session.get('release'), sess + '__1')
def test_postprocess_wrong_process_name(self): """If a wrong process name is given, update returns False and prints an error message""" b = Bank('local') b.options.stop_after = 'download' b.update() self.assertFalse(b.session.get_status('postprocess')) b2 = Bank('local') b2.options.from_task = 'postprocess' b2.options.release = b.session.get('release') b2.options.process = 'fake' self.assertFalse(b2.update()) self.assertFalse(b2.session.get_status('postprocess')) self.assertEqual(b.session.get_full_release_directory(), b2.session.get_full_release_directory())
def test_remove(self): ''' test removal of a production dir ''' b = Bank('local') b.update() self.assertTrue(os.path.exists(b.session.get_full_release_directory())) self.assertTrue(len(b.bank['production']) == 1) b.remove(b.session.get('release')) self.assertFalse(os.path.exists( b.session.get_full_release_directory())) b = Bank('local') self.assertTrue(len(b.bank['production']) == 0)
def test_publish(self): ''' Update a bank, then publish it ''' b = Bank('local') b.update() current_link = os.path.join(b.config.get('data.dir'), b.config.get('dir.version'), 'current') self.assertFalse(os.path.exists(current_link)) self.assertTrue(b.bank['current'] is None) b.publish() self.assertTrue(os.path.exists(current_link)) self.assertTrue(b.bank['current'] == b.session._session['id'])
def test_clean_old_sessions(self): """ Checks a session is used if present """ b = Bank("local") for i in range(1, 5): s = Session("alu", self.config, UpdateWorkflow.FLOW) s._session["status"][Workflow.FLOW_INIT] = True b.session = s b.save_session() b2 = Bank("local") b2.update() b2.clean_old_sessions() self.assertTrue(len(b2.bank["sessions"]) == 1)
def test_freeze(self): b = Bank('local') b.update() rel = b.session.get('release') b.freeze(rel) prod = b.get_production(rel) self.assertTrue(prod['freeze'] == True) res = b.remove(rel) self.assertTrue(res == False) b.unfreeze(rel) prod = b.get_production(rel) self.assertTrue(prod['freeze'] == False) res = b.remove(rel) self.assertTrue(res == True)
def test_owner(self): ''' test ACL with owner ''' b = Bank('local') res = b.update() self.assertTrue(res) b.set_owner('sample') b2 = Bank('local') try: res = b2.update() self.fail('not owner, should not be allowed') except Exception as e: pass
def test_clean_old_sessions(self): ''' Checks a session is used if present ''' b = Bank('local') for i in range(1,5): s = Session('alu', self.config, UpdateWorkflow.FLOW) s._session['status'][Workflow.FLOW_INIT] = True b.session = s b.save_session() b2 = Bank('local') b2.update() b2.clean_old_sessions() self.assertTrue(len(b2.bank['sessions']) == 1)
def test_freeze(self): b = Bank("local") b.update() rel = b.session.get("release") b.freeze(rel) prod = b.get_production(rel) self.assertTrue(prod["freeze"] == True) res = b.remove(rel) self.assertTrue(res == False) b.unfreeze(rel) prod = b.get_production(rel) self.assertTrue(prod["freeze"] == False) res = b.remove(rel) self.assertTrue(res == True)
def test_clean_old_sessions(self): ''' Checks a session is used if present ''' b = Bank('local') for i in range(1, 5): s = Session('alu', self.config, UpdateWorkflow.FLOW) s._session['status'][Workflow.FLOW_INIT] = True b.session = s b.save_session() b2 = Bank('local') b2.update() b2.clean_old_sessions() self.assertTrue(len(b2.bank['sessions']) == 1)
def test_owner(self): """ test ACL with owner """ b = Bank("local") res = b.update() self.assertTrue(res) b.set_owner("sample") b2 = Bank("local") try: res = b2.update() self.fail("not owner, should not be allowed") except Exception as e: pass
def test_mix_stop_from_task4(self): ''' Get a first release, then fromscratch --stop-after, then restart from-task ''' b = Bank('local') b.update() rel = b.session.get('release') b2 = Bank('local') b2.options.stop_before = 'download' b2.options.fromscratch = True res = b2.update() b3 = Bank('local') b3.options.from_task = 'postprocess' res = b3.update() self.assertFalse(res)
def execute(self, options): ''' Execute update or remove command ''' start_time = datetime.datetime.now() start_time = time.mktime(start_time.timetuple()) is_ok = None is_updated = False action = None try: options.no_log = False if options.update: action = 'update' self.__start_action(options.bank, action) bmaj = Bank(options.bank, options) self.logger.debug('Log file: ' + bmaj.config.log_file) is_ok = bmaj.update(depends=True) is_updated = bmaj.session.get('update') Notify.notifyBankAction(bmaj) self.__end_action() elif options.remove or options.removeall: action = 'remove' self.__start_action(options.bank, action) if options.removeall: bmaj = Bank(options.bank, options, no_log=True) print('Log file: ' + bmaj.config.log_file) is_ok = bmaj.removeAll(options.force) else: bmaj = Bank(options.bank, options) self.logger.debug('Log file: ' + bmaj.config.log_file) is_ok = bmaj.remove(options.release) Notify.notifyBankAction(bmaj) self.__end_action() elif options.removepending: bmaj = Bank(options.bank, options, no_log=True) bmaj.remove_pending(options.release) elif options.repair: action = 'repair' self.__start_action(options.bank, action) bmaj = Bank(options.bank, options) self.logger.debug('Log file: ' + bmaj.config.log_file) is_ok = bmaj.repair() is_updated = bmaj.session.get('update') Notify.notifyBankAction(bmaj) self.__end_action() except Exception as e: self.logger.exception('Exception: ' + str(e)) is_ok = False end_time = datetime.datetime.now() end_time = time.mktime(end_time.timetuple()) execution_time = end_time - start_time return { 'error': not is_ok, 'execution_time': execution_time, 'action': action, 'updated': is_updated }
def biomaj_bank_update(options): ''' Update a bank ''' if not options.bank: return (False, "Bank name is missing") banks = options.bank.split(',') gres = True msg = '' for bank in banks: options.bank = bank bmaj = Bank(bank, options=options, no_log=True) if bmaj.is_locked(): return (False, 'Bank is locked due to an other action') check_status = bmaj.check() if not check_status: msg += 'Skip bank ' + options.bank + ': wrong config\n' gres = False continue else: msg += 'Bank update request sent for ' + options.bank + '\n' if not options.proxy: res = bmaj.update(depends=True) return (res, '') res = biomaj_bank_update_request(options) if not res: msg += 'Failed to send update request for ' + options.bank + '\n' if not gres: return (False, msg) return (True, msg)
def test_mix_stop_from_task(self): ''' Get a first release, then fromscratch --stop-after, then restart from-task ''' b = Bank('local') b.update() rel = b.session.get('release') b2 = Bank('local') b2.options.stop_after = 'download' b2.options.fromscratch = True res = b2.update() self.assertTrue(b2.session.get('release') == rel + '__1') b3 = Bank('local') res = b3.update() self.assertTrue(b3.session.get('release') == rel + '__1') self.assertTrue(res)
def test_download_from_list(self): """ Use remote.list to define a list of files to download """ b = Bank('local') fd, file_path = tempfile.mkstemp() try: b.config.set('remote.list', file_path) with os.fdopen(fd, 'w') as tmp: tmp.write('[{"name": "test_100.txt", "root": "' + b.config.get('remote.dir') + '"}]') b.update() self.assertTrue(b.session.get('update')) finally: #os.remove(file_path) print(file_path)
def test_directhttp_release(self): b = Bank('directhttp') res = b.update() self.assertTrue(b.session.get('update')) self.assertTrue( os.path.exists(b.session.get_full_release_directory() + '/flat/debian/README.html'))
def execute(self, options): ''' List remote content ''' start_time = datetime.datetime.now() start_time = time.mktime(start_time.timetuple()) error = None try: options.no_log = False if options.update: bmaj = Bank(options.bank, options) self.logger.debug('Log file: ' + bmaj.config.log_file) error = bmaj.update(depends=True) Notify.notifyBankAction(bmaj) elif options.remove or options.removeall: if options.removeall: bmaj = Bank(options.bank, options, no_log=True) print('Log file: ' + bmaj.config.log_file) error = bmaj.removeAll(options.force) else: bmaj = Bank(options.bank, options) self.logger.debug('Log file: ' + bmaj.config.log_file) error = bmaj.remove(options.release) Notify.notifyBankAction(bmaj) except Exception as e: self.logger.exception('Exception: ' + str(e)) error = True end_time = datetime.datetime.now() end_time = time.mktime(end_time.timetuple()) execution_time = end_time - start_time return {'error': error, 'execution_time': execution_time}
def test_computed_nofile(self): b = Bank('computed2') b.load_session(UpdateWorkflow.FLOW) b.session.config.set('protocol', 'none') b.session.config.set('sub1.files.move', 'flat/test_.*') res = b.update(True) self.assertTrue(res) self.assertTrue( os.path.exists(b.session.get_full_release_directory() + '/sub1/flat/test_100.txt'))
def test_computed_ref_release(self): b = Bank('computed2') res = b.update(True) b2 = Bank('sub1') b2release = b2.bank['production'][len(b2.bank['production']) - 1]['release'] brelease = b.bank['production'][len(b.bank['production']) - 1]['release'] self.assertTrue(res) self.assertTrue(brelease == b2release)
def test_get_release(self): ''' Get release ''' b = Bank('alu') b.load_session(UpdateWorkflow.FLOW) res = b.update() self.assertTrue(b.session.get('update')) self.assertTrue(res) self.assertTrue(b.session._session['release'] is not None)
def test_postprocesses_restart_from_proc(self): b = Bank('localprocess') b.update() proc1file = os.path.join(b.session.get_full_release_directory(), 'proc1.txt') proc2file = os.path.join(b.session.get_full_release_directory(), 'proc2.txt') self.assertTrue(os.path.exists(proc1file)) self.assertTrue(os.path.exists(proc2file)) os.remove(proc1file) os.remove(proc2file) # Restart from postprocess, reexecute all processes b2 = Bank('localprocess') b2.options.from_task = 'postprocess' b2.options.release = b.session.get('release') b2.update() self.assertTrue(os.path.exists(proc1file)) self.assertTrue(os.path.exists(proc2file)) os.remove(proc1file) os.remove(proc2file) # Restart from postprocess, but at process PROC2 and following b3 = Bank('localprocess') b3.options.from_task = 'postprocess' b3.options.process = 'PROC2' b3.options.release = b.session.get('release') b3.update() #self.assertFalse(os.path.exists(proc1file)) self.assertTrue(os.path.exists(proc2file))
def test_multi(self): b = Bank('multi') res = b.update() with open( os.path.join(b.session.get_full_release_directory(), 'flat/test1.json'), 'r') as content_file: content = content_file.read() my_json = json.loads(content) self.assertTrue(my_json['args']['key1'] == 'value1') with open( os.path.join(b.session.get_full_release_directory(), 'flat/test2.json'), 'r') as content_file: content = content_file.read() my_json = json.loads(content) self.assertTrue(my_json['form']['key1'] == 'value1')
def test_release_control(self): ''' Try updating twice, at second time, modify one file (same date), bank should update ''' b = Bank('local') b.update() b.session.config.set('keep.old.version', '3') self.assertTrue(b.session.get('update')) remote_file = b.session.config.get('remote.dir') + 'test2.fasta' os.utime(remote_file, None) # Update test2.fasta and set release.control b.session.config.set('release.control', 'true') b.update() self.assertTrue(b.session.get('update')) b.update() self.assertFalse(b.session.get('update')) b.session.config.set('remote.files', '^test2.fasta') b.update() self.assertTrue(b.session.get('update'))