def test_properties_override(self):
BiomajConfig.load_config(self.utils.global_properties,
allow_user_config=False)
config = BiomajConfig('local')
ldap_host = config.get('ldap.host')
self.assertTrue(ldap_host == 'localhost')
os.environ['BIOMAJ_LDAP_HOST'] = 'someserver'
ldap_host = config.get('ldap.host')
self.assertTrue(ldap_host == 'someserver')
def tearDown(self):
# Delete lock files
for bank_name in self.BANKS:
config = BiomajConfig(bank_name)
data_dir = config.get('data.dir')
lock_file = os.path.join(data_dir, 'local.lock')
if os.path.exists(lock_file):
os.remove(lock_file)
self.utils.clean()
def setUp(self):
self.utils = UtilsForTest()
BiomajConfig.load_config(self.utils.global_properties,
allow_user_config=False)
# Clean banks used in tests
for bank_name in self.BANKS:
# Delete all releases
b = Bank(bank_name)
b.banks.remove({})
# Delete lock files
config = BiomajConfig(bank_name)
data_dir = config.get('data.dir')
lock_file = os.path.join(data_dir, 'local.lock')
if os.path.exists(lock_file):
os.remove(lock_file)
class TestBiomajSetup(unittest.TestCase):
def setUp(self):
self.utils = UtilsForTest()
curdir = os.path.dirname(os.path.realpath(__file__))
BiomajConfig.load_config(self.utils.global_properties,
allow_user_config=False)
# Delete all banks
b = Bank('alu')
b.banks.remove({})
self.config = BiomajConfig('alu')
data_dir = self.config.get('data.dir')
lock_file = os.path.join(data_dir, 'alu.lock')
if os.path.exists(lock_file):
os.remove(lock_file)
def tearDown(self):
data_dir = self.config.get('data.dir')
lock_file = os.path.join(data_dir, 'alu.lock')
if os.path.exists(lock_file):
os.remove(lock_file)
self.utils.clean()
def test_new_bank(self):
"""
Checks bank init
"""
b = Bank('alu')
def test_new_session(self):
"""
Checks an empty session is created
"""
b = Bank('alu')
b.load_session(UpdateWorkflow.FLOW)
for key in b.session._session['status'].keys():
self.assertFalse(b.session.get_status(key))
def test_session_reload_notover(self):
"""
Checks a session is used if present
"""
b = Bank('alu')
for i in range(1, 5):
s = Session('alu', self.config, UpdateWorkflow.FLOW)
s._session['status'][Workflow.FLOW_INIT] = True
b.session = s
b.save_session()
b = Bank('alu')
b.load_session(UpdateWorkflow.FLOW)
self.assertTrue(b.session.get_status(Workflow.FLOW_INIT))
def test_clean_old_sessions(self):
"""
Checks a session is used if present
"""
b = Bank('local')
for i in range(1, 5):
s = Session('alu', self.config, UpdateWorkflow.FLOW)
s._session['status'][Workflow.FLOW_INIT] = True
b.session = s
b.save_session()
b2 = Bank('local')
b2.update()
b2.clean_old_sessions()
self.assertTrue(len(b2.bank['sessions']) == 1)
def test_session_reload_over(self):
"""
Checks a session if is not over
"""
b = Bank('alu')
for i in range(1, 5):
s = Session('alu', self.config, UpdateWorkflow.FLOW)
s._session['status'][Workflow.FLOW_INIT] = True
s._session['status'][Workflow.FLOW_OVER] = True
b.session = s
b.save_session()
b = Bank('alu')
b.load_session(UpdateWorkflow.FLOW)
self.assertFalse(b.session.get_status(Workflow.FLOW_INIT))
def test_bank_list(self):
b1 = Bank('alu')
b2 = Bank('local')
banks = Bank.list()
self.assertTrue(len(banks) == 2)
@attr('test')
@attr('network')
def test_get_release(self):
"""
Get release
"""
b = Bank('alu')
b.load_session(UpdateWorkflow.FLOW)
res = b.update()
self.assertTrue(b.session.get('update'))
self.assertTrue(res)
self.assertTrue(b.session._session['release'] is not None)
@attr('network')
def test_remove_session(self):
b = Bank('alu')
for i in range(1, 5):
s = Session('alu', self.config, UpdateWorkflow.FLOW)
s._session['status'][Workflow.FLOW_INIT] = True
b.session = s
b.save_session()
self.assertTrue(len(b.bank['sessions']) == 4)
b.remove_session(b.session.get('id'))
self.assertTrue(len(b.bank['sessions']) == 3)
@attr('process')
def test_postprocesses_setup(self):
b = Bank('localprocess')
pfactory = PostProcessFactory(b)
pfactory.run(True)
self.assertTrue(len(pfactory.threads_tasks[0]) == 2)
self.assertTrue(len(pfactory.threads_tasks[1]) == 1)
@attr('process')
def test_postprocesses_exec_again(self):
"""
Execute once, set a status to false, check that False processes are executed
"""
b = Bank('localprocess')
pfactory = PostProcessFactory(b)
pfactory.run()
self.assertTrue(pfactory.blocks['BLOCK1']['META0']['PROC0'])
self.assertTrue(pfactory.blocks['BLOCK2']['META1']['PROC1'])
self.assertTrue(pfactory.blocks['BLOCK2']['META1']['PROC2'])
blocks = copy.deepcopy(pfactory.blocks)
blocks['BLOCK2']['META1']['PROC2'] = False
pfactory2 = PostProcessFactory(b, blocks)
pfactory2.run()
self.assertTrue(pfactory2.blocks['BLOCK2']['META1']['PROC2'])
@attr('process')
def test_preprocesses(self):
b = Bank('localprocess')
pfactory = PreProcessFactory(b)
pfactory.run()
self.assertTrue(pfactory.meta_status['META0']['PROC0'])
@attr('process')
def test_removeprocesses(self):
b = Bank('localprocess')
pfactory = RemoveProcessFactory(b)
pfactory.run()
self.assertTrue(pfactory.meta_status['META0']['PROC0'])
def test_dependencies_list(self):
b = Bank('computed')
deps = b.get_dependencies()
self.assertTrue(len(deps) == 2)
class TestBiomajHTTPDownload(unittest.TestCase):
"""
Test HTTP downloader
"""
def setUp(self):
self.utils = UtilsForTest()
BiomajConfig.load_config(self.utils.global_properties,
allow_user_config=False)
self.config = BiomajConfig('testhttp')
self.http_parse = HTTPParse(
self.config.get('http.parse.dir.line'),
self.config.get('http.parse.file.line'),
int(self.config.get('http.group.dir.name')),
int(self.config.get('http.group.dir.date')),
int(self.config.get('http.group.file.name')),
int(self.config.get('http.group.file.date')),
self.config.get('http.group.file.date_format', None),
int(self.config.get('http.group.file.size')))
def tearDown(self):
self.utils.clean()
def test_http_list(self):
httpd = CurlDownload('http', 'ftp2.fr.debian.org', '/debian/dists/',
self.http_parse)
(file_list, dir_list) = httpd.list()
httpd.close()
self.assertTrue(len(file_list) == 1)
def test_http_list_dateregexp(self):
#self.http_parse.file_date_format = "%%d-%%b-%%Y %%H:%%M"
self.http_parse.file_date_format = "%%Y-%%m-%%d %%H:%%M"
httpd = CurlDownload('http', 'ftp2.fr.debian.org', '/debian/dists/',
self.http_parse)
(file_list, dir_list) = httpd.list()
httpd.close()
self.assertTrue(len(file_list) == 1)
def test_http_download_no_size(self):
self.http_parse = HTTPParse(
self.config.get('http.parse.dir.line'),
self.config.get('http.parse.file.line'),
int(self.config.get('http.group.dir.name')),
int(self.config.get('http.group.dir.date')),
int(self.config.get('http.group.file.name')),
int(self.config.get('http.group.file.date')),
self.config.get('http.group.file.date_format', None), -1)
self.http_parse.file_date_format = "%%Y-%%m-%%d %%H:%%M"
httpd = CurlDownload('http', 'ftp2.fr.debian.org', '/debian/dists/',
self.http_parse)
(file_list, dir_list) = httpd.list()
httpd.match([r'^README$'], file_list, dir_list)
httpd.download(self.utils.data_dir)
httpd.close()
self.assertTrue(len(httpd.files_to_download) == 1)
def test_http_download_no_date(self):
self.http_parse = HTTPParse(
self.config.get('http.parse.dir.line'),
self.config.get('http.parse.file.line'),
int(self.config.get('http.group.dir.name')),
int(self.config.get('http.group.dir.date')),
int(self.config.get('http.group.file.name')), -1,
self.config.get('http.group.file.date_format', None),
int(self.config.get('http.group.file.size')))
httpd = CurlDownload('http', 'ftp2.fr.debian.org', '/debian/dists/',
self.http_parse)
(file_list, dir_list) = httpd.list()
httpd.match([r'^README$'], file_list, dir_list)
httpd.download(self.utils.data_dir)
httpd.close()
self.assertTrue(len(httpd.files_to_download) == 1)
def test_http_download(self):
self.http_parse.file_date_format = "%%Y-%%m-%%d %%H:%%M"
httpd = CurlDownload('http', 'ftp2.fr.debian.org', '/debian/dists/',
self.http_parse)
(file_list, dir_list) = httpd.list()
print(str(file_list))
httpd.match([r'^README$'], file_list, dir_list)
httpd.download(self.utils.data_dir)
httpd.close()
self.assertTrue(len(httpd.files_to_download) == 1)
def test_http_download_in_subdir(self):
self.http_parse.file_date_format = "%%Y-%%m-%%d %%H:%%M"
httpd = CurlDownload('http', 'ftp2.fr.debian.org', '/debian/',
self.http_parse)
(file_list, dir_list) = httpd.list()
httpd.match([r'^dists/README$'], file_list, dir_list)
httpd.download(self.utils.data_dir)
httpd.close()
self.assertTrue(len(httpd.files_to_download) == 1)
class TestElastic(unittest.TestCase):
"""
test indexing and search
"""
def setUp(self):
BmajIndex.es = None
self.utils = UtilsForTest()
curdir = os.path.dirname(os.path.realpath(__file__))
BiomajConfig.load_config(self.utils.global_properties,
allow_user_config=False)
if BmajIndex.do_index == False:
self.skipTest(
"Skipping indexing tests due to elasticsearch not available")
# Delete all banks
b = Bank('local')
b.banks.remove({})
BmajIndex.delete_all_bank('local')
self.config = BiomajConfig('local')
data_dir = self.config.get('data.dir')
lock_file = os.path.join(data_dir, 'local.lock')
if os.path.exists(lock_file):
os.remove(lock_file)
def tearDown(self):
data_dir = self.config.get('data.dir')
lock_file = os.path.join(data_dir, 'local.lock')
if os.path.exists(lock_file):
os.remove(lock_file)
self.utils.clean()
BmajIndex.delete_all_bank('test')
def test_index(self):
BmajIndex.do_index = True
prod = {
"data_dir": "/tmp/test/data",
"formats": {
"fasta": [{
"files": ["fasta/chr1.fa", "fasta/chr2.fa"],
"types": ["nucleic"],
"tags": {
"organism": "hg19"
}
}],
"blast": [{
"files": ["blast/chr1/chr1db"],
"types": ["nucleic"],
"tags": {
"chr": "chr1",
"organism": "hg19"
}
}]
},
"freeze": False,
"session": 1416229253.930908,
"prod_dir": "alu-2003-11-26",
"release": "2003-11-26",
"types": ["nucleic"]
}
BmajIndex.add('test', prod, True)
query = {'query': {'match': {'bank': 'test'}}}
res = BmajIndex.search(query)
self.assertTrue(len(res) == 2)
def test_remove_all(self):
self.test_index()
query = {'query': {'match': {'bank': 'test'}}}
BmajIndex.delete_all_bank('test')
res = BmajIndex.search(query)
self.assertTrue(len(res) == 0)
class TestBiomajFunctional(unittest.TestCase):
def setUp(self):
self.utils = UtilsForTest()
curdir = os.path.dirname(os.path.realpath(__file__))
BiomajConfig.load_config(self.utils.global_properties, allow_user_config=False)
#Delete all banks
b = Bank('local')
b.banks.remove({})
self.config = BiomajConfig('local')
data_dir = self.config.get('data.dir')
lock_file = os.path.join(data_dir,'local.lock')
if os.path.exists(lock_file):
os.remove(lock_file)
def tearDown(self):
data_dir = self.config.get('data.dir')
lock_file = os.path.join(data_dir,'local.lock')
if os.path.exists(lock_file):
os.remove(lock_file)
self.utils.clean()
def test_extract_release_from_file_name(self):
b = Bank('local')
b.load_session(UpdateWorkflow.FLOW)
b.session.config.set('release.file', 'test_(\d+)\.txt')
b.session.config.set('release.regexp', '')
w = UpdateWorkflow(b)
w.wf_release()
self.assertTrue(b.session.get('release') == '100')
def test_remoterelease_check(self):
b = Bank('local')
b.load_session(ReleaseCheckWorkflow.FLOW)
b.session.config.set('release.file', 'test_(\d+)\.txt')
b.session.config.set('release.regexp', '')
workflow = ReleaseCheckWorkflow(b)
res = workflow.start()
remoterelease = b.session.get('remoterelease')
self.assertTrue(remoterelease == '100')
def test_extract_release_from_file_content(self):
b = Bank('local')
b.load_session(UpdateWorkflow.FLOW)
b.session.config.set('release.file', 'test_100\.txt')
b.session.config.set('release.regexp', 'Release\s*(\d+)')
w = UpdateWorkflow(b)
w.wf_release()
self.assertTrue(b.session.get('release') == '103')
def test_publish(self):
"""
Update a bank, then publish it
"""
b = Bank('local')
b.update()
current_link = os.path.join(b.config.get('data.dir'),
b.config.get('dir.version'),
'current')
self.assertFalse(os.path.exists(current_link))
self.assertTrue(b.bank['current'] is None)
b.publish()
self.assertTrue(os.path.exists(current_link))
self.assertTrue(b.bank['current'] == b.session._session['id'])
# Should test this on local downloader, changing 1 file to force update,
# else we would get same bank and there would be no update
def test_no_update(self):
"""
Try updating twice, at second time, bank should not be updated
"""
b = Bank('local')
b.update()
self.assertTrue(b.session.get('update'))
b.update()
self.assertFalse(b.session.get('update'))
self.assertFalse(b.session.get_status(Workflow.FLOW_POSTPROCESS))
@attr('remotelist')
def test_download_from_list(self):
"""
Use remote.list to define a list of files to download
"""
b = Bank('local')
fd, file_path = tempfile.mkstemp()
try:
b.config.set('remote.list', file_path)
with os.fdopen(fd, 'w') as tmp:
tmp.write('[{"name": "test_100.txt", "root": "' + b.config.get('remote.dir') + '"}]')
b.update()
self.assertTrue(b.session.get('update'))
finally:
#os.remove(file_path)
print(file_path)
@attr('release')
def test_release_control(self):
"""
Try updating twice, at second time, modify one file (same date),
bank should update
"""
b = Bank('local')
b.update()
b.session.config.set('keep.old.version', '3')
self.assertTrue(b.session.get('update'))
remote_file = b.session.config.get('remote.dir') + 'test2.fasta'
os.utime(remote_file, None)
# Update test2.fasta and set release.control
b.session.config.set('release.control', 'true')
b.update()
self.assertTrue(b.session.get('update'))
b.update()
self.assertFalse(b.session.get('update'))
b.session.config.set('copy.skip', '1')
b.session.config.set('remote.files', '^test2.fasta')
b.update()
self.assertTrue(b.session.get('update'))
def test_update_hardlinks(self):
"""
Update a bank twice with hard links^. Files copied from previous release
must be links.
"""
b = Bank('local')
b.config.set('keep.old.version', '3')
b.config.set('use_hardlinks', '1')
# First update
b.update()
self.assertTrue(b.session.get('update'))
old_release = b.session.get_full_release_directory()
# Update test.fasta to force update (not that this file is modified in the
# source tree)
remote_file = b.session.config.get('remote.dir') + 'test.fasta.gz'
stat = os.stat(remote_file)
one_day = 3600 * 24
os.utime(remote_file, (stat.st_atime + one_day, stat.st_atime + one_day))
# Second update
b.update()
self.assertTrue(b.session.get('update'))
new_release = b.session.get_full_release_directory()
# Test that test2.fasta in both release are the same file (we can't use
# test.fasta because it is uncompressed and then not the same file)
file_old_release = os.path.join(old_release, 'flat', 'test2.fasta')
file_new_release = os.path.join(new_release, 'flat', 'test2.fasta')
try:
self.assertTrue(os.path.samefile(file_old_release, file_new_release))
except AssertionError:
msg = "In %s: copy worked but hardlinks were not used." % self.id()
logging.info(msg)
# Restore date (otherwise repeated tests fail)
os.utime(remote_file, (stat.st_atime, stat.st_atime))
def test_fromscratch_update(self):
"""
Try updating twice, at second time, bank should be updated (force with fromscratc)
"""
b = Bank('local')
b.update()
self.assertTrue(b.session.get('update'))
sess = b.session.get('release')
b.options.fromscratch = True
b.update()
self.assertTrue(b.session.get('update'))
self.assertEqual(b.session.get('release'), sess+'__1')
def test_fromscratch_update_with_release(self):
"""
Try updating twice, at second time, bank should be updated (force with fromscratch)
Use case with release defined in release file
"""
b = Bank('local')
b.load_session(UpdateWorkflow.FLOW)
b.session.config.set('release.file', 'test_(\d+)\.txt')
b.session.config.set('release.regexp', '')
w = UpdateWorkflow(b)
w.wf_release()
self.assertTrue(b.session.get('release') == '100')
os.makedirs(b.session.get_full_release_directory())
w = UpdateWorkflow(b)
# Reset release
b.session.set('release', None)
w.options.fromscratch = True
w.wf_release()
self.assertTrue(b.session.get('release') == '100__1')
def test_mix_stop_from_task(self):
"""
Get a first release, then fromscratch --stop-after, then restart from-task
"""
b = Bank('local')
b.update()
rel = b.session.get('release')
b2 = Bank('local')
b2.options.stop_after = 'download'
b2.options.fromscratch = True
res = b2.update()
self.assertTrue(b2.session.get('release') == rel+'__1')
b3 = Bank('local')
res = b3.update()
self.assertTrue(b3.session.get('release') == rel+'__1')
self.assertTrue(res)
def test_mix_stop_from_task2(self):
"""
Get a first release, then fromscratch --stop-after, then restart from-task
"""
b = Bank('local')
b.update()
rel = b.session.get('release')
b2 = Bank('local')
b2.options.stop_after = 'download'
b2.options.fromscratch = True
res = b2.update()
self.assertTrue(b2.session.get('release') == rel+'__1')
b3 = Bank('local')
res = b3.update()
b2.options.from_task = 'download'
self.assertTrue(b3.session.get('release') == rel+'__1')
self.assertTrue(res)
def test_mix_stop_from_task3(self):
"""
Get a first release, then fromscratch --stop-after, then restart from-task
"""
b = Bank('local')
b.update()
rel = b.session.get('release')
b2 = Bank('local')
b2.options.stop_after = 'download'
b2.options.fromscratch = True
res = b2.update()
self.assertTrue(b2.session.get('release') == rel+'__1')
b3 = Bank('local')
res = b3.update()
b2.options.from_task = 'postprocess'
self.assertTrue(b3.session.get('release') == rel+'__1')
self.assertTrue(res)
def test_mix_stop_from_task4(self):
"""
Get a first release, then fromscratch --stop-after, then restart from-task
"""
b = Bank('local')
b.update()
rel = b.session.get('release')
b2 = Bank('local')
b2.options.stop_before = 'download'
b2.options.fromscratch = True
res = b2.update()
b3 = Bank('local')
b3.options.from_task = 'postprocess'
res = b3.update()
self.assertFalse(res)
def test_delete_old_dirs(self):
"""
Try updating 3 times, oldest dir should be removed
"""
b = Bank('local')
b.removeAll(True)
b = Bank('local')
b.update()
self.assertTrue(b.session.get('update'))
b.options.fromscratch = True
b.update()
self.assertTrue(b.session.get('update'))
self.assertTrue(len(b.bank['production']) == 2)
b.update()
self.assertTrue(b.session.get('update'))
# one new dir, but olders must be deleted
self.assertTrue(len(b.bank['production']) == 2)
def test_delete_old_dirs_with_freeze(self):
"""
Try updating 3 times, oldest dir should be removed but not freezed releases
"""
b = Bank('local')
b.removeAll(True)
b = Bank('local')
b.update()
b.freeze(b.session.get('release'))
self.assertTrue(b.session.get('update'))
b.options.fromscratch = True
b.update()
b.freeze(b.session.get('release'))
self.assertTrue(b.session.get('update'))
self.assertTrue(len(b.bank['production']) == 2)
b.update()
self.assertTrue(b.session.get('update'))
# one new dir, but olders must be deleted
self.assertTrue(len(b.bank['production']) == 3)
def test_removeAll(self):
b = Bank('local')
b.update()
b.removeAll()
self.assertFalse(os.path.exists(b.get_data_dir()))
bdb = b.banks.find_one({'name': b.name})
self.assertTrue(bdb is None)
def test_remove(self):
"""
test removal of a production dir
"""
b = Bank('local')
b.update()
self.assertTrue(os.path.exists(b.session.get_full_release_directory()))
self.assertTrue(len(b.bank['production'])==1)
b.remove(b.session.get('release'))
self.assertFalse(os.path.exists(b.session.get_full_release_directory()))
b = Bank('local')
self.assertTrue(len(b.bank['production'])==0)
def test_update_stop_after(self):
b = Bank('local')
b.options.stop_after = 'download'
b.update()
self.assertTrue(b.session.get_status('download'))
self.assertFalse(b.session.get_status('postprocess'))
def test_update_stop_before(self):
b = Bank('local')
b.options.stop_before = 'postprocess'
b.update()
self.assertTrue(b.session.get_status('download'))
self.assertFalse(b.session.get_status('postprocess'))
def test_reupdate_from_task(self):
b = Bank('local')
b.options.stop_after = 'download'
b.update()
self.assertFalse(b.session.get_status('postprocess'))
b2 = Bank('local')
b2.options.from_task = 'postprocess'
b2.options.release = b.session.get('release')
b2.update()
self.assertTrue(b2.session.get_status('postprocess'))
self.assertEqual(b.session.get_full_release_directory(), b2.session.get_full_release_directory())
def test_reupdate_from_task_error(self):
b = Bank('local')
b.options.stop_after = 'check'
b.update()
self.assertFalse(b.session.get_status('postprocess'))
b2 = Bank('local')
b2.options.from_task = 'postprocess'
b2.options.release = b.session.get('release')
res = b2.update()
self.assertFalse(res)
def test_reupdate_from_task_wrong_release(self):
b = Bank('local')
b.options.stop_after = 'download'
b.update()
self.assertFalse(b.session.get_status('postprocess'))
b2 = Bank('local')
b2.options.from_task = 'postprocess'
b2.options.release = 'wrongrelease'
res = b2.update()
self.assertFalse(res)
@attr('process')
def test_postprocesses_restart_from_proc(self):
b = Bank('localprocess')
b.update()
proc1file = os.path.join(b.session.get_full_release_directory(),'proc1.txt')
proc2file = os.path.join(b.session.get_full_release_directory(),'proc2.txt')
self.assertTrue(os.path.exists(proc1file))
self.assertTrue(os.path.exists(proc2file))
os.remove(proc1file)
os.remove(proc2file)
# Restart from postprocess, reexecute all processes
b2 = Bank('localprocess')
b2.options.from_task = 'postprocess'
b2.options.release = b.session.get('release')
b2.update()
self.assertTrue(os.path.exists(proc1file))
self.assertTrue(os.path.exists(proc2file))
os.remove(proc1file)
os.remove(proc2file)
# Restart from postprocess, but at process PROC2 and following
b3 = Bank('localprocess')
b3.options.from_task = 'postprocess'
b3.options.process = 'PROC2'
b3.options.release = b.session.get('release')
b3.update()
#self.assertFalse(os.path.exists(proc1file))
self.assertTrue(os.path.exists(proc2file))
@attr('process')
def test_postprocess_wrong_process_name(self):
"""If a wrong process name is given, update returns False and prints an error message"""
b = Bank('local')
b.options.stop_after = 'download'
b.update()
self.assertFalse(b.session.get_status('postprocess'))
b2 = Bank('local')
b2.options.from_task = 'postprocess'
b2.options.release = b.session.get('release')
b2.options.process = 'fake'
self.assertFalse(b2.update())
self.assertFalse(b2.session.get_status('postprocess'))
self.assertEqual(b.session.get_full_release_directory(), b2.session.get_full_release_directory())
def test_computed(self):
b = Bank('computed')
res = b.update(True)
self.assertTrue(res)
self.assertTrue(os.path.exists(b.session.get_full_release_directory()+'/sub1/flat/test_100.txt'))
self.assertTrue(b.session.get('update'))
# Check that, with depends non updated, bank is not updated itself
nextb = Bank('computed')
res = nextb.update(True)
self.assertFalse(nextb.session.get('update'))
@attr('nofile')
def test_computed_nofile(self):
b = Bank('computed2')
b.load_session(UpdateWorkflow.FLOW)
b.session.config.set('protocol', 'none')
b.session.config.set('sub1.files.move', 'flat/test_.*')
res = b.update(True)
self.assertTrue(res)
self.assertTrue(os.path.exists(b.session.get_full_release_directory()+'/sub1/flat/test_100.txt'))
def test_computed_ref_release(self):
b = Bank('computed2')
res = b.update(True)
b2 = Bank('sub1')
b2release = b2.bank['production'][len(b2.bank['production'])-1]['release']
brelease = b.bank['production'][len(b.bank['production'])-1]['release']
self.assertTrue(res)
self.assertTrue(brelease == b2release)
@attr('computed')
def test_computed_ref_release(self):
b = Bank('computed2')
res = b.update(True)
self.assertTrue(b.session.get('update'))
b2 = Bank('computed2')
res = b2.update(True)
self.assertFalse(b2.session.get('update'))
def test_computederror(self):
b = Bank('computederror')
res = b.update(True)
self.assertFalse(res)
self.assertTrue(b.session._session['depends']['sub2'])
self.assertFalse(b.session._session['depends']['error'])
@attr('directrelease')
def test_directhttp_release(self):
b = Bank('directhttp')
res = b.update()
self.assertTrue(b.session.get('update'))
self.assertTrue(os.path.exists(b.session.get_full_release_directory()+'/flat/debian/README.html'))
# print str(b.session.get('release'))
# print str(b.session.get('remoterelease'))
@attr('network')
def test_multi(self):
b = Bank('multi')
res = b.update()
with open(os.path.join(b.session.get_full_release_directory(),'flat/test1.json'), 'r') as content_file:
content = content_file.read()
my_json = json.loads(content)
self.assertTrue(my_json['args']['key1'] == 'value1')
with open(os.path.join(b.session.get_full_release_directory(),'flat/test2.json'), 'r') as content_file:
content = content_file.read()
my_json = json.loads(content)
self.assertTrue(my_json['form']['key1'] == 'value1')
def test_freeze(self):
b = Bank('local')
b.update()
rel = b.session.get('release')
b.freeze(rel)
prod = b.get_production(rel)
self.assertTrue(prod['freeze'] == True)
res = b.remove(rel)
self.assertTrue(res == False)
b.unfreeze(rel)
prod = b.get_production(rel)
self.assertTrue(prod['freeze'] == False)
res = b.remove(rel)
self.assertTrue(res == True)
def test_stats(self):
b = Bank('local')
b.update()
rel = b.session.get('release')
stats = Bank.get_banks_disk_usage()
self.assertTrue(stats[0]['size']>0)
for release in stats[0]['releases']:
if release['name'] == rel:
self.assertTrue(release['size']>0)
@attr('process')
def test_processes_meta_data(self):
b = Bank('localprocess')
b.update()
formats = b.session.get('formats')
self.assertTrue(len(formats['blast'])==2)
self.assertTrue(len(formats['test'][0]['files'])==3)
@attr('process')
def test_search(self):
b = Bank('localprocess')
b.update()
search_res = Bank.search(['blast'],[])
self.assertTrue(len(search_res)==1)
search_res = Bank.search([],['nucleic'])
self.assertTrue(len(search_res)==1)
search_res = Bank.search(['blast'],['nucleic'])
self.assertTrue(len(search_res)==1)
search_res = Bank.search(['blast'],['proteic'])
self.assertTrue(len(search_res)==0)
def test_owner(self):
"""
test ACL with owner
"""
b = Bank('local')
res = b.update()
self.assertTrue(res)
b.set_owner('sample')
b2 = Bank('local')
try:
res = b2.update()
self.fail('not owner, should not be allowed')
except Exception as e:
pass
class TestBiomajSetup(unittest.TestCase):
def setUp(self):
self.utils = UtilsForTest()
curdir = os.path.dirname(os.path.realpath(__file__))
BiomajConfig.load_config(self.utils.global_properties, allow_user_config=False)
# Delete all banks
b = Bank('alu')
b.banks.remove({})
self.config = BiomajConfig('alu')
data_dir = self.config.get('data.dir')
lock_file = os.path.join(data_dir,'alu.lock')
if os.path.exists(lock_file):
os.remove(lock_file)
def tearDown(self):
data_dir = self.config.get('data.dir')
lock_file = os.path.join(data_dir,'alu.lock')
if os.path.exists(lock_file):
os.remove(lock_file)
self.utils.clean()
def test_new_bank(self):
"""
Checks bank init
"""
b = Bank('alu')
def test_new_session(self):
"""
Checks an empty session is created
"""
b = Bank('alu')
b.load_session(UpdateWorkflow.FLOW)
for key in b.session._session['status'].keys():
self.assertFalse(b.session.get_status(key))
def test_session_reload_notover(self):
"""
Checks a session is used if present
"""
b = Bank('alu')
for i in range(1, 5):
s = Session('alu', self.config, UpdateWorkflow.FLOW)
s._session['status'][Workflow.FLOW_INIT] = True
b.session = s
b.save_session()
b = Bank('alu')
b.load_session(UpdateWorkflow.FLOW)
self.assertTrue(b.session.get_status(Workflow.FLOW_INIT))
def test_clean_old_sessions(self):
"""
Checks a session is used if present
"""
b = Bank('local')
for i in range(1,5):
s = Session('alu', self.config, UpdateWorkflow.FLOW)
s._session['status'][Workflow.FLOW_INIT] = True
b.session = s
b.save_session()
b2 = Bank('local')
b2.update()
b2.clean_old_sessions()
self.assertTrue(len(b2.bank['sessions']) == 1)
def test_session_reload_over(self):
"""
Checks a session if is not over
"""
b = Bank('alu')
for i in range(1,5):
s = Session('alu', self.config, UpdateWorkflow.FLOW)
s._session['status'][Workflow.FLOW_INIT] = True
s._session['status'][Workflow.FLOW_OVER] = True
b.session = s
b.save_session()
b = Bank('alu')
b.load_session(UpdateWorkflow.FLOW)
self.assertFalse(b.session.get_status(Workflow.FLOW_INIT))
def test_bank_list(self):
b1 = Bank('alu')
b2 = Bank('local')
banks = Bank.list()
self.assertTrue(len(banks) == 2)
@attr('test')
@attr('network')
def test_get_release(self):
"""
Get release
"""
b = Bank('alu')
b.load_session(UpdateWorkflow.FLOW)
res = b.update()
self.assertTrue(b.session.get('update'))
self.assertTrue(res)
self.assertTrue(b.session._session['release'] is not None)
def test_remove_session(self):
b = Bank('alu')
for i in range(1,5):
s = Session('alu', self.config, UpdateWorkflow.FLOW)
s._session['status'][Workflow.FLOW_INIT] = True
b.session = s
b.save_session()
self.assertTrue(len(b.bank['sessions'])==4)
b.remove_session(b.session.get('id'))
self.assertTrue(len(b.bank['sessions'])==3)
@attr('process')
def test_postprocesses_setup(self):
b = Bank('localprocess')
pfactory = PostProcessFactory(b)
pfactory.run(True)
self.assertTrue(len(pfactory.threads_tasks[0])==2)
self.assertTrue(len(pfactory.threads_tasks[1])==1)
@attr('process')
def test_postprocesses_exec_again(self):
"""
Execute once, set a status to false, check that False processes are executed
"""
b = Bank('localprocess')
pfactory = PostProcessFactory(b)
pfactory.run()
self.assertTrue(pfactory.blocks['BLOCK1']['META0']['PROC0'])
self.assertTrue(pfactory.blocks['BLOCK2']['META1']['PROC1'])
self.assertTrue(pfactory.blocks['BLOCK2']['META1']['PROC2'])
blocks = copy.deepcopy(pfactory.blocks)
blocks['BLOCK2']['META1']['PROC2'] = False
pfactory2 = PostProcessFactory(b, blocks)
pfactory2.run()
self.assertTrue(pfactory2.blocks['BLOCK2']['META1']['PROC2'])
@attr('process')
def test_preprocesses(self):
b = Bank('localprocess')
pfactory = PreProcessFactory(b)
pfactory.run()
self.assertTrue(pfactory.meta_status['META0']['PROC0'])
@attr('process')
def test_removeprocesses(self):
b = Bank('localprocess')
pfactory = RemoveProcessFactory(b)
pfactory.run()
self.assertTrue(pfactory.meta_status['META0']['PROC0'])
def test_dependencies_list(self):
b = Bank('computed')
deps = b.get_dependencies()
self.assertTrue(len(deps)==2)
class TestBiomajFunctional(unittest.TestCase):
def setUp(self):
self.utils = UtilsForTest()
curdir = os.path.dirname(os.path.realpath(__file__))
BiomajConfig.load_config(self.utils.global_properties,
allow_user_config=False)
#Delete all banks
b = Bank('local')
b.banks.remove({})
self.config = BiomajConfig('local')
data_dir = self.config.get('data.dir')
lock_file = os.path.join(data_dir, 'local.lock')
if os.path.exists(lock_file):
os.remove(lock_file)
def tearDown(self):
data_dir = self.config.get('data.dir')
lock_file = os.path.join(data_dir, 'local.lock')
if os.path.exists(lock_file):
os.remove(lock_file)
self.utils.clean()
def test_extract_release_from_file_name(self):
b = Bank('local')
b.load_session(UpdateWorkflow.FLOW)
b.session.config.set('release.file', 'test_(\d+)\.txt')
b.session.config.set('release.regexp', '')
w = UpdateWorkflow(b)
w.wf_release()
self.assertTrue(b.session.get('release') == '100')
def test_remoterelease_check(self):
b = Bank('local')
b.load_session(ReleaseCheckWorkflow.FLOW)
b.session.config.set('release.file', 'test_(\d+)\.txt')
b.session.config.set('release.regexp', '')
workflow = ReleaseCheckWorkflow(b)
res = workflow.start()
remoterelease = b.session.get('remoterelease')
self.assertTrue(remoterelease == '100')
def test_extract_release_from_file_content(self):
b = Bank('local')
b.load_session(UpdateWorkflow.FLOW)
b.session.config.set('release.file', 'test_100\.txt')
b.session.config.set('release.regexp', 'Release\s*(\d+)')
w = UpdateWorkflow(b)
w.wf_release()
self.assertTrue(b.session.get('release') == '103')
def test_publish(self):
"""
Update a bank, then publish it
"""
b = Bank('local')
b.update()
current_link = os.path.join(b.config.get('data.dir'),
b.config.get('dir.version'), 'current')
self.assertFalse(os.path.exists(current_link))
self.assertTrue(b.bank['current'] is None)
b.publish()
self.assertTrue(os.path.exists(current_link))
self.assertTrue(b.bank['current'] == b.session._session['id'])
# Should test this on local downloader, changing 1 file to force update,
# else we would get same bank and there would be no update
def test_no_update(self):
"""
Try updating twice, at second time, bank should not be updated
"""
b = Bank('local')
b.update()
self.assertTrue(b.session.get('update'))
b.update()
self.assertFalse(b.session.get('update'))
self.assertFalse(b.session.get_status(Workflow.FLOW_POSTPROCESS))
@attr('remotelist')
def test_download_from_list(self):
"""
Use remote.list to define a list of files to download
"""
b = Bank('local')
fd, file_path = tempfile.mkstemp()
try:
b.config.set('remote.list', file_path)
with os.fdopen(fd, 'w') as tmp:
tmp.write('[{"name": "test_100.txt", "root": "' +
b.config.get('remote.dir') + '"}]')
b.update()
self.assertTrue(b.session.get('update'))
finally:
#os.remove(file_path)
print(file_path)
@attr('release')
def test_release_control(self):
"""
Try updating twice, at second time, modify one file (same date),
bank should update
"""
b = Bank('local')
b.update()
b.session.config.set('keep.old.version', '3')
self.assertTrue(b.session.get('update'))
remote_file = b.session.config.get('remote.dir') + 'test2.fasta'
os.utime(remote_file, None)
# Update test2.fasta and set release.control
b.session.config.set('release.control', 'true')
b.update()
self.assertTrue(b.session.get('update'))
b.update()
self.assertFalse(b.session.get('update'))
b.session.config.set('copy.skip', '1')
b.session.config.set('remote.files', '^test2.fasta')
b.update()
self.assertTrue(b.session.get('update'))
def test_fromscratch_update(self):
"""
Try updating twice, at second time, bank should be updated (force with fromscratc)
"""
b = Bank('local')
b.update()
self.assertTrue(b.session.get('update'))
sess = b.session.get('release')
b.options.fromscratch = True
b.update()
self.assertTrue(b.session.get('update'))
self.assertEqual(b.session.get('release'), sess + '__1')
def test_fromscratch_update_with_release(self):
"""
Try updating twice, at second time, bank should be updated (force with fromscratch)
Use case with release defined in release file
"""
b = Bank('local')
b.load_session(UpdateWorkflow.FLOW)
b.session.config.set('release.file', 'test_(\d+)\.txt')
b.session.config.set('release.regexp', '')
w = UpdateWorkflow(b)
w.wf_release()
self.assertTrue(b.session.get('release') == '100')
os.makedirs(b.session.get_full_release_directory())
w = UpdateWorkflow(b)
# Reset release
b.session.set('release', None)
w.options.fromscratch = True
w.wf_release()
self.assertTrue(b.session.get('release') == '100__1')
def test_mix_stop_from_task(self):
"""
Get a first release, then fromscratch --stop-after, then restart from-task
"""
b = Bank('local')
b.update()
rel = b.session.get('release')
b2 = Bank('local')
b2.options.stop_after = 'download'
b2.options.fromscratch = True
res = b2.update()
self.assertTrue(b2.session.get('release') == rel + '__1')
b3 = Bank('local')
res = b3.update()
self.assertTrue(b3.session.get('release') == rel + '__1')
self.assertTrue(res)
def test_mix_stop_from_task2(self):
"""
Get a first release, then fromscratch --stop-after, then restart from-task
"""
b = Bank('local')
b.update()
rel = b.session.get('release')
b2 = Bank('local')
b2.options.stop_after = 'download'
b2.options.fromscratch = True
res = b2.update()
self.assertTrue(b2.session.get('release') == rel + '__1')
b3 = Bank('local')
res = b3.update()
b2.options.from_task = 'download'
self.assertTrue(b3.session.get('release') == rel + '__1')
self.assertTrue(res)
def test_mix_stop_from_task3(self):
"""
Get a first release, then fromscratch --stop-after, then restart from-task
"""
b = Bank('local')
b.update()
rel = b.session.get('release')
b2 = Bank('local')
b2.options.stop_after = 'download'
b2.options.fromscratch = True
res = b2.update()
self.assertTrue(b2.session.get('release') == rel + '__1')
b3 = Bank('local')
res = b3.update()
b2.options.from_task = 'postprocess'
self.assertTrue(b3.session.get('release') == rel + '__1')
self.assertTrue(res)
def test_mix_stop_from_task4(self):
"""
Get a first release, then fromscratch --stop-after, then restart from-task
"""
b = Bank('local')
b.update()
rel = b.session.get('release')
b2 = Bank('local')
b2.options.stop_before = 'download'
b2.options.fromscratch = True
res = b2.update()
b3 = Bank('local')
b3.options.from_task = 'postprocess'
res = b3.update()
self.assertFalse(res)
def test_delete_old_dirs(self):
"""
Try updating 3 times, oldest dir should be removed
"""
b = Bank('local')
b.removeAll(True)
b = Bank('local')
b.update()
self.assertTrue(b.session.get('update'))
b.options.fromscratch = True
b.update()
self.assertTrue(b.session.get('update'))
self.assertTrue(len(b.bank['production']) == 2)
b.update()
self.assertTrue(b.session.get('update'))
# one new dir, but olders must be deleted
self.assertTrue(len(b.bank['production']) == 2)
def test_delete_old_dirs_with_freeze(self):
"""
Try updating 3 times, oldest dir should be removed but not freezed releases
"""
b = Bank('local')
b.removeAll(True)
b = Bank('local')
b.update()
b.freeze(b.session.get('release'))
self.assertTrue(b.session.get('update'))
b.options.fromscratch = True
b.update()
b.freeze(b.session.get('release'))
self.assertTrue(b.session.get('update'))
self.assertTrue(len(b.bank['production']) == 2)
b.update()
self.assertTrue(b.session.get('update'))
# one new dir, but olders must be deleted
self.assertTrue(len(b.bank['production']) == 3)
def test_removeAll(self):
b = Bank('local')
b.update()
b.removeAll()
self.assertFalse(os.path.exists(b.get_data_dir()))
bdb = b.banks.find_one({'name': b.name})
self.assertTrue(bdb is None)
def test_remove(self):
"""
test removal of a production dir
"""
b = Bank('local')
b.update()
self.assertTrue(os.path.exists(b.session.get_full_release_directory()))
self.assertTrue(len(b.bank['production']) == 1)
b.remove(b.session.get('release'))
self.assertFalse(os.path.exists(
b.session.get_full_release_directory()))
b = Bank('local')
self.assertTrue(len(b.bank['production']) == 0)
def test_update_stop_after(self):
b = Bank('local')
b.options.stop_after = 'download'
b.update()
self.assertTrue(b.session.get_status('download'))
self.assertFalse(b.session.get_status('postprocess'))
def test_update_stop_before(self):
b = Bank('local')
b.options.stop_before = 'postprocess'
b.update()
self.assertTrue(b.session.get_status('download'))
self.assertFalse(b.session.get_status('postprocess'))
def test_reupdate_from_task(self):
b = Bank('local')
b.options.stop_after = 'download'
b.update()
self.assertFalse(b.session.get_status('postprocess'))
b2 = Bank('local')
b2.options.from_task = 'postprocess'
b2.options.release = b.session.get('release')
b2.update()
self.assertTrue(b2.session.get_status('postprocess'))
self.assertEqual(b.session.get_full_release_directory(),
b2.session.get_full_release_directory())
def test_reupdate_from_task_error(self):
b = Bank('local')
b.options.stop_after = 'check'
b.update()
self.assertFalse(b.session.get_status('postprocess'))
b2 = Bank('local')
b2.options.from_task = 'postprocess'
b2.options.release = b.session.get('release')
res = b2.update()
self.assertFalse(res)
def test_reupdate_from_task_wrong_release(self):
b = Bank('local')
b.options.stop_after = 'download'
b.update()
self.assertFalse(b.session.get_status('postprocess'))
b2 = Bank('local')
b2.options.from_task = 'postprocess'
b2.options.release = 'wrongrelease'
res = b2.update()
self.assertFalse(res)
@attr('process')
def test_postprocesses_restart_from_proc(self):
b = Bank('localprocess')
b.update()
proc1file = os.path.join(b.session.get_full_release_directory(),
'proc1.txt')
proc2file = os.path.join(b.session.get_full_release_directory(),
'proc2.txt')
self.assertTrue(os.path.exists(proc1file))
self.assertTrue(os.path.exists(proc2file))
os.remove(proc1file)
os.remove(proc2file)
# Restart from postprocess, reexecute all processes
b2 = Bank('localprocess')
b2.options.from_task = 'postprocess'
b2.options.release = b.session.get('release')
b2.update()
self.assertTrue(os.path.exists(proc1file))
self.assertTrue(os.path.exists(proc2file))
os.remove(proc1file)
os.remove(proc2file)
# Restart from postprocess, but at process PROC2 and following
b3 = Bank('localprocess')
b3.options.from_task = 'postprocess'
b3.options.process = 'PROC2'
b3.options.release = b.session.get('release')
b3.update()
#self.assertFalse(os.path.exists(proc1file))
self.assertTrue(os.path.exists(proc2file))
@attr('process')
def test_postprocess_wrong_process_name(self):
"""If a wrong process name is given, update returns False and prints an error message"""
b = Bank('local')
b.options.stop_after = 'download'
b.update()
self.assertFalse(b.session.get_status('postprocess'))
b2 = Bank('local')
b2.options.from_task = 'postprocess'
b2.options.release = b.session.get('release')
b2.options.process = 'fake'
self.assertFalse(b2.update())
self.assertFalse(b2.session.get_status('postprocess'))
self.assertEqual(b.session.get_full_release_directory(),
b2.session.get_full_release_directory())
def test_computed(self):
b = Bank('computed')
res = b.update(True)
self.assertTrue(res)
self.assertTrue(
os.path.exists(b.session.get_full_release_directory() +
'/sub1/flat/test_100.txt'))
self.assertTrue(b.session.get('update'))
# Check that, with depends non updated, bank is not updated itself
nextb = Bank('computed')
res = nextb.update(True)
self.assertFalse(nextb.session.get('update'))
@attr('nofile')
def test_computed_nofile(self):
b = Bank('computed2')
b.load_session(UpdateWorkflow.FLOW)
b.session.config.set('protocol', 'none')
b.session.config.set('sub1.files.move', 'flat/test_.*')
res = b.update(True)
self.assertTrue(res)
self.assertTrue(
os.path.exists(b.session.get_full_release_directory() +
'/sub1/flat/test_100.txt'))
def test_computed_ref_release(self):
b = Bank('computed2')
res = b.update(True)
b2 = Bank('sub1')
b2release = b2.bank['production'][len(b2.bank['production']) -
1]['release']
brelease = b.bank['production'][len(b.bank['production']) -
1]['release']
self.assertTrue(res)
self.assertTrue(brelease == b2release)
@attr('computed')
def test_computed_ref_release(self):
b = Bank('computed2')
res = b.update(True)
self.assertTrue(b.session.get('update'))
b2 = Bank('computed2')
res = b2.update(True)
self.assertFalse(b2.session.get('update'))
def test_computederror(self):
b = Bank('computederror')
res = b.update(True)
self.assertFalse(res)
self.assertTrue(b.session._session['depends']['sub2'])
self.assertFalse(b.session._session['depends']['error'])
@attr('directrelease')
def test_directhttp_release(self):
b = Bank('directhttp')
res = b.update()
self.assertTrue(b.session.get('update'))
self.assertTrue(
os.path.exists(b.session.get_full_release_directory() +
'/flat/debian/README.html'))
# print str(b.session.get('release'))
# print str(b.session.get('remoterelease'))
@attr('network')
def test_multi(self):
b = Bank('multi')
res = b.update()
with open(
os.path.join(b.session.get_full_release_directory(),
'flat/test1.json'), 'r') as content_file:
content = content_file.read()
my_json = json.loads(content)
self.assertTrue(my_json['args']['key1'] == 'value1')
with open(
os.path.join(b.session.get_full_release_directory(),
'flat/test2.json'), 'r') as content_file:
content = content_file.read()
my_json = json.loads(content)
self.assertTrue(my_json['form']['key1'] == 'value1')
def test_freeze(self):
b = Bank('local')
b.update()
rel = b.session.get('release')
b.freeze(rel)
prod = b.get_production(rel)
self.assertTrue(prod['freeze'] == True)
res = b.remove(rel)
self.assertTrue(res == False)
b.unfreeze(rel)
prod = b.get_production(rel)
self.assertTrue(prod['freeze'] == False)
res = b.remove(rel)
self.assertTrue(res == True)
def test_stats(self):
b = Bank('local')
b.update()
rel = b.session.get('release')
stats = Bank.get_banks_disk_usage()
self.assertTrue(stats[0]['size'] > 0)
for release in stats[0]['releases']:
if release['name'] == rel:
self.assertTrue(release['size'] > 0)
@attr('process')
def test_processes_meta_data(self):
b = Bank('localprocess')
b.update()
formats = b.session.get('formats')
self.assertTrue(len(formats['blast']) == 2)
self.assertTrue(len(formats['test'][0]['files']) == 3)
@attr('process')
def test_search(self):
b = Bank('localprocess')
b.update()
search_res = Bank.search(['blast'], [])
self.assertTrue(len(search_res) == 1)
search_res = Bank.search([], ['nucleic'])
self.assertTrue(len(search_res) == 1)
search_res = Bank.search(['blast'], ['nucleic'])
self.assertTrue(len(search_res) == 1)
search_res = Bank.search(['blast'], ['proteic'])
self.assertTrue(len(search_res) == 0)
def test_owner(self):
"""
test ACL with owner
"""
b = Bank('local')
res = b.update()
self.assertTrue(res)
b.set_owner('sample')
b2 = Bank('local')
try:
res = b2.update()
self.fail('not owner, should not be allowed')
except Exception as e:
pass