def test_array_indices():
job = mj.daligner_job_array(1,
config.get('general', 'database'),
config.get('general', 'script_directory'),
reservation_token='test-token')
assert_equals(job._array_index_str(), '1')
job = mj.daligner_job_array(100,
config.get('general', 'database'),
config.get('general', 'script_directory'),
reservation_token='test-token')
assert_equals(job._array_index_str(), '1-100')
job = mj.daligner_job_array(200,
config.get('general', 'database'),
config.get('general', 'script_directory'),
reservation_token='test-token')
assert_equals(job._array_index_str(), '1-200')
job = mj.daligner_job_array(201,
config.get('general', 'database'),
config.get('general', 'script_directory'),
reservation_token='test-token')
assert_equals(job._array_index_str(), '1-201')
job = mj.daligner_job_array(201,
config.get('general', 'database'),
config.get('general', 'script_directory'),
reservation_token='test-token',
max_simultaneous_tasks=10)
assert_equals(job._array_index_str(), '1-201%10')
def test_exclusive():
db1 = sqlite3.connect(config.get('general', 'database'), timeout=0)
c1 = db1.cursor()
db2 = sqlite3.connect(config.get('general', 'database'), timeout=0)
c2 = db2.cursor()
select_query = '''SELECT rowid FROM daligner_job
WHERE status = "NOTSTARTED" LIMIT 10'''
c1.execute('BEGIN EXCLUSIVE')
with assert_raises(sqlite3.OperationalError) as oe:
c2.execute(select_query)
c1.execute(select_query)
rowids1 = [x[0] for x in c1.fetchall()]
with assert_raises(sqlite3.OperationalError) as oe:
c2.execute(select_query)
c1.execute('UPDATE daligner_job SET status = "RESERVED" WHERE rowid IN({0})' \
.format(','.join('?' for x in rowids1)), tuple(rowids1))
with assert_raises(sqlite3.OperationalError) as oe:
c2.execute(select_query)
db1.commit()
c2.execute(select_query)
rowids2 = [x[0] for x in c2.fetchall()]
assert_true(len(rowids1), len(rowids2))
assert_true(len(set(rowids1 + rowids2)), len(rowids1) + len(rowids2))
def test_prepare_script_sbatch_args():
job = mj.prepare_job('test',
config.get('general', 'fasta'),
config.getint('general', 'blocksize'),
script_directory=config.get('general',
'script_directory'),
log_directory=config.get('general', 'log_directory'))
cmd = job.start(dryrun=True)
assert_true('-o {0}/marvel_prepare.log' \
.format(config.get('general', 'log_directory')) in cmd)
def test_prepare_script():
job = mj.prepare_job('test',
config.get('general', 'fasta'),
config.getint('general', 'blocksize'),
script_directory=config.get('general',
'script_directory'),
log_directory=config.get('general', 'log_directory'))
assert_false(os.path.exists(job.filename))
job.start(dryrun=True)
assert_true(os.path.exists(job.filename))
modtime = os.path.getmtime(job.filename)
job = mj.prepare_job('test',
config.get('general', 'fasta'),
config.getint('general', 'blocksize'),
script_directory=config.get('general',
'script_directory'),
log_directory=config.get('general', 'log_directory'))
time.sleep(0.1)
job.start(dryrun=True)
assert_equals(modtime, os.path.getmtime(job.filename))
job = mj.prepare_job('test',
config.get('general', 'fasta'),
config.getint('general', 'blocksize') + 10,
script_directory=config.get('general',
'script_directory'),
log_directory=config.get('general', 'log_directory'))
time.sleep(0.1)
job.start(dryrun=True)
assert_not_equals(modtime, os.path.getmtime(job.filename))
def test_masking_server_script():
job = mj.masking_server_job('test', 20,
config.get('DMserver', 'checkpoint_file'),
script_directory=config.get('general',
'script_directory'),
log_directory=config.get('general',
'log_directory'),
port=12345,
threads=4)
assert_false(os.path.isfile(job.filename))
job.save_script()
assert_true(os.path.isfile(job.filename))
def test_update_daligner_jobs():
jobs = db.get_daligner_jobs(5, status=mj.slurm_utils.status.notstarted)
db.update_daligner_jobs(jobs, config.get('general', 'log_directory'))
jobs = db.get_daligner_jobs(75, status=mj.slurm_utils.status.running)
assert_equals(len(jobs), 75)
db.update_daligner_jobs(jobs, config.get('general', 'log_directory'))
jobs = db.get_daligner_jobs(status=mj.slurm_utils.status.running)
assert_equals(len(jobs), 25)
jobs = db.get_daligner_jobs(status=mj.slurm_utils.status.completed)
assert_equals(len(jobs), 100)
def test_array_script():
job = mj.daligner_job_array(10,
config.get('general', 'database'),
config.get('general', 'script_directory'),
reservation_token='test-token')
assert_equals(
job.filename,
os.path.join(config.get('general', 'script_directory'),
'daligner_array.sh'))
assert_false(os.path.isfile(job.filename))
job.save_script()
assert_true(os.path.isfile(job.filename))
os.remove(job.filename)
def test_marvel_job_single_line():
script_filename = os.path.join(config.get('general', 'script_directory'),
'single_line_script.sh')
job = mj.job.marvel_job(['executable', '-o', 'test_output'], 'testjob',
script_filename)
assert_equal(job.filename, script_filename)
assert_false(os.path.isfile(script_filename))
job.save_script()
assert_true(os.path.isfile(script_filename))
# Should have shebang, set statement, and the call to the executable
assert_equal(len(str(job).splitlines()), 3)
def test_marvel_job_multiple_lines():
script_filename = os.path.join(config.get('general', 'script_directory'),
'multiline_script.sh')
job = mj.job.marvel_job(
[['x=$(prepare stuff)'], ['executable', '-o', 'test_output'],
['postprocess', '--for-real']], 'multiline_test', script_filename)
assert_equal(job.filename, script_filename)
assert_false(os.path.isfile(job.filename))
job.save_script()
assert_true(os.path.isfile(job.filename))
assert_equal(len(str(job).splitlines()), 5)
def test_file_creation():
job1 = mj.daligner_job_array(10,
config.get('general', 'database'),
config.get('general', 'script_directory'),
log_directory=config.get(
'general', 'log_directory'),
jobs_per_task=10,
reservation_token='test-token')
cmd1 = job1.start(dryrun=True)
modtime = os.path.getmtime(job1.filename)
assert_true(cmd1.endswith(' test-token'))
job2 = mj.daligner_job_array(100,
config.get('general', 'database'),
config.get('general', 'script_directory'),
jobs_per_task=100,
reservation_token='test-token2')
cmd2 = job2.start(dryrun=True)
time.sleep(0.1)
assert_equals(modtime, os.path.getmtime(job2.filename))
assert_true(cmd2.endswith(' test-token2'))
os.remove(job2.filename)
assert_false(os.path.exists(job1.filename))
def test_masking_server_sbatch_args():
job = mj.masking_server_job('test', 20,
config.get('DMserver', 'checkpoint_file'),
script_directory=config.get('general',
'script_directory'),
log_directory=config.get('general',
'log_directory'),
port=12345,
threads=4,
cluster='snowy')
cmd = job.start(dryrun=True)
with open(job.filename) as f:
str_script = f.read()
assert_true('-M snowy' in cmd)
assert_false('#SBATCH -C' in str_script)
assert_true('-o {0}/marvel_masking.log' \
.format(config.get('general', 'log_directory')) \
in cmd)
job = mj.masking_server_job('test', 20,
config.get('DMserver', 'checkpoint_file'),
script_directory=config.get('general',
'script_directory'),
log_directory=config.get('general',
'log_directory'),
port=12345,
threads=4,
cluster='rackham',
constraint='mem256GB')
cmd = job.start(dryrun=True)
with open(job.filename) as f:
str_script = f.read()
assert_false('-M snowy' in cmd)
assert_true('-M rackham' in cmd)
assert_true('#SBATCH -C mem256GB' in str_script)
assert_true('-o {0}/marvel_masking.log' \
.format(config.get('general', 'log_directory')) \
in cmd)
def test_array_submit():
job = mj.daligner_job_array(10,
config.get('general', 'database'),
config.get('general', 'script_directory'),
log_directory=config.get(
'general', 'log_directory'),
reservation_token='test-token')
assert_false(os.path.isfile(job.filename))
cmd = job.start(dryrun=True)
assert_true(os.path.isfile(job.filename))
assert_true('-o {0}' \
.format(os.path.join(config.get('general',
'log_directory'),
'daligner_array_test-token_%a_%A_%a.log')) \
in cmd)
job = mj.daligner_job_array(201,
config.get('general', 'database'),
config.get('general', 'script_directory'),
reservation_token='test-token')
cmd = job.start(dryrun=True)
assert_true('-o daligner_array_test-token_%a_%A_%a.log' in cmd)
os.remove(job.filename)
def test_database_backup():
backup_filename = '{0}.backup'.format(config.get('general', 'database'))
db.backup(backup_filename)
assert_true(os.path.isfile(backup_filename))