def get_stats():
redis_store = get_db()
pending = redis_store.llen(app.config['DEFAULT_QUEUE'])
legacy = redis_store.llen(app.config['LEGACY_QUEUE'])
fast = redis_store.llen(app.config['FAST_QUEUE'])
running = redis_store.llen('jobs:running')
# carry over jobs count from the old database from the config
total_jobs = app.config['OLD_JOB_COUNT'] + redis_store.llen('jobs:completed') + \
redis_store.llen('jobs:failed') + redis_store.llen('jobs:removed')
if pending + running + fast + legacy > 0:
status = 'working'
else:
status = 'idle'
ts_queued, ts_queued_m = _get_job_timestamps(_get_oldest_job(app.config['DEFAULT_QUEUE']))
ts_fast, ts_fast_m = _get_job_timestamps(_get_oldest_job(app.config['FAST_QUEUE']))
ts_legacy, ts_legacy_m = _get_job_timestamps(_get_oldest_job(app.config['LEGACY_QUEUE']))
return jsonify(status=status, queue_length=pending, running=running,
fast=fast, ts_fast=ts_fast, ts_fast_m=ts_fast_m,
legacy=legacy, ts_legacy=ts_legacy, ts_legacy_m=ts_legacy_m,
total_jobs=total_jobs,
ts_queued=ts_queued, ts_queued_m=ts_queued_m)
def test__copy_files(app, mocker):
fake_db = get_db()
assert app.config['FAKE_DB']
old_job = Job(fake_db, 'bacteria-old')
old_job.filename = 'fake.fa'
old_job.gff3 = 'fake.gff'
new_job = Job.fromExisting('bacteria-new', old_job)
fake_makedirs = mocker.patch('os.makedirs')
fake_copyfile = mocker.patch('shutil.copyfile')
utils._copy_files('fake_base', old_job, new_job)
new_job_basedir = os.path.join('fake_base', new_job.job_id, 'input')
fake_makedirs.assert_called_once_with(new_job_basedir, exist_ok=True)
old_filename = os.path.join('fake_base', old_job.job_id, 'input',
old_job.filename)
old_gff3 = os.path.join('fake_base', old_job.job_id, 'input', old_job.gff3)
new_filename = os.path.join('fake_base', new_job.job_id, 'input',
new_job.filename)
new_gff3 = os.path.join('fake_base', new_job.job_id, 'input', new_job.gff3)
fake_copyfile.assert_has_calls(
[call(old_filename, new_filename),
call(old_gff3, new_gff3)])
def test__submit_job_minimal(app):
"""Test fast mode job submission works as expected"""
fake_db = get_db()
queue = app.config['FAST_QUEUE']
assert app.config['FAKE_DB']
old_len = fake_db.llen(queue)
job = Job(fake_db, 'taxon-fake')
job.minimal = True
job.commit()
utils._submit_job(fake_db, job, app.config)
assert old_len + 1 == fake_db.llen(queue)
fake_db.ltrim(app.config['DOWNLOAD_QUEUE'], 2, 1) # clear queue
job = Job(fake_db, 'taxon-fake')
job.minimal = True
job.needs_download = True
job.commit()
utils._submit_job(fake_db, job, app.config)
assert 1 == fake_db.llen(app.config['DOWNLOAD_QUEUE'])
job.fetch()
assert job.target_queues == [queue]
def test__dark_launch_job(app, mocker):
fake_db = get_db()
assert app.config['FAKE_DB']
app.config['DARK_LAUNCH_PERCENTAGE'] = 10
fake_randrange = mocker.patch('random.randrange', return_value=15)
old_len = fake_db.llen('jobs:development')
job = Job(fake_db, 'taxon-fake')
job.commit()
utils._dark_launch_job(fake_db, job, app.config)
assert fake_db.llen('jobs:development') == old_len
fake_randrange = mocker.patch('random.randrange', return_value=5)
utils._dark_launch_job(fake_db, job, app.config)
assert fake_db.llen('jobs:development') == old_len + 1
dark_job_id = fake_db.lrange('jobs:development', -1, -1)[0]
dark_job = Job(fake_db, dark_job_id).fetch()
assert dark_job.original_id == job.job_id
# trim with start > end empties the list
fake_db.ltrim('jobs:downloads', 2, 1)
job.needs_download = True
job.commit()
utils._dark_launch_job(fake_db, job, app.config)
assert fake_db.llen('jobs:downloads') == 1
dark_job_id = fake_db.lrange('jobs:downloads', -1, -1)[0]
dark_job = Job(fake_db, dark_job_id).fetch()
assert dark_job.original_id == job.job_id
assert dark_job.target_queues == ['jobs:development']
def server_status():
redis_store = get_db()
pending = redis_store.llen('jobs:queued')
long_running = redis_store.llen("jobs:timeconsuming")
running = redis_store.llen('jobs:running')
# carry over jobs count from the old database from the config
total_jobs = app.config['OLD_JOB_COUNT'] + redis_store.llen('jobs:completed') + \
redis_store.llen('jobs:failed')
if pending + long_running + running > 0:
status = 'working'
else:
status = 'idle'
ts_queued, ts_queued_m = _get_job_timestamps(
_get_oldest_job("jobs:queued"))
ts_timeconsuming, ts_timeconsuming_m = _get_job_timestamps(
_get_oldest_job("jobs:timeconsuming"))
return jsonify(status=status,
queue_length=pending,
running=running,
long_running=long_running,
total_jobs=total_jobs,
ts_queued=ts_queued,
ts_queued_m=ts_queued_m,
ts_timeconsuming=ts_timeconsuming,
ts_timeconsuming_m=ts_timeconsuming_m)
def test__submit_job_vip(app):
"""Test VIP job submission works as expected"""
fake_db = get_db()
queue = app.config['PRIORITY_QUEUE']
assert app.config['FAKE_DB']
old_len = fake_db.llen(queue)
app.config['VIP_USERS'].add('*****@*****.**')
# No priority queue for Bob
job = Job(fake_db, 'taxon-fake')
job.email = '*****@*****.**'
job.commit()
utils._submit_job(fake_db, job, app.config)
assert old_len == fake_db.llen(queue)
# Priority queue for Alice
job = Job(fake_db, 'taxon-fake')
job.email = '*****@*****.**'
job.commit()
utils._submit_job(fake_db, job, app.config)
assert old_len + 1 == fake_db.llen(queue)
# Priority queue when downloading
fake_db.ltrim(app.config['DOWNLOAD_QUEUE'], 2, 1) # clear queue
job = Job(fake_db, 'taxon-fake')
job.email = '*****@*****.**'
job.needs_download = True
job.commit()
utils._submit_job(fake_db, job, app.config)
assert 1 == fake_db.llen(app.config['DOWNLOAD_QUEUE'])
job.fetch()
assert job.target_queues == [queue]
def test__submit_job_ip_waitlist(app):
"""Test job submission waitlisting by IP works as expected"""
fake_db = get_db()
ip = "192.168.0.1"
queue = "{}:{}".format(app.config['WAITLIST_PREFIX'], ip)
assert app.config['FAKE_DB']
app.config['MAX_JOBS_PER_USER'] = -1
job = Job(fake_db, 'taxon-fake')
job.ip_addr = ip
job.commit()
utils._submit_job(fake_db, job, app.config)
assert 1 == fake_db.llen(queue)
fake_db.ltrim(app.config['DOWNLOAD_QUEUE'], 2, 1) # clear queue
job = Job(fake_db, 'taxon-fake')
job.ip_addr = ip
job.needs_download = True
job.commit()
utils._submit_job(fake_db, job, app.config)
assert 1 == fake_db.llen(app.config['DOWNLOAD_QUEUE'])
job.fetch()
assert job.target_queues == [app.config['DEFAULT_QUEUE'], queue]
def test__submit_job_email_waitlist(app):
"""Test job submission waitlisting by email works as expected"""
fake_db = get_db()
email = "*****@*****.**"
queue = "{}:{}".format(app.config['WAITLIST_PREFIX'], email)
assert app.config['FAKE_DB']
app.config['MAX_JOBS_PER_USER'] = -1
job = Job(fake_db, 'taxon-fake')
job.email = email
job.commit()
utils._submit_job(fake_db, job, app.config)
assert 1 == fake_db.llen(queue)
fake_db.ltrim(app.config['DOWNLOAD_QUEUE'], 2, 1) # clear queue
job = Job(fake_db, 'taxon-fake')
job.email = email
job.needs_download = True
job.commit()
utils._submit_job(fake_db, job, app.config)
assert 1 == fake_db.llen(app.config['DOWNLOAD_QUEUE'])
job.fetch()
assert job.target_queues == [app.config['DEFAULT_QUEUE'], queue]
def test__submit_job_legacy(app):
"""Test legacy job submission works as expected"""
fake_db = get_db()
queue = app.config['LEGACY_QUEUE']
legacy_jobtype = app.config['LEGACY_JOBTYPE']
assert app.config['FAKE_DB']
old_len = fake_db.llen(queue)
job = Job(fake_db, 'taxon-fake')
job.jobtype = legacy_jobtype
job.commit()
utils._submit_job(fake_db, job, app.config)
assert old_len + 1 == fake_db.llen(queue)
fake_db.ltrim(app.config['DOWNLOAD_QUEUE'], 2, 1) # clear queue
job = Job(fake_db, 'taxon-fake')
job.jobtype = legacy_jobtype
job.needs_download = True
job.commit()
utils._submit_job(fake_db, job, app.config)
assert 1 == fake_db.llen(app.config['DOWNLOAD_QUEUE'])
job.fetch()
assert job.target_queues == [queue]
def display(task_id):
redis_store = get_db()
results_path = app.config['RESULTS_URL']
res = redis_store.hgetall(u'job:%s' % task_id)
if res == {}:
abort(404)
else:
job = Job(**res)
return render_template('display.html', job=job, results_path=results_path)
def _get_oldest_job(queue):
"""Get the oldest job in a queue"""
redis_store = get_db()
try:
res = redis_store.hgetall("job:%s" %
redis_store.lrange(queue, -1, -1)[0])
except IndexError:
return None
return Job(**res)
def test_api_submit_download(client):
"""Test submitting a job with a download from NCBI"""
data = dict(ncbi='FAKE')
response = client.post(url_for('api_submit'), data=data)
assert 200 == response.status_code
assert 'id' in response.json
job_key = 'job:{}'.format(response.json['id'])
redis = get_db()
assert redis.exists(job_key)
def test__submit_job(app):
"""Test job submission works as expected"""
fake_db = get_db()
assert app.config['FAKE_DB']
old_len = fake_db.llen('jobs:queued')
job = Job(fake_db, 'taxon-fake')
utils._submit_job(fake_db, job, app.config)
assert old_len + 1 == fake_db.llen('jobs:queued')
def test_api_submit_upload(client, fake_sequence):
"""Test submitting a job with an uploaded file"""
fake_fh = open(str(fake_sequence), 'rb')
data = dict(seq=fake_fh)
response = client.post(url_for('api_submit'), data=data)
assert 200 == response.status_code
assert 'id' in response.json
job_key = 'job:{}'.format(response.json['id'])
redis = get_db()
assert redis.exists(job_key)
def _get_oldest_job(queue):
"""Get the oldest job in a queue"""
redis_store = get_db()
try:
job_id = redis_store.lrange(queue, -1, -1)[0]
except IndexError:
return None
job = Job(redis_store, job_id)
job.fetch()
return job
def status(task_id):
redis_store = get_db()
res = redis_store.hgetall(u'job:%s' % task_id)
if res == {}:
abort(404)
job = Job(**res)
if job.status == 'done':
result_url = "%s/%s" % (app.config['RESULTS_URL'], job.uid)
if job.jobtype == 'antismash':
result_url += "/display.xhtml"
else:
result_url += "/index.html"
res['result_url'] = result_url
res['short_status'] = job.get_short_status()
return jsonify(res)
def get_news():
"""Display current notices"""
redis_store = get_db()
notices = []
for notice_id in redis_store.keys('notice:*'):
notice = Notice(redis_store, notice_id[7:])
try:
notice.fetch()
except ValueError:
continue
if notice.show_from > datetime.utcnow():
# show_from is in the future, don't show this yet
continue
notices.append(notice.to_dict())
return jsonify(notices=notices)
def test_api_submit_upload_leading_dash(client, tmpdir_factory):
"""Test submitting a job with an uploaded file with a leading dash"""
fake_sequence = tmpdir_factory.mktemp('to_upload').join('-test.fa')
fake_sequence.write(b'>test\nATGACCGAGAGTACATAG\n')
full_path = str(fake_sequence)
dirname = os.path.dirname(full_path)
filename = os.path.basename(full_path)
oldwd = os.getcwd()
os.chdir(dirname)
fake_fh = open(filename, 'rb')
os.chdir(oldwd)
data = dict(seq=fake_fh)
response = client.post(url_for('api_submit'), data=data)
assert 200 == response.status_code
assert 'id' in response.json
job_key = 'job:{}'.format(response.json['id'])
redis = get_db()
assert redis.exists(job_key)
assert redis.hget(job_key, 'filename') == 'test.fa'
def status(task_id):
redis_store = get_db()
job = Job(redis_store, task_id)
try:
job.fetch()
except ValueError:
# TODO: Write a json error handler for 404 errors
abort(404)
res = job.to_dict()
if job.state == 'done':
result_url = "%s/%s/index.html" % (app.config['RESULTS_URL'], job.job_id)
res['result_url'] = result_url
res['added_ts'] = job.added.strftime("%Y-%m-%dT%H:%M:%SZ")
res['last_changed_ts'] = job.last_changed.strftime("%Y-%m-%dT%H:%M:%SZ")
# TODO: This fixes old web UIs while stupid browser caching is going on. Can be removed soon, I hope.
# I hate browser caches.
res['short_status'] = job.state
return jsonify(res)
def test_api_status_pending(client):
"""Test reading the status of a job"""
data = dict(ncbi='FAKE')
response = client.post(url_for('api_submit'), data=data)
job_id = response.json['id']
response = client.get(url_for('status', task_id=job_id))
assert 200 == response.status_code
assert response.json['state'] == 'queued'
redis = get_db()
job = Job(redis, job_id)
job.fetch()
job.state = 'done'
job.commit()
response = client.get(url_for('status', task_id=job_id))
assert 200 == response.status_code
assert 'result_url' in response.json
response = client.get(url_for('status', task_id='nonexistent'))
assert 404 == response.status_code
def dispatch_job():
"""Internal helper to dispatch a new job"""
redis_store = get_db()
taxon = app.config['TAXON']
job_id = _generate_jobid(taxon)
job = Job(redis_store, job_id)
if 'X-Forwarded-For' in request.headers:
job.ip_addr = request.headers.getlist("X-Forwarded-For")[0].rpartition(
' ')[-1]
else:
job.ip_addr = request.remote_addr or 'untrackable'
ncbi = request.form.get('ncbi', '').strip()
val = request.form.get('email', '').strip()
if val:
job.email = val
job.minimal = _get_checkbox(request, 'minimal')
job.all_orfs = _get_checkbox(request, 'all_orfs')
job.smcogs = _get_checkbox(request, 'smcogs')
job.clusterblast = _get_checkbox(request, 'clusterblast')
job.knownclusterblast = _get_checkbox(request, 'knownclusterblast')
job.subclusterblast = _get_checkbox(request, 'subclusterblast')
job.cc_mibig = _get_checkbox(request, 'cc_mibig')
job.jobtype = request.form.get('jobtype', app.config['DEFAULT_JOBTYPE'])
if job.jobtype not in (app.config['LEGACY_JOBTYPE'],
app.config['DEFAULT_JOBTYPE']):
raise BadRequest(f"Invalid jobtype {job.jobtype}")
genefinder = request.form.get('genefinder', '')
if genefinder:
job.genefinder = genefinder
hmmdetection_strictness = request.form.get('hmmdetection_strictness', '')
if hmmdetection_strictness:
job.hmmdetection_strictness = hmmdetection_strictness
val = request.form.get('from', 0, type=int)
if val:
job.from_pos = val
val = request.form.get('to', 0, type=int)
if val:
job.to_pos = val
job.asf = _get_checkbox(request, 'asf')
job.tta = _get_checkbox(request, 'tta')
job.cassis = _get_checkbox(request, 'cassis')
job.clusterhmmer = _get_checkbox(request, 'clusterhmmer')
job.pfam2go = _get_checkbox(request, 'pfam2go')
job.rre = _get_checkbox(request, 'rre')
job.tigrfam = _get_checkbox(request, 'tigrfam')
dirname = path.join(app.config['RESULTS_PATH'], job.job_id, 'input')
os.makedirs(dirname)
if ncbi != '':
if ' ' in ncbi:
raise BadRequest("Spaces are not allowed in an NCBI ID.")
job.download = ncbi
job.needs_download = True
else:
upload = request.files['seq']
if upload is not None:
filename = secure_filename(upload.filename)
upload.save(path.join(dirname, filename))
if not path.exists(path.join(dirname, filename)):
raise BadRequest("Could not save file!")
job.filename = filename
job.needs_download = False
else:
raise BadRequest("Uploading input file failed!")
if 'gff3' in request.files:
gff_upload = request.files['gff3']
if gff_upload is not None:
gff_filename = secure_filename(gff_upload.filename)
gff_upload.save(path.join(dirname, gff_filename))
if not path.exists(path.join(dirname, gff_filename)):
raise BadRequest("Could not save GFF file!")
job.gff3 = gff_filename
if 'sideload' in request.files:
sideload = request.files['sideload']
if sideload is not None:
sideload_filename = secure_filename(sideload.filename)
sideload.save(path.join(dirname, sideload_filename))
if not path.exists(path.join(dirname, sideload_filename)):
raise BadRequest("Could not save sideload info file!")
job.sideload = sideload_filename
job.trace.append("{}-api".format(platform.node()))
_submit_job(redis_store, job, app.config)
_dark_launch_job(redis_store, job, app.config)
return job
def current_notices():
"Display current notices"
redis_store = get_db()
rets = redis_store.keys('notice:*')
notices = [redis_store.hgetall(n) for n in rets]
return jsonify(notices=notices)
def new():
redis_store = get_db()
error = None
results_path = app.config['RESULTS_URL']
old_email = ''
try:
if request.method == 'POST':
kwargs = {}
kwargs['ncbi'] = request.form.get('ncbi', '').strip()
kwargs['email'] = request.form.get('email', '').strip()
old_email = kwargs['email']
# This webapi is for plantiSMASH, so create plantiSMASH jobs
kwargs['jobtype'] = 'plantismash'
clusterblast = request.form.get('clusterblast', u'off')
knownclusterblast = request.form.get('knownclusterblast', u'off')
subclusterblast = request.form.get('subclusterblast', u'off')
fullhmmer = request.form.get('fullhmmer', u'off')
coexpress_mad = request.form.get('coexpress_mad', u'')
kwargs['cdh_cutoff'] = request.form.get('cdh_cutoff',
0.5,
type=float)
kwargs['min_domain_number'] = request.form.get('min_domain_number',
2,
type=int)
# Use boolean values instead of "on/off" strings
kwargs['clusterblast'] = (clusterblast == u'on')
kwargs['knownclusterblast'] = (knownclusterblast == u'on')
try:
kwargs['min_mad'] = int(coexpress_mad)
except ValueError:
pass
job = Job(**kwargs)
dirname = path.join(app.config['RESULTS_PATH'], job.uid)
os.mkdir(dirname)
upload = None
if kwargs['ncbi'] != '':
job.download = kwargs['ncbi']
else:
upload = request.files['seq']
if upload is not None:
filename = secure_filename(upload.filename)
upload.save(path.join(dirname, filename))
if not path.exists(path.join(dirname, filename)):
raise Exception("Could not save file!")
job.filename = filename
else:
raise Exception("Uploading input file failed!")
if 'gff' in request.files:
gff = request.files['gff']
if gff is not None and gff.filename != '':
filename = secure_filename(gff.filename)
gff.save(path.join(dirname, filename))
if not path.exists(path.join(dirname, filename)):
raise Exception("Could not save file!")
job.gff3 = filename
if 'coexpress_file' in request.files:
coexpress_file = request.files['coexpress_file']
if coexpress_file is not None and coexpress_file.filename != '':
filename = secure_filename(coexpress_file.filename)
coexpress_file.save(path.join(dirname, filename))
if not path.exists(path.join(dirname, filename)):
raise Exception("Could not save file!")
job.coexpress = True
_, ext = path.splitext(filename)
if ext.lower() == '.soft':
job.soft_file = filename
elif ext.lower() == '.csv':
job.csv_file = filename
else:
job.coexpress = False
_submit_job(redis_store, job)
return redirect(url_for('.display', task_id=job.uid))
except Exception as e:
error = unicode(e)
return render_template('new.html',
error=error,
old_email=old_email,
results_path=results_path)
def show_notices():
"Show current notices"
redis_store = get_db()
rets = redis_store.keys('notice:*')
notices = [Notice(**redis_store.hgetall(i)) for i in rets]
return render_template('notices.html', notices=notices, skip_notices=True)
