def get_stats(): redis_store = get_db() pending = redis_store.llen(app.config['DEFAULT_QUEUE']) legacy = redis_store.llen(app.config['LEGACY_QUEUE']) fast = redis_store.llen(app.config['FAST_QUEUE']) running = redis_store.llen('jobs:running') # carry over jobs count from the old database from the config total_jobs = app.config['OLD_JOB_COUNT'] + redis_store.llen('jobs:completed') + \ redis_store.llen('jobs:failed') + redis_store.llen('jobs:removed') if pending + running + fast + legacy > 0: status = 'working' else: status = 'idle' ts_queued, ts_queued_m = _get_job_timestamps(_get_oldest_job(app.config['DEFAULT_QUEUE'])) ts_fast, ts_fast_m = _get_job_timestamps(_get_oldest_job(app.config['FAST_QUEUE'])) ts_legacy, ts_legacy_m = _get_job_timestamps(_get_oldest_job(app.config['LEGACY_QUEUE'])) return jsonify(status=status, queue_length=pending, running=running, fast=fast, ts_fast=ts_fast, ts_fast_m=ts_fast_m, legacy=legacy, ts_legacy=ts_legacy, ts_legacy_m=ts_legacy_m, total_jobs=total_jobs, ts_queued=ts_queued, ts_queued_m=ts_queued_m)
def test__copy_files(app, mocker): fake_db = get_db() assert app.config['FAKE_DB'] old_job = Job(fake_db, 'bacteria-old') old_job.filename = 'fake.fa' old_job.gff3 = 'fake.gff' new_job = Job.fromExisting('bacteria-new', old_job) fake_makedirs = mocker.patch('os.makedirs') fake_copyfile = mocker.patch('shutil.copyfile') utils._copy_files('fake_base', old_job, new_job) new_job_basedir = os.path.join('fake_base', new_job.job_id, 'input') fake_makedirs.assert_called_once_with(new_job_basedir, exist_ok=True) old_filename = os.path.join('fake_base', old_job.job_id, 'input', old_job.filename) old_gff3 = os.path.join('fake_base', old_job.job_id, 'input', old_job.gff3) new_filename = os.path.join('fake_base', new_job.job_id, 'input', new_job.filename) new_gff3 = os.path.join('fake_base', new_job.job_id, 'input', new_job.gff3) fake_copyfile.assert_has_calls( [call(old_filename, new_filename), call(old_gff3, new_gff3)])
def test__submit_job_minimal(app): """Test fast mode job submission works as expected""" fake_db = get_db() queue = app.config['FAST_QUEUE'] assert app.config['FAKE_DB'] old_len = fake_db.llen(queue) job = Job(fake_db, 'taxon-fake') job.minimal = True job.commit() utils._submit_job(fake_db, job, app.config) assert old_len + 1 == fake_db.llen(queue) fake_db.ltrim(app.config['DOWNLOAD_QUEUE'], 2, 1) # clear queue job = Job(fake_db, 'taxon-fake') job.minimal = True job.needs_download = True job.commit() utils._submit_job(fake_db, job, app.config) assert 1 == fake_db.llen(app.config['DOWNLOAD_QUEUE']) job.fetch() assert job.target_queues == [queue]
def test__dark_launch_job(app, mocker): fake_db = get_db() assert app.config['FAKE_DB'] app.config['DARK_LAUNCH_PERCENTAGE'] = 10 fake_randrange = mocker.patch('random.randrange', return_value=15) old_len = fake_db.llen('jobs:development') job = Job(fake_db, 'taxon-fake') job.commit() utils._dark_launch_job(fake_db, job, app.config) assert fake_db.llen('jobs:development') == old_len fake_randrange = mocker.patch('random.randrange', return_value=5) utils._dark_launch_job(fake_db, job, app.config) assert fake_db.llen('jobs:development') == old_len + 1 dark_job_id = fake_db.lrange('jobs:development', -1, -1)[0] dark_job = Job(fake_db, dark_job_id).fetch() assert dark_job.original_id == job.job_id # trim with start > end empties the list fake_db.ltrim('jobs:downloads', 2, 1) job.needs_download = True job.commit() utils._dark_launch_job(fake_db, job, app.config) assert fake_db.llen('jobs:downloads') == 1 dark_job_id = fake_db.lrange('jobs:downloads', -1, -1)[0] dark_job = Job(fake_db, dark_job_id).fetch() assert dark_job.original_id == job.job_id assert dark_job.target_queues == ['jobs:development']
def server_status(): redis_store = get_db() pending = redis_store.llen('jobs:queued') long_running = redis_store.llen("jobs:timeconsuming") running = redis_store.llen('jobs:running') # carry over jobs count from the old database from the config total_jobs = app.config['OLD_JOB_COUNT'] + redis_store.llen('jobs:completed') + \ redis_store.llen('jobs:failed') if pending + long_running + running > 0: status = 'working' else: status = 'idle' ts_queued, ts_queued_m = _get_job_timestamps( _get_oldest_job("jobs:queued")) ts_timeconsuming, ts_timeconsuming_m = _get_job_timestamps( _get_oldest_job("jobs:timeconsuming")) return jsonify(status=status, queue_length=pending, running=running, long_running=long_running, total_jobs=total_jobs, ts_queued=ts_queued, ts_queued_m=ts_queued_m, ts_timeconsuming=ts_timeconsuming, ts_timeconsuming_m=ts_timeconsuming_m)
def test__submit_job_vip(app): """Test VIP job submission works as expected""" fake_db = get_db() queue = app.config['PRIORITY_QUEUE'] assert app.config['FAKE_DB'] old_len = fake_db.llen(queue) app.config['VIP_USERS'].add('*****@*****.**') # No priority queue for Bob job = Job(fake_db, 'taxon-fake') job.email = '*****@*****.**' job.commit() utils._submit_job(fake_db, job, app.config) assert old_len == fake_db.llen(queue) # Priority queue for Alice job = Job(fake_db, 'taxon-fake') job.email = '*****@*****.**' job.commit() utils._submit_job(fake_db, job, app.config) assert old_len + 1 == fake_db.llen(queue) # Priority queue when downloading fake_db.ltrim(app.config['DOWNLOAD_QUEUE'], 2, 1) # clear queue job = Job(fake_db, 'taxon-fake') job.email = '*****@*****.**' job.needs_download = True job.commit() utils._submit_job(fake_db, job, app.config) assert 1 == fake_db.llen(app.config['DOWNLOAD_QUEUE']) job.fetch() assert job.target_queues == [queue]
def test__submit_job_ip_waitlist(app): """Test job submission waitlisting by IP works as expected""" fake_db = get_db() ip = "192.168.0.1" queue = "{}:{}".format(app.config['WAITLIST_PREFIX'], ip) assert app.config['FAKE_DB'] app.config['MAX_JOBS_PER_USER'] = -1 job = Job(fake_db, 'taxon-fake') job.ip_addr = ip job.commit() utils._submit_job(fake_db, job, app.config) assert 1 == fake_db.llen(queue) fake_db.ltrim(app.config['DOWNLOAD_QUEUE'], 2, 1) # clear queue job = Job(fake_db, 'taxon-fake') job.ip_addr = ip job.needs_download = True job.commit() utils._submit_job(fake_db, job, app.config) assert 1 == fake_db.llen(app.config['DOWNLOAD_QUEUE']) job.fetch() assert job.target_queues == [app.config['DEFAULT_QUEUE'], queue]
def test__submit_job_email_waitlist(app): """Test job submission waitlisting by email works as expected""" fake_db = get_db() email = "*****@*****.**" queue = "{}:{}".format(app.config['WAITLIST_PREFIX'], email) assert app.config['FAKE_DB'] app.config['MAX_JOBS_PER_USER'] = -1 job = Job(fake_db, 'taxon-fake') job.email = email job.commit() utils._submit_job(fake_db, job, app.config) assert 1 == fake_db.llen(queue) fake_db.ltrim(app.config['DOWNLOAD_QUEUE'], 2, 1) # clear queue job = Job(fake_db, 'taxon-fake') job.email = email job.needs_download = True job.commit() utils._submit_job(fake_db, job, app.config) assert 1 == fake_db.llen(app.config['DOWNLOAD_QUEUE']) job.fetch() assert job.target_queues == [app.config['DEFAULT_QUEUE'], queue]
def test__submit_job_legacy(app): """Test legacy job submission works as expected""" fake_db = get_db() queue = app.config['LEGACY_QUEUE'] legacy_jobtype = app.config['LEGACY_JOBTYPE'] assert app.config['FAKE_DB'] old_len = fake_db.llen(queue) job = Job(fake_db, 'taxon-fake') job.jobtype = legacy_jobtype job.commit() utils._submit_job(fake_db, job, app.config) assert old_len + 1 == fake_db.llen(queue) fake_db.ltrim(app.config['DOWNLOAD_QUEUE'], 2, 1) # clear queue job = Job(fake_db, 'taxon-fake') job.jobtype = legacy_jobtype job.needs_download = True job.commit() utils._submit_job(fake_db, job, app.config) assert 1 == fake_db.llen(app.config['DOWNLOAD_QUEUE']) job.fetch() assert job.target_queues == [queue]
def display(task_id): redis_store = get_db() results_path = app.config['RESULTS_URL'] res = redis_store.hgetall(u'job:%s' % task_id) if res == {}: abort(404) else: job = Job(**res) return render_template('display.html', job=job, results_path=results_path)
def _get_oldest_job(queue): """Get the oldest job in a queue""" redis_store = get_db() try: res = redis_store.hgetall("job:%s" % redis_store.lrange(queue, -1, -1)[0]) except IndexError: return None return Job(**res)
def test_api_submit_download(client): """Test submitting a job with a download from NCBI""" data = dict(ncbi='FAKE') response = client.post(url_for('api_submit'), data=data) assert 200 == response.status_code assert 'id' in response.json job_key = 'job:{}'.format(response.json['id']) redis = get_db() assert redis.exists(job_key)
def test__submit_job(app): """Test job submission works as expected""" fake_db = get_db() assert app.config['FAKE_DB'] old_len = fake_db.llen('jobs:queued') job = Job(fake_db, 'taxon-fake') utils._submit_job(fake_db, job, app.config) assert old_len + 1 == fake_db.llen('jobs:queued')
def test_api_submit_upload(client, fake_sequence): """Test submitting a job with an uploaded file""" fake_fh = open(str(fake_sequence), 'rb') data = dict(seq=fake_fh) response = client.post(url_for('api_submit'), data=data) assert 200 == response.status_code assert 'id' in response.json job_key = 'job:{}'.format(response.json['id']) redis = get_db() assert redis.exists(job_key)
def _get_oldest_job(queue): """Get the oldest job in a queue""" redis_store = get_db() try: job_id = redis_store.lrange(queue, -1, -1)[0] except IndexError: return None job = Job(redis_store, job_id) job.fetch() return job
def status(task_id): redis_store = get_db() res = redis_store.hgetall(u'job:%s' % task_id) if res == {}: abort(404) job = Job(**res) if job.status == 'done': result_url = "%s/%s" % (app.config['RESULTS_URL'], job.uid) if job.jobtype == 'antismash': result_url += "/display.xhtml" else: result_url += "/index.html" res['result_url'] = result_url res['short_status'] = job.get_short_status() return jsonify(res)
def get_news(): """Display current notices""" redis_store = get_db() notices = [] for notice_id in redis_store.keys('notice:*'): notice = Notice(redis_store, notice_id[7:]) try: notice.fetch() except ValueError: continue if notice.show_from > datetime.utcnow(): # show_from is in the future, don't show this yet continue notices.append(notice.to_dict()) return jsonify(notices=notices)
def test_api_submit_upload_leading_dash(client, tmpdir_factory): """Test submitting a job with an uploaded file with a leading dash""" fake_sequence = tmpdir_factory.mktemp('to_upload').join('-test.fa') fake_sequence.write(b'>test\nATGACCGAGAGTACATAG\n') full_path = str(fake_sequence) dirname = os.path.dirname(full_path) filename = os.path.basename(full_path) oldwd = os.getcwd() os.chdir(dirname) fake_fh = open(filename, 'rb') os.chdir(oldwd) data = dict(seq=fake_fh) response = client.post(url_for('api_submit'), data=data) assert 200 == response.status_code assert 'id' in response.json job_key = 'job:{}'.format(response.json['id']) redis = get_db() assert redis.exists(job_key) assert redis.hget(job_key, 'filename') == 'test.fa'
def status(task_id): redis_store = get_db() job = Job(redis_store, task_id) try: job.fetch() except ValueError: # TODO: Write a json error handler for 404 errors abort(404) res = job.to_dict() if job.state == 'done': result_url = "%s/%s/index.html" % (app.config['RESULTS_URL'], job.job_id) res['result_url'] = result_url res['added_ts'] = job.added.strftime("%Y-%m-%dT%H:%M:%SZ") res['last_changed_ts'] = job.last_changed.strftime("%Y-%m-%dT%H:%M:%SZ") # TODO: This fixes old web UIs while stupid browser caching is going on. Can be removed soon, I hope. # I hate browser caches. res['short_status'] = job.state return jsonify(res)
def test_api_status_pending(client): """Test reading the status of a job""" data = dict(ncbi='FAKE') response = client.post(url_for('api_submit'), data=data) job_id = response.json['id'] response = client.get(url_for('status', task_id=job_id)) assert 200 == response.status_code assert response.json['state'] == 'queued' redis = get_db() job = Job(redis, job_id) job.fetch() job.state = 'done' job.commit() response = client.get(url_for('status', task_id=job_id)) assert 200 == response.status_code assert 'result_url' in response.json response = client.get(url_for('status', task_id='nonexistent')) assert 404 == response.status_code
def dispatch_job(): """Internal helper to dispatch a new job""" redis_store = get_db() taxon = app.config['TAXON'] job_id = _generate_jobid(taxon) job = Job(redis_store, job_id) if 'X-Forwarded-For' in request.headers: job.ip_addr = request.headers.getlist("X-Forwarded-For")[0].rpartition( ' ')[-1] else: job.ip_addr = request.remote_addr or 'untrackable' ncbi = request.form.get('ncbi', '').strip() val = request.form.get('email', '').strip() if val: job.email = val job.minimal = _get_checkbox(request, 'minimal') job.all_orfs = _get_checkbox(request, 'all_orfs') job.smcogs = _get_checkbox(request, 'smcogs') job.clusterblast = _get_checkbox(request, 'clusterblast') job.knownclusterblast = _get_checkbox(request, 'knownclusterblast') job.subclusterblast = _get_checkbox(request, 'subclusterblast') job.cc_mibig = _get_checkbox(request, 'cc_mibig') job.jobtype = request.form.get('jobtype', app.config['DEFAULT_JOBTYPE']) if job.jobtype not in (app.config['LEGACY_JOBTYPE'], app.config['DEFAULT_JOBTYPE']): raise BadRequest(f"Invalid jobtype {job.jobtype}") genefinder = request.form.get('genefinder', '') if genefinder: job.genefinder = genefinder hmmdetection_strictness = request.form.get('hmmdetection_strictness', '') if hmmdetection_strictness: job.hmmdetection_strictness = hmmdetection_strictness val = request.form.get('from', 0, type=int) if val: job.from_pos = val val = request.form.get('to', 0, type=int) if val: job.to_pos = val job.asf = _get_checkbox(request, 'asf') job.tta = _get_checkbox(request, 'tta') job.cassis = _get_checkbox(request, 'cassis') job.clusterhmmer = _get_checkbox(request, 'clusterhmmer') job.pfam2go = _get_checkbox(request, 'pfam2go') job.rre = _get_checkbox(request, 'rre') job.tigrfam = _get_checkbox(request, 'tigrfam') dirname = path.join(app.config['RESULTS_PATH'], job.job_id, 'input') os.makedirs(dirname) if ncbi != '': if ' ' in ncbi: raise BadRequest("Spaces are not allowed in an NCBI ID.") job.download = ncbi job.needs_download = True else: upload = request.files['seq'] if upload is not None: filename = secure_filename(upload.filename) upload.save(path.join(dirname, filename)) if not path.exists(path.join(dirname, filename)): raise BadRequest("Could not save file!") job.filename = filename job.needs_download = False else: raise BadRequest("Uploading input file failed!") if 'gff3' in request.files: gff_upload = request.files['gff3'] if gff_upload is not None: gff_filename = secure_filename(gff_upload.filename) gff_upload.save(path.join(dirname, gff_filename)) if not path.exists(path.join(dirname, gff_filename)): raise BadRequest("Could not save GFF file!") job.gff3 = gff_filename if 'sideload' in request.files: sideload = request.files['sideload'] if sideload is not None: sideload_filename = secure_filename(sideload.filename) sideload.save(path.join(dirname, sideload_filename)) if not path.exists(path.join(dirname, sideload_filename)): raise BadRequest("Could not save sideload info file!") job.sideload = sideload_filename job.trace.append("{}-api".format(platform.node())) _submit_job(redis_store, job, app.config) _dark_launch_job(redis_store, job, app.config) return job
def current_notices(): "Display current notices" redis_store = get_db() rets = redis_store.keys('notice:*') notices = [redis_store.hgetall(n) for n in rets] return jsonify(notices=notices)
def new(): redis_store = get_db() error = None results_path = app.config['RESULTS_URL'] old_email = '' try: if request.method == 'POST': kwargs = {} kwargs['ncbi'] = request.form.get('ncbi', '').strip() kwargs['email'] = request.form.get('email', '').strip() old_email = kwargs['email'] # This webapi is for plantiSMASH, so create plantiSMASH jobs kwargs['jobtype'] = 'plantismash' clusterblast = request.form.get('clusterblast', u'off') knownclusterblast = request.form.get('knownclusterblast', u'off') subclusterblast = request.form.get('subclusterblast', u'off') fullhmmer = request.form.get('fullhmmer', u'off') coexpress_mad = request.form.get('coexpress_mad', u'') kwargs['cdh_cutoff'] = request.form.get('cdh_cutoff', 0.5, type=float) kwargs['min_domain_number'] = request.form.get('min_domain_number', 2, type=int) # Use boolean values instead of "on/off" strings kwargs['clusterblast'] = (clusterblast == u'on') kwargs['knownclusterblast'] = (knownclusterblast == u'on') try: kwargs['min_mad'] = int(coexpress_mad) except ValueError: pass job = Job(**kwargs) dirname = path.join(app.config['RESULTS_PATH'], job.uid) os.mkdir(dirname) upload = None if kwargs['ncbi'] != '': job.download = kwargs['ncbi'] else: upload = request.files['seq'] if upload is not None: filename = secure_filename(upload.filename) upload.save(path.join(dirname, filename)) if not path.exists(path.join(dirname, filename)): raise Exception("Could not save file!") job.filename = filename else: raise Exception("Uploading input file failed!") if 'gff' in request.files: gff = request.files['gff'] if gff is not None and gff.filename != '': filename = secure_filename(gff.filename) gff.save(path.join(dirname, filename)) if not path.exists(path.join(dirname, filename)): raise Exception("Could not save file!") job.gff3 = filename if 'coexpress_file' in request.files: coexpress_file = request.files['coexpress_file'] if coexpress_file is not None and coexpress_file.filename != '': filename = secure_filename(coexpress_file.filename) coexpress_file.save(path.join(dirname, filename)) if not path.exists(path.join(dirname, filename)): raise Exception("Could not save file!") job.coexpress = True _, ext = path.splitext(filename) if ext.lower() == '.soft': job.soft_file = filename elif ext.lower() == '.csv': job.csv_file = filename else: job.coexpress = False _submit_job(redis_store, job) return redirect(url_for('.display', task_id=job.uid)) except Exception as e: error = unicode(e) return render_template('new.html', error=error, old_email=old_email, results_path=results_path)
def show_notices(): "Show current notices" redis_store = get_db() rets = redis_store.keys('notice:*') notices = [Notice(**redis_store.hgetall(i)) for i in rets] return render_template('notices.html', notices=notices, skip_notices=True)