async def find(req): db = req.app["db"] ids = req.query.get("ids", False) if ids: return json_response(await db.subtraction.distinct("_id")) host_count = await db.subtraction.count({"is_host": True}) ready_host_count = await db.subtraction.count({"is_host": True, "ready": True}) term = req.query.get("find", None) db_query = dict() if term: db_query.update(compose_regex_query(term, ["_id"])) data = await paginate( db.subtraction, db_query, req.query, sort="_id", projection=virtool.db.subtractions.PROJECTION ) data.update({ "host_count": host_count, "ready_host_count": ready_host_count }) return json_response(data)
async def find(req): """ Return a list of indexes. """ db = req.app["db"] ready = req.query.get("ready", False) if not ready: data = await virtool.db.indexes.find(db, req.query) return json_response(data) pipeline = [ { "$match": { "ready": True } }, { "$sort": { "version": -1 } }, { "$group": { "_id": "$reference.id", "index": { "$first": "$_id" }, "version": { "$first": "$version" } } } ] ready_indexes = list() async for agg in db.indexes.aggregate(pipeline): reference_name = await virtool.db.utils.get_one_field(db.references, "name", agg["_id"]) ready_indexes.append({ "id": agg["index"], "version": agg["version"], "reference": { "id": agg["_id"], "name": reference_name } }) return json_response(ready_indexes)
async def edit(req): """ Update specific fields in the sample document. """ db = req.app["db"] data = req["data"] sample_id = req.match_info["sample_id"] if not await virtool.db.samples.check_rights(db, sample_id, req["client"]): return insufficient_rights() message = await virtool.db.samples.check_name(db, req.app["settings"], data["name"], sample_id=sample_id) if message: return bad_request(message) document = await db.samples.find_one_and_update( {"_id": sample_id}, {"$set": data}, projection=virtool.db.samples.LIST_PROJECTION) processed = virtool.utils.base_processor(document) return json_response(processed)
async def install(req): db = req.app["db"] releases = await virtool.db.utils.get_one_field(db.status, "releases", "software") try: latest_release = releases[0] except IndexError: return not_found("Could not find latest uninstalled release") process = await virtool.db.processes.register( db, "update_software", file_size=latest_release["size"]) await db.status.update_one( {"_id": "software"}, {"$set": { "process": process, "updating": True }}) update = virtool.github.create_update_subdocument( latest_release, False, req["client"].user_id, virtool.utils.timestamp()) await aiojobs.aiohttp.spawn( req, virtool.db.software.install(req.app, latest_release, process["id"])) return json_response(update)
async def list_releases(req): try: releases = await virtool.db.software.fetch_and_update_releases(req.app) except aiohttp.ClientConnectorError: return bad_gateway("Could not connection to www.virtool.ca") return json_response(releases)
async def get(req): """ Get the complete representation of a specific reference. """ db = req.app["db"] ref_id = req.match_info["ref_id"] document = await db.references.find_one(ref_id) if not document: return not_found() try: internal_control_id = document["internal_control"]["id"] except (KeyError, TypeError): internal_control_id = None computed = await asyncio.shield( virtool.db.references.get_computed(db, ref_id, internal_control_id)) users = await asyncio.shield( virtool.db.users.attach_identicons(db, document["users"])) document.update({**computed, "users": users}) return json_response(virtool.utils.base_processor(document))
async def edit_user(req): db = req.app["db"] data = req["data"] ref_id = req.match_info["ref_id"] user_id = req.match_info["user_id"] document = await db.references.find_one( { "_id": ref_id, "users.id": user_id }, ["groups", "users"]) if document is None: return not_found() if not await virtool.db.references.check_right(req, ref_id, "modify"): return insufficient_rights() subdocument = await virtool.db.references.edit_group_or_user( db, ref_id, user_id, "users", data) if subdocument is None: return not_found() subdocument = await virtool.db.users.attach_identicons(db, subdocument) return json_response(subdocument)
async def add_user(req): db = req.app["db"] data = req["data"] ref_id = req.match_info["ref_id"] document = await db.references.find_one(ref_id, ["groups", "users"]) if document is None: return not_found() if not await virtool.db.references.check_right(req, ref_id, "modify"): return insufficient_rights() try: subdocument = await virtool.db.references.add_group_or_user( db, ref_id, "users", data) except virtool.errors.DatabaseError as err: if "already exists" in str(err): return bad_request("User already exists") if "does not exist" in str(err): return bad_request("User does not exist") raise headers = {"Location": f"/api/refs/{ref_id}/users/{subdocument['id']}"} subdocument = await virtool.db.users.attach_identicons(db, subdocument) return json_response(subdocument, headers=headers, status=201)
async def remove(req): """ Remove a reference and its otus, history, and indexes. """ db = req.app["db"] ref_id = req.match_info["ref_id"] if not await virtool.db.utils.id_exists(db.references, ref_id): return not_found() if not await virtool.db.references.check_right(req, ref_id, "remove"): return insufficient_rights() user_id = req["client"].user_id context = {"ref_id": ref_id, "user_id": user_id} process = await virtool.db.processes.register(db, "delete_reference", context=context) await db.references.delete_one({"_id": ref_id}) p = virtool.db.references.RemoveReferenceProcess(req.app, process["id"]) await aiojobs.aiohttp.spawn(req, p.run()) headers = {"Content-Location": f"/api/processes/{process['id']}"} return json_response(process, 202, headers)
async def set_rights(req): """ Change rights settings for the specified sample document. """ db = req.app["db"] data = req["data"] sample_id = req.match_info["sample_id"] if not await db.samples.count({"_id": sample_id}): return not_found() user_id = req["client"].user_id # Only update the document if the connected user owns the samples or is an administrator. if not req[ "client"].administrator and user_id != await virtool.db.samples.get_sample_owner( db, sample_id): return insufficient_rights("Must be administrator or sample owner") group = data.get("group", None) if group: existing_group_ids = await db.groups.distinct("_id") + ["none"] if group not in existing_group_ids: return bad_request("Group does not exist") # Update the sample document with the new rights. document = await db.samples.find_one_and_update( {"_id": sample_id}, {"$set": data}, projection=virtool.db.samples.RIGHTS_PROJECTION) return json_response(document)
async def get_isolate(req): """ Get a complete specific isolate sub-document, including its sequences. """ db = req.app["db"] otu_id = req.match_info["otu_id"] isolate_id = req.match_info["isolate_id"] document = await db.otus.find_one( { "_id": otu_id, "isolates.id": isolate_id }, ["isolates"]) if not document: return not_found() isolate = dict(virtool.otus.find_isolate(document["isolates"], isolate_id), sequences=[]) cursor = db.sequences.find({ "otu_id": otu_id, "isolate_id": isolate_id }, { "otu_id": False, "isolate_id": False }) async for sequence in cursor: sequence["id"] = sequence.pop("_id") isolate["sequences"].append(sequence) return json_response(isolate)
async def update_permissions(req): """ Updates the permissions of a given group. """ db = req.app["db"] data = req["data"] group_id = req.match_info["group_id"] old_document = await db.groups.find_one({"_id": group_id}, ["permissions"]) if not old_document: return not_found() old_document["permissions"].update(data["permissions"]) # Get the current permissions dict for the passed group id. document = await db.groups.find_one_and_update( {"_id": group_id}, {"$set": { "permissions": old_document["permissions"] }}) await virtool.db.groups.update_member_users(db, group_id) return json_response(virtool.utils.base_processor(document))
async def install(req): """ Install the latest official HMM database from GitHub. """ db = req.app["db"] user_id = req["client"].user_id if await db.status.count({"_id": "hmm", "updates.ready": False}): return conflict("Install already in progress") process = await virtool.db.processes.register(db, "install_hmms") document = await db.status.find_one_and_update( {"_id": "hmm"}, {"$set": { "process": { "id": process["id"] } }}) release = document.get("release", None) if release is None: return bad_request("Target release does not exist") update = virtool.github.create_update_subdocument(release, False, user_id) await db.status.update_one({"_id": "hmm"}, {"$push": {"updates": update}}) await aiojobs.aiohttp.spawn( req, virtool.db.hmm.install(req.app, process["id"], release, user_id)) return json_response(update)
async def find_history(req): """ Find history changes for a specific index. """ db = req.app["db"] index_id = req.match_info["index_id"] if not await db.indexes.count({"_id": index_id}): return not_found() term = req.query.get("term", None) db_query = { "index.id": index_id } if term: db_query.update(compose_regex_query(term, ["otu.name", "user.id"])) data = await paginate( db.history, db_query, req.query, sort=[("otu.name", 1), ("otu.version", -1)], projection=virtool.db.history.LIST_PROJECTION, reverse=True ) return json_response(data)
async def get(req): """ Get a complete analysis document. """ db = req.app["db"] analysis_id = req.match_info["analysis_id"] document = await db.analyses.find_one(analysis_id) if document is None: return not_found() sample = await db.samples.find_one({"_id": document["sample"]["id"]}, {"quality": False}) if not sample: return bad_request("Parent sample does not exist") read, _ = virtool.samples.get_sample_rights(sample, req["client"]) if not read: return insufficient_rights() if document["ready"]: document = await virtool.db.analyses.format_analysis( db, req.app["settings"], document) document["subtraction"] = {"id": sample["subtraction"]["id"]} return json_response(virtool.utils.base_processor(document))
async def edit(req): """ Edit the user account. """ db = req.app["db"] data = await req.json() user_id = req["client"].user_id password = data.get("password", None) if password is not None and len( password) < req.app["settings"]["minimum_password_length"]: raise bad_request("Password does not meet minimum length requirement") update = dict() if password: try: update = await virtool.db.account.compose_password_update( db, user_id, data["old_password"], password) except ValueError as err: if "Invalid credentials" in str(err): return bad_request("Invalid credentials") raise if "email" in data: update["email"] = data["email"] document = await db.users.find_one_and_update( {"_id": user_id}, {"$set": update}, projection=virtool.db.users.ACCOUNT_PROJECTION) return json_response(virtool.utils.base_processor(document))
async def find(req): """ Find files based on an optional text query that is matched against file names. Only ready, unreserved files are returned. """ db = req.app["db"] base_query = {"ready": True, "reserved": False} file_type = req.query.get("type", None) db_query = dict() if file_type: base_query["type"] = file_type data = await paginate(db.files, db_query, req.query, sort=[("uploaded_at", pymongo.DESCENDING)], projection=virtool.db.files.PROJECTION, base_query=base_query) return json_response(data)
async def update(req): """ Update application settings based on request data. """ raw_data = await req.json() data = {key: req["data"][key] for key in raw_data} proc = data.get("proc", None) mem = data.get("mem", None) settings = req.app["settings"] error_message = virtool.settings.check_resource_limits( proc, mem, settings.data) if error_message: return conflict(error_message) proc = proc or settings["proc"] mem = mem or settings["mem"] error_message = virtool.settings.check_task_specific_limits( proc, mem, data) if error_message: return conflict(error_message) app_settings = req.app["settings"] app_settings.update(data) await app_settings.write() return json_response(app_settings.data)
async def find(req): """ Return a list of job documents. """ db = req.app["db"] term = req.query.get("term", None) db_query = dict() if term: db_query.update(compose_regex_query(term, ["task", "user.id"])) data = await paginate( db.jobs, db_query, req.query, projection=virtool.db.jobs.PROJECTION, processor=virtool.db.jobs.processor ) data["documents"].sort(key=lambda d: d["created_at"]) return json_response(data)
async def find(req): db = req.app["db"] term = req.query.get("find", None) db_query = dict() if term: db_query = compose_regex_query(term, ["name", "data_type"]) base_query = virtool.db.references.compose_base_find_query( req["client"].user_id, req["client"].administrator, req["client"].groups) data = await paginate(db.references, db_query, req.query, sort="name", base_query=base_query, processor=virtool.utils.base_processor, projection=virtool.db.references.PROJECTION) for d in data["documents"]: latest_build, otu_count, unbuilt_count = await asyncio.gather( virtool.db.references.get_latest_build(db, d["id"]), virtool.db.references.get_otu_count(db, d["id"]), virtool.db.references.get_unbuilt_count(db, d["id"])) d.update({ "latest_build": latest_build, "otu_count": otu_count, "unbuilt_change_count": unbuilt_count }) return json_response(data)
async def upload(req): db = req.app["db"] file_type = req.match_info["file_type"] if file_type not in FILE_TYPES: return not_found() errors = naive_validator(req) if errors: return invalid_query(errors) filename = req.query["name"] document = await virtool.db.files.create(db, filename, file_type, user_id=req["client"].user_id) file_id = document["id"] await naive_writer(req, file_id) headers = {"Location": f"/api/files/{file_id}"} return json_response(document, status=201, headers=headers)
async def blast(req): """ BLAST a contig sequence that is part of a NuVs result record. The resulting BLAST data will be attached to that sequence. """ db = req.app["db"] settings = req.app["settings"] analysis_id = req.match_info["analysis_id"] sequence_index = int(req.match_info["sequence_index"]) document = await db.analyses.find_one( {"_id": analysis_id}, ["ready", "algorithm", "results", "sample"]) if not document: return not_found("Analysis not found") if document["algorithm"] != "nuvs": return conflict("Not a NuVs analysis") if not document["ready"]: return conflict("Analysis is still running") sequence = virtool.analyses.find_nuvs_sequence_by_index( document, sequence_index) if sequence is None: return not_found("Sequence not found") sample = await db.samples.find_one({"_id": document["sample"]["id"]}, virtool.db.samples.PROJECTION) if not sample: return bad_request("Parent sample does not exist") _, write = virtool.samples.get_sample_rights(sample, req["client"]) if not write: return insufficient_rights() # Start a BLAST at NCBI with the specified sequence. Return a RID that identifies the BLAST run. rid, _ = await virtool.bio.initialize_ncbi_blast(req.app["settings"], sequence) blast_data, document = await virtool.db.analyses.update_nuvs_blast( db, settings, analysis_id, sequence_index, rid) # Wait on BLAST request as a Task until the it completes on NCBI. At that point the sequence in the DB will be # updated with the BLAST result. await aiojobs.aiohttp.spawn( req, virtool.bio.wait_for_blast_result(db, req.app["settings"], analysis_id, sequence_index, rid)) headers = { "Location": f"/api/analyses/{analysis_id}/{sequence_index}/blast" } return json_response(blast_data, headers=headers, status=201)
async def create(req): """ Add a new user to the user database. """ db = req.app["db"] data = await req.json() settings = req.app["settings"] if data["user_id"] == "virtool": return bad_request("Reserved user name: virtool") if len(data["password"]) < settings["minimum_password_length"]: return bad_request("Password does not meet length requirement") user_id = data["user_id"] try: document = await virtool.db.users.create(db, user_id, data["password"], data["force_reset"]) except virtool.errors.DatabaseError: return bad_request("User already exists") headers = {"Location": "/api/users/" + user_id} return json_response(virtool.utils.base_processor( {key: document[key] for key in virtool.db.users.PROJECTION}), headers=headers, status=201)
async def unavailable(req): return json_response( { "id": "requires_setup", "message": "Server is not configured" }, status=503, headers={"Location": "/setup"})
async def wrapped(req): if not public and not req["client"].user_id: return json_response( { "id": "requires_authorization", "message": "Requires authorization" }, status=401) if not req["client"].administrator: if admin: return json_response( { "id": "not_permitted", "message": "Requires administrative privilege" }, status=403) if permission and not req["client"].permissions[permission]: return json_response( { "id": "not_permitted", "message": "Not permitted" }, status=403) content_type = req.headers.get("Content-type", "") if "multipart/form-data" not in content_type: try: data = await req.json() except (json.decoder.JSONDecodeError, UnicodeDecodeError): data = dict() if schema: v = Validator(schema, purge_unknown=True) if not v.validate(data): return invalid_input(v.errors) data = v.document req["data"] = data return await handler(req)
async def find(req): """ Get a list of all existing group documents. """ cursor = req.app["db"].groups.find() return json_response( [virtool.utils.base_processor(d) async for d in cursor])
async def get_api_keys(req): db = req.app["db"] user_id = req["client"].user_id cursor = db.keys.find({"user.id": user_id}, API_KEY_PROJECTION) return json_response([d async for d in cursor], status=200)
async def middleware(req, handler): try: return await handler(req) except virtool.errors.ProxyError as err: return json_response({ "id": "proxy_error", "message": str(err) }, status=500) except aiohttp.ClientProxyConnectionError: return json_response( { "id": "proxy_error", "message": "Could not connect to proxy" }, status=500)
async def get_resources(req): """ Get a object describing compute resource usage on the server. """ resources = virtool.resources.get() req.app["resources"] = resources return json_response(resources)
async def get(req): db = req.app["db"] await virtool.db.software.fetch_and_update_releases(req.app, ignore_errors=True) document = await db.status.find_one("software") return json_response(virtool.utils.base_processor(document))