async def find(req):
db = req.app["db"]
ids = req.query.get("ids", False)
if ids:
return json_response(await db.subtraction.distinct("_id"))
host_count = await db.subtraction.count({"is_host": True})
ready_host_count = await db.subtraction.count({"is_host": True, "ready": True})
term = req.query.get("find", None)
db_query = dict()
if term:
db_query.update(compose_regex_query(term, ["_id"]))
data = await paginate(
db.subtraction,
db_query,
req.query,
sort="_id",
projection=virtool.db.subtractions.PROJECTION
)
data.update({
"host_count": host_count,
"ready_host_count": ready_host_count
})
return json_response(data)
async def find(req):
"""
Return a list of indexes.
"""
db = req.app["db"]
ready = req.query.get("ready", False)
if not ready:
data = await virtool.db.indexes.find(db, req.query)
return json_response(data)
pipeline = [
{
"$match": {
"ready": True
}
},
{
"$sort": {
"version": -1
}
},
{
"$group": {
"_id": "$reference.id",
"index": {
"$first": "$_id"
},
"version": {
"$first": "$version"
}
}
}
]
ready_indexes = list()
async for agg in db.indexes.aggregate(pipeline):
reference_name = await virtool.db.utils.get_one_field(db.references, "name", agg["_id"])
ready_indexes.append({
"id": agg["index"],
"version": agg["version"],
"reference": {
"id": agg["_id"],
"name": reference_name
}
})
return json_response(ready_indexes)
async def edit(req):
"""
Update specific fields in the sample document.
"""
db = req.app["db"]
data = req["data"]
sample_id = req.match_info["sample_id"]
if not await virtool.db.samples.check_rights(db, sample_id, req["client"]):
return insufficient_rights()
message = await virtool.db.samples.check_name(db,
req.app["settings"],
data["name"],
sample_id=sample_id)
if message:
return bad_request(message)
document = await db.samples.find_one_and_update(
{"_id": sample_id}, {"$set": data},
projection=virtool.db.samples.LIST_PROJECTION)
processed = virtool.utils.base_processor(document)
return json_response(processed)
async def install(req):
db = req.app["db"]
releases = await virtool.db.utils.get_one_field(db.status, "releases",
"software")
try:
latest_release = releases[0]
except IndexError:
return not_found("Could not find latest uninstalled release")
process = await virtool.db.processes.register(
db, "update_software", file_size=latest_release["size"])
await db.status.update_one(
{"_id": "software"}, {"$set": {
"process": process,
"updating": True
}})
update = virtool.github.create_update_subdocument(
latest_release, False, req["client"].user_id,
virtool.utils.timestamp())
await aiojobs.aiohttp.spawn(
req, virtool.db.software.install(req.app, latest_release,
process["id"]))
return json_response(update)
async def list_releases(req):
try:
releases = await virtool.db.software.fetch_and_update_releases(req.app)
except aiohttp.ClientConnectorError:
return bad_gateway("Could not connection to www.virtool.ca")
return json_response(releases)
async def get(req):
"""
Get the complete representation of a specific reference.
"""
db = req.app["db"]
ref_id = req.match_info["ref_id"]
document = await db.references.find_one(ref_id)
if not document:
return not_found()
try:
internal_control_id = document["internal_control"]["id"]
except (KeyError, TypeError):
internal_control_id = None
computed = await asyncio.shield(
virtool.db.references.get_computed(db, ref_id, internal_control_id))
users = await asyncio.shield(
virtool.db.users.attach_identicons(db, document["users"]))
document.update({**computed, "users": users})
return json_response(virtool.utils.base_processor(document))
async def edit_user(req):
db = req.app["db"]
data = req["data"]
ref_id = req.match_info["ref_id"]
user_id = req.match_info["user_id"]
document = await db.references.find_one(
{
"_id": ref_id,
"users.id": user_id
}, ["groups", "users"])
if document is None:
return not_found()
if not await virtool.db.references.check_right(req, ref_id, "modify"):
return insufficient_rights()
subdocument = await virtool.db.references.edit_group_or_user(
db, ref_id, user_id, "users", data)
if subdocument is None:
return not_found()
subdocument = await virtool.db.users.attach_identicons(db, subdocument)
return json_response(subdocument)
async def add_user(req):
db = req.app["db"]
data = req["data"]
ref_id = req.match_info["ref_id"]
document = await db.references.find_one(ref_id, ["groups", "users"])
if document is None:
return not_found()
if not await virtool.db.references.check_right(req, ref_id, "modify"):
return insufficient_rights()
try:
subdocument = await virtool.db.references.add_group_or_user(
db, ref_id, "users", data)
except virtool.errors.DatabaseError as err:
if "already exists" in str(err):
return bad_request("User already exists")
if "does not exist" in str(err):
return bad_request("User does not exist")
raise
headers = {"Location": f"/api/refs/{ref_id}/users/{subdocument['id']}"}
subdocument = await virtool.db.users.attach_identicons(db, subdocument)
return json_response(subdocument, headers=headers, status=201)
async def remove(req):
"""
Remove a reference and its otus, history, and indexes.
"""
db = req.app["db"]
ref_id = req.match_info["ref_id"]
if not await virtool.db.utils.id_exists(db.references, ref_id):
return not_found()
if not await virtool.db.references.check_right(req, ref_id, "remove"):
return insufficient_rights()
user_id = req["client"].user_id
context = {"ref_id": ref_id, "user_id": user_id}
process = await virtool.db.processes.register(db,
"delete_reference",
context=context)
await db.references.delete_one({"_id": ref_id})
p = virtool.db.references.RemoveReferenceProcess(req.app, process["id"])
await aiojobs.aiohttp.spawn(req, p.run())
headers = {"Content-Location": f"/api/processes/{process['id']}"}
return json_response(process, 202, headers)
async def set_rights(req):
"""
Change rights settings for the specified sample document.
"""
db = req.app["db"]
data = req["data"]
sample_id = req.match_info["sample_id"]
if not await db.samples.count({"_id": sample_id}):
return not_found()
user_id = req["client"].user_id
# Only update the document if the connected user owns the samples or is an administrator.
if not req[
"client"].administrator and user_id != await virtool.db.samples.get_sample_owner(
db, sample_id):
return insufficient_rights("Must be administrator or sample owner")
group = data.get("group", None)
if group:
existing_group_ids = await db.groups.distinct("_id") + ["none"]
if group not in existing_group_ids:
return bad_request("Group does not exist")
# Update the sample document with the new rights.
document = await db.samples.find_one_and_update(
{"_id": sample_id}, {"$set": data},
projection=virtool.db.samples.RIGHTS_PROJECTION)
return json_response(document)
async def get_isolate(req):
"""
Get a complete specific isolate sub-document, including its sequences.
"""
db = req.app["db"]
otu_id = req.match_info["otu_id"]
isolate_id = req.match_info["isolate_id"]
document = await db.otus.find_one(
{
"_id": otu_id,
"isolates.id": isolate_id
}, ["isolates"])
if not document:
return not_found()
isolate = dict(virtool.otus.find_isolate(document["isolates"], isolate_id),
sequences=[])
cursor = db.sequences.find({
"otu_id": otu_id,
"isolate_id": isolate_id
}, {
"otu_id": False,
"isolate_id": False
})
async for sequence in cursor:
sequence["id"] = sequence.pop("_id")
isolate["sequences"].append(sequence)
return json_response(isolate)
async def update_permissions(req):
"""
Updates the permissions of a given group.
"""
db = req.app["db"]
data = req["data"]
group_id = req.match_info["group_id"]
old_document = await db.groups.find_one({"_id": group_id}, ["permissions"])
if not old_document:
return not_found()
old_document["permissions"].update(data["permissions"])
# Get the current permissions dict for the passed group id.
document = await db.groups.find_one_and_update(
{"_id": group_id},
{"$set": {
"permissions": old_document["permissions"]
}})
await virtool.db.groups.update_member_users(db, group_id)
return json_response(virtool.utils.base_processor(document))
async def install(req):
"""
Install the latest official HMM database from GitHub.
"""
db = req.app["db"]
user_id = req["client"].user_id
if await db.status.count({"_id": "hmm", "updates.ready": False}):
return conflict("Install already in progress")
process = await virtool.db.processes.register(db, "install_hmms")
document = await db.status.find_one_and_update(
{"_id": "hmm"}, {"$set": {
"process": {
"id": process["id"]
}
}})
release = document.get("release", None)
if release is None:
return bad_request("Target release does not exist")
update = virtool.github.create_update_subdocument(release, False, user_id)
await db.status.update_one({"_id": "hmm"}, {"$push": {"updates": update}})
await aiojobs.aiohttp.spawn(
req, virtool.db.hmm.install(req.app, process["id"], release, user_id))
return json_response(update)
async def find_history(req):
"""
Find history changes for a specific index.
"""
db = req.app["db"]
index_id = req.match_info["index_id"]
if not await db.indexes.count({"_id": index_id}):
return not_found()
term = req.query.get("term", None)
db_query = {
"index.id": index_id
}
if term:
db_query.update(compose_regex_query(term, ["otu.name", "user.id"]))
data = await paginate(
db.history,
db_query,
req.query,
sort=[("otu.name", 1), ("otu.version", -1)],
projection=virtool.db.history.LIST_PROJECTION,
reverse=True
)
return json_response(data)
async def get(req):
"""
Get a complete analysis document.
"""
db = req.app["db"]
analysis_id = req.match_info["analysis_id"]
document = await db.analyses.find_one(analysis_id)
if document is None:
return not_found()
sample = await db.samples.find_one({"_id": document["sample"]["id"]},
{"quality": False})
if not sample:
return bad_request("Parent sample does not exist")
read, _ = virtool.samples.get_sample_rights(sample, req["client"])
if not read:
return insufficient_rights()
if document["ready"]:
document = await virtool.db.analyses.format_analysis(
db, req.app["settings"], document)
document["subtraction"] = {"id": sample["subtraction"]["id"]}
return json_response(virtool.utils.base_processor(document))
async def edit(req):
"""
Edit the user account.
"""
db = req.app["db"]
data = await req.json()
user_id = req["client"].user_id
password = data.get("password", None)
if password is not None and len(
password) < req.app["settings"]["minimum_password_length"]:
raise bad_request("Password does not meet minimum length requirement")
update = dict()
if password:
try:
update = await virtool.db.account.compose_password_update(
db, user_id, data["old_password"], password)
except ValueError as err:
if "Invalid credentials" in str(err):
return bad_request("Invalid credentials")
raise
if "email" in data:
update["email"] = data["email"]
document = await db.users.find_one_and_update(
{"_id": user_id}, {"$set": update},
projection=virtool.db.users.ACCOUNT_PROJECTION)
return json_response(virtool.utils.base_processor(document))
async def find(req):
"""
Find files based on an optional text query that is matched against file names. Only ready, unreserved files are
returned.
"""
db = req.app["db"]
base_query = {"ready": True, "reserved": False}
file_type = req.query.get("type", None)
db_query = dict()
if file_type:
base_query["type"] = file_type
data = await paginate(db.files,
db_query,
req.query,
sort=[("uploaded_at", pymongo.DESCENDING)],
projection=virtool.db.files.PROJECTION,
base_query=base_query)
return json_response(data)
async def update(req):
"""
Update application settings based on request data.
"""
raw_data = await req.json()
data = {key: req["data"][key] for key in raw_data}
proc = data.get("proc", None)
mem = data.get("mem", None)
settings = req.app["settings"]
error_message = virtool.settings.check_resource_limits(
proc, mem, settings.data)
if error_message:
return conflict(error_message)
proc = proc or settings["proc"]
mem = mem or settings["mem"]
error_message = virtool.settings.check_task_specific_limits(
proc, mem, data)
if error_message:
return conflict(error_message)
app_settings = req.app["settings"]
app_settings.update(data)
await app_settings.write()
return json_response(app_settings.data)
async def find(req):
"""
Return a list of job documents.
"""
db = req.app["db"]
term = req.query.get("term", None)
db_query = dict()
if term:
db_query.update(compose_regex_query(term, ["task", "user.id"]))
data = await paginate(
db.jobs,
db_query,
req.query,
projection=virtool.db.jobs.PROJECTION,
processor=virtool.db.jobs.processor
)
data["documents"].sort(key=lambda d: d["created_at"])
return json_response(data)
async def find(req):
db = req.app["db"]
term = req.query.get("find", None)
db_query = dict()
if term:
db_query = compose_regex_query(term, ["name", "data_type"])
base_query = virtool.db.references.compose_base_find_query(
req["client"].user_id, req["client"].administrator,
req["client"].groups)
data = await paginate(db.references,
db_query,
req.query,
sort="name",
base_query=base_query,
processor=virtool.utils.base_processor,
projection=virtool.db.references.PROJECTION)
for d in data["documents"]:
latest_build, otu_count, unbuilt_count = await asyncio.gather(
virtool.db.references.get_latest_build(db, d["id"]),
virtool.db.references.get_otu_count(db, d["id"]),
virtool.db.references.get_unbuilt_count(db, d["id"]))
d.update({
"latest_build": latest_build,
"otu_count": otu_count,
"unbuilt_change_count": unbuilt_count
})
return json_response(data)
async def upload(req):
db = req.app["db"]
file_type = req.match_info["file_type"]
if file_type not in FILE_TYPES:
return not_found()
errors = naive_validator(req)
if errors:
return invalid_query(errors)
filename = req.query["name"]
document = await virtool.db.files.create(db,
filename,
file_type,
user_id=req["client"].user_id)
file_id = document["id"]
await naive_writer(req, file_id)
headers = {"Location": f"/api/files/{file_id}"}
return json_response(document, status=201, headers=headers)
async def blast(req):
"""
BLAST a contig sequence that is part of a NuVs result record. The resulting BLAST data will be attached to that
sequence.
"""
db = req.app["db"]
settings = req.app["settings"]
analysis_id = req.match_info["analysis_id"]
sequence_index = int(req.match_info["sequence_index"])
document = await db.analyses.find_one(
{"_id": analysis_id}, ["ready", "algorithm", "results", "sample"])
if not document:
return not_found("Analysis not found")
if document["algorithm"] != "nuvs":
return conflict("Not a NuVs analysis")
if not document["ready"]:
return conflict("Analysis is still running")
sequence = virtool.analyses.find_nuvs_sequence_by_index(
document, sequence_index)
if sequence is None:
return not_found("Sequence not found")
sample = await db.samples.find_one({"_id": document["sample"]["id"]},
virtool.db.samples.PROJECTION)
if not sample:
return bad_request("Parent sample does not exist")
_, write = virtool.samples.get_sample_rights(sample, req["client"])
if not write:
return insufficient_rights()
# Start a BLAST at NCBI with the specified sequence. Return a RID that identifies the BLAST run.
rid, _ = await virtool.bio.initialize_ncbi_blast(req.app["settings"],
sequence)
blast_data, document = await virtool.db.analyses.update_nuvs_blast(
db, settings, analysis_id, sequence_index, rid)
# Wait on BLAST request as a Task until the it completes on NCBI. At that point the sequence in the DB will be
# updated with the BLAST result.
await aiojobs.aiohttp.spawn(
req,
virtool.bio.wait_for_blast_result(db, req.app["settings"], analysis_id,
sequence_index, rid))
headers = {
"Location": f"/api/analyses/{analysis_id}/{sequence_index}/blast"
}
return json_response(blast_data, headers=headers, status=201)
async def create(req):
"""
Add a new user to the user database.
"""
db = req.app["db"]
data = await req.json()
settings = req.app["settings"]
if data["user_id"] == "virtool":
return bad_request("Reserved user name: virtool")
if len(data["password"]) < settings["minimum_password_length"]:
return bad_request("Password does not meet length requirement")
user_id = data["user_id"]
try:
document = await virtool.db.users.create(db, user_id, data["password"],
data["force_reset"])
except virtool.errors.DatabaseError:
return bad_request("User already exists")
headers = {"Location": "/api/users/" + user_id}
return json_response(virtool.utils.base_processor(
{key: document[key]
for key in virtool.db.users.PROJECTION}),
headers=headers,
status=201)
async def unavailable(req):
return json_response(
{
"id": "requires_setup",
"message": "Server is not configured"
},
status=503,
headers={"Location": "/setup"})
async def wrapped(req):
if not public and not req["client"].user_id:
return json_response(
{
"id": "requires_authorization",
"message": "Requires authorization"
},
status=401)
if not req["client"].administrator:
if admin:
return json_response(
{
"id": "not_permitted",
"message": "Requires administrative privilege"
},
status=403)
if permission and not req["client"].permissions[permission]:
return json_response(
{
"id": "not_permitted",
"message": "Not permitted"
},
status=403)
content_type = req.headers.get("Content-type", "")
if "multipart/form-data" not in content_type:
try:
data = await req.json()
except (json.decoder.JSONDecodeError, UnicodeDecodeError):
data = dict()
if schema:
v = Validator(schema, purge_unknown=True)
if not v.validate(data):
return invalid_input(v.errors)
data = v.document
req["data"] = data
return await handler(req)
async def find(req):
"""
Get a list of all existing group documents.
"""
cursor = req.app["db"].groups.find()
return json_response(
[virtool.utils.base_processor(d) async for d in cursor])
async def get_api_keys(req):
db = req.app["db"]
user_id = req["client"].user_id
cursor = db.keys.find({"user.id": user_id}, API_KEY_PROJECTION)
return json_response([d async for d in cursor], status=200)
async def middleware(req, handler):
try:
return await handler(req)
except virtool.errors.ProxyError as err:
return json_response({
"id": "proxy_error",
"message": str(err)
},
status=500)
except aiohttp.ClientProxyConnectionError:
return json_response(
{
"id": "proxy_error",
"message": "Could not connect to proxy"
},
status=500)
async def get_resources(req):
"""
Get a object describing compute resource usage on the server.
"""
resources = virtool.resources.get()
req.app["resources"] = resources
return json_response(resources)
async def get(req):
db = req.app["db"]
await virtool.db.software.fetch_and_update_releases(req.app,
ignore_errors=True)
document = await db.status.find_one("software")
return json_response(virtool.utils.base_processor(document))
