def serve(context, config, host, port, debug, livereload):
"""Start the web server."""
pymongo_config = dict(
MONGO_HOST=context.obj['host'],
MONGO_PORT=context.obj['port'],
MONGO_DBNAME=context.obj['mongodb'],
MONGO_USERNAME=context.obj['username'],
MONGO_PASSWORD=context.obj['password'],
)
valid_connection = check_connection(
host=pymongo_config['MONGO_HOST'],
port=pymongo_config['MONGO_PORT'],
username=pymongo_config['MONGO_USERNAME'],
password=pymongo_config['MONGO_PASSWORD'],
)
log.info("Test if mongod is running")
if not valid_connection:
log.warning("Connection could not be established")
log.info("Is mongod running?")
context.abort()
config = os.path.abspath(config) if config else None
app = create_app(config=pymongo_config, config_file=config)
if livereload:
server = Server(app.wsgi_app)
server.serve(host=host, port=port, debug=debug)
else:
app.run(host=host, port=port, debug=debug)
def serve(context, config, host, port, debug, livereload):
"""Start the web server."""
pymongo_config = dict(
MONGO_HOST=context.obj['host'],
MONGO_PORT=context.obj['port'],
MONGO_DBNAME=context.obj['mongodb'],
MONGO_USERNAME=context.obj['username'],
MONGO_PASSWORD=context.obj['password'],
)
valid_connection = check_connection(
host=pymongo_config['MONGO_HOST'],
port=pymongo_config['MONGO_PORT'],
username=pymongo_config['MONGO_USERNAME'],
password=pymongo_config['MONGO_PASSWORD'],
authdb=context.obj['authdb'],
)
log.info("Test if mongod is running")
if not valid_connection:
log.warning("Connection could not be established")
log.info("Is mongod running?")
context.abort()
config = os.path.abspath(config) if config else None
app = create_app(config=pymongo_config, config_file=config)
if livereload:
server = Server(app.wsgi_app)
server.serve(host=host, port=port, debug=debug)
else:
app.run(host=host, port=port, debug=debug)
def cli(context, mongodb, username, password, authdb, host, port, loglevel,
config, demo):
"""scout: manage interactions with a scout instance."""
coloredlogs.install(level=loglevel)
log.info("Running scout version %s", __version__)
log.debug("Debug logging enabled.")
mongo_config = {}
cli_config = {}
if config:
log.debug("Use config file %s", config)
with open(config, 'r') as in_handle:
cli_config = yaml.load(in_handle)
mongo_config['mongodb'] = (mongodb or cli_config.get('mongodb') or 'scout')
if demo:
mongo_config['mongodb'] = 'scout-demo'
mongo_config['host'] = (host or cli_config.get('host') or 'localhost')
mongo_config['port'] = (port or cli_config.get('port') or 27017)
mongo_config['username'] = username or cli_config.get('username')
mongo_config['password'] = password or cli_config.get('password')
mongo_config['authdb'] = authdb or cli_config.get(
'authdb') or mongo_config['mongodb']
# mongo uri looks like:
# mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]]
if context.invoked_subcommand in ('setup', 'serve'):
mongo_config['adapter'] = None
else:
log.info("Setting database name to %s", mongo_config['mongodb'])
log.debug("Setting host to %s", mongo_config['host'])
log.debug("Setting port to %s", mongo_config['port'])
valid_connection = check_connection(
host=mongo_config['host'],
port=mongo_config['port'],
username=mongo_config['username'],
password=mongo_config['password'],
authdb=mongo_config['authdb'],
)
log.info("Test if mongod is running")
if not valid_connection:
log.warning("Connection could not be established")
context.abort()
try:
client = get_connection(**mongo_config)
except ConnectionFailure:
context.abort()
database = client[mongo_config['mongodb']]
log.info("Setting up a mongo adapter")
mongo_config['adapter'] = MongoAdapter(database)
context.obj = mongo_config
def serve(host, port, debug, livereload, test):
"""Start the web server."""
pymongo_config = dict(
MONGO_HOST=current_app.config.get("MONGO_HOST", "localhost"),
MONGO_PORT=current_app.config.get("MONGO_PORT", 27017),
MONGO_DBNAME=current_app.config.get("MONGO_DBNAME", "scout"),
MONGO_USERNAME=current_app.config.get("MONGO_USERNAME", None),
MONGO_PASSWORD=current_app.config.get("MONGO_PASSWORD", None),
)
valid_connection = check_connection(
host=pymongo_config["MONGO_HOST"],
port=pymongo_config["MONGO_PORT"],
username=pymongo_config["MONGO_USERNAME"],
password=pymongo_config["MONGO_PASSWORD"],
authdb=current_app.config.get("MONGO_DBNAME", "scout"),
)
LOG.info("Test if mongod is running")
if not valid_connection:
LOG.warning("Connection could not be established")
LOG.info("Is mongod running?")
raise click.Abort()
if test:
LOG.info("Connection could be established")
return
if livereload:
server = Server(current_app.wsgi_app)
server.serve(host=host, port=port, debug=debug)
else:
return run_simple(
hostname=host,
port=port,
application=current_app,
use_reloader=False,
use_debugger=debug,
)
def update_panels(context, mongodb, username, password, authdb, host, port,
loglevel, config):
"""scout: manage interactions with a scout instance."""
coloredlogs.install(level=loglevel)
LOG.info("Running scout version %s", __version__)
LOG.debug("Debug logging enabled.")
mongo_config = {}
cli_config = {}
if config:
LOG.debug("Use config file %s", config)
with open(config, "r") as in_handle:
cli_config = yaml.load(in_handle, Loader=yaml.FullLoader)
mongo_config["mongodb"] = mongodb or cli_config.get("mongodb") or "scout"
mongo_config["host"] = host or cli_config.get("host") or "localhost"
mongo_config["port"] = port or cli_config.get("port") or 27017
mongo_config["username"] = username or cli_config.get("username")
mongo_config["password"] = password or cli_config.get("password")
mongo_config["authdb"] = (authdb or cli_config.get("authdb")
or mongo_config["mongodb"])
mongo_config["omim_api_key"] = cli_config.get("omim_api_key")
LOG.info("Setting database name to %s", mongo_config["mongodb"])
LOG.debug("Setting host to %s", mongo_config["host"])
LOG.debug("Setting port to %s", mongo_config["port"])
valid_connection = check_connection(
host=mongo_config["host"],
port=mongo_config["port"],
username=mongo_config["username"],
password=mongo_config["password"],
authdb=mongo_config["authdb"],
)
LOG.info("Test if mongod is running")
if not valid_connection:
LOG.warning("Connection could not be established")
context.abort()
try:
client = get_connection(**mongo_config)
except ConnectionFailure:
context.abort()
database = client[mongo_config["mongodb"]]
LOG.info("Setting up a mongo adapter")
mongo_config["client"] = client
adapter = MongoAdapter(database)
requests = []
for case_obj in adapter.case_collection.find():
# pp(case_obj)
gene_to_panels = adapter.gene_to_panels(case_obj)
variants = adapter.variant_collection.find({
"case_id": case_obj["_id"],
"category": "snv",
"variant_type": "clinical"
})
for variant_obj in variants:
panel_names = set()
for hgnc_id in variant_obj["hgnc_ids"]:
gene_panels = gene_to_panels.get(hgnc_id, set())
panel_names = panel_names.union(gene_panels)
if panel_names:
operation = pymongo.UpdateOne(
{"_id": variant_obj["_id"]},
{"$set": {
"panels": list(panel_names)
}})
requests.append(operation)
if len(requests) > 5000:
adapter.variant_collection.bulk_write(requests, ordered=False)
requests = []
if requests:
adapter.variant_collection.bulk_write(requests, ordered=False)
requests = []
def update_panels(context, mongodb, username, password, authdb, host, port, loglevel, config):
"""scout: manage interactions with a scout instance."""
coloredlogs.install(level=loglevel)
LOG.info("Running scout version %s", __version__)
LOG.debug("Debug logging enabled.")
mongo_config = {}
cli_config = {}
if config:
LOG.debug("Use config file %s", config)
with open(config, 'r') as in_handle:
cli_config = yaml.load(in_handle, Loader=yaml.FullLoader)
mongo_config['mongodb'] = (mongodb or cli_config.get('mongodb') or 'scout')
mongo_config['host'] = (host or cli_config.get('host') or 'localhost')
mongo_config['port'] = (port or cli_config.get('port') or 27017)
mongo_config['username'] = username or cli_config.get('username')
mongo_config['password'] = password or cli_config.get('password')
mongo_config['authdb'] = authdb or cli_config.get('authdb') or mongo_config['mongodb']
mongo_config['omim_api_key'] = cli_config.get('omim_api_key')
LOG.info("Setting database name to %s", mongo_config['mongodb'])
LOG.debug("Setting host to %s", mongo_config['host'])
LOG.debug("Setting port to %s", mongo_config['port'])
valid_connection = check_connection(
host=mongo_config['host'],
port=mongo_config['port'],
username=mongo_config['username'],
password=mongo_config['password'],
authdb=mongo_config['authdb'],
)
LOG.info("Test if mongod is running")
if not valid_connection:
LOG.warning("Connection could not be established")
context.abort()
try:
client = get_connection(**mongo_config)
except ConnectionFailure:
context.abort()
database = client[mongo_config['mongodb']]
LOG.info("Setting up a mongo adapter")
mongo_config['client'] = client
adapter = MongoAdapter(database)
requests = []
for case_obj in adapter.case_collection.find():
# pp(case_obj)
gene_to_panels = adapter.gene_to_panels(case_obj)
variants = adapter.variant_collection.find({
'case_id': case_obj['_id'],
'category': 'snv',
'variant_type': 'clinical',
})
for variant_obj in variants:
panel_names = set()
for hgnc_id in variant_obj['hgnc_ids']:
gene_panels = gene_to_panels.get(hgnc_id, set())
panel_names = panel_names.union(gene_panels)
if panel_names:
operation = pymongo.UpdateOne(
{'_id': variant_obj['_id']},
{
'$set': {
'panels': list(panel_names)
}
})
requests.append(operation)
if len(requests) > 5000:
adapter.variant_collection.bulk_write(requests, ordered=False)
requests = []
if requests:
adapter.variant_collection.bulk_write(requests, ordered=False)
requests = []
def update_panels(context, mongodb, username, password, authdb, host, port,
loglevel, config):
"""scout: manage interactions with a scout instance."""
coloredlogs.install(level=loglevel)
LOG.info("Running scout version %s", __version__)
LOG.debug("Debug logging enabled.")
mongo_config = {}
cli_config = {}
if config:
LOG.debug("Use config file %s", config)
with open(config, 'r') as in_handle:
cli_config = yaml.load(in_handle)
mongo_config['mongodb'] = (mongodb or cli_config.get('mongodb') or 'scout')
mongo_config['host'] = (host or cli_config.get('host') or 'localhost')
mongo_config['port'] = (port or cli_config.get('port') or 27017)
mongo_config['username'] = username or cli_config.get('username')
mongo_config['password'] = password or cli_config.get('password')
mongo_config['authdb'] = authdb or cli_config.get(
'authdb') or mongo_config['mongodb']
mongo_config['omim_api_key'] = cli_config.get('omim_api_key')
LOG.info("Setting database name to %s", mongo_config['mongodb'])
LOG.debug("Setting host to %s", mongo_config['host'])
LOG.debug("Setting port to %s", mongo_config['port'])
valid_connection = check_connection(
host=mongo_config['host'],
port=mongo_config['port'],
username=mongo_config['username'],
password=mongo_config['password'],
authdb=mongo_config['authdb'],
)
LOG.info("Test if mongod is running")
if not valid_connection:
LOG.warning("Connection could not be established")
context.abort()
try:
client = get_connection(**mongo_config)
except ConnectionFailure:
context.abort()
database = client[mongo_config['mongodb']]
LOG.info("Setting up a mongo adapter")
mongo_config['client'] = client
adapter = MongoAdapter(database)
requests = []
for case_obj in adapter.case_collection.find():
# pp(case_obj)
gene_to_panels = adapter.gene_to_panels(case_obj)
variants = adapter.variant_collection.find({
'case_id': case_obj['_id'],
'category': 'snv',
'variant_type': 'clinical',
})
for variant_obj in variants:
panel_names = set()
for hgnc_id in variant_obj['hgnc_ids']:
gene_panels = gene_to_panels.get(hgnc_id, set())
panel_names = panel_names.union(gene_panels)
if panel_names:
operation = pymongo.UpdateOne(
{'_id': variant_obj['_id']},
{'$set': {
'panels': list(panel_names)
}})
requests.append(operation)
if len(requests) > 5000:
adapter.variant_collection.bulk_write(requests, ordered=False)
requests = []
if requests:
adapter.variant_collection.bulk_write(requests, ordered=False)
requests = []
def merge_users(
context, mongodb, username, password, authdb, host, port, loglevel, config, live
):
"""scout: manage interactions with a scout instance."""
coloredlogs.install(level=loglevel)
LOG.info("Running scout version %s", __version__)
LOG.debug("Debug logging enabled.")
mongo_config = {}
cli_config = {}
if config:
LOG.debug("Use config file %s", config)
with open(config, "r") as in_handle:
cli_config = yaml.load(in_handle, Loader=yaml.FullLoader)
mongo_config["mongodb"] = mongodb or cli_config.get("mongodb") or "scout"
mongo_config["host"] = host or cli_config.get("host") or "localhost"
mongo_config["port"] = port or cli_config.get("port") or 27017
mongo_config["username"] = username or cli_config.get("username")
mongo_config["password"] = password or cli_config.get("password")
mongo_config["authdb"] = (
authdb or cli_config.get("authdb") or mongo_config["mongodb"]
)
mongo_config["omim_api_key"] = cli_config.get("omim_api_key")
# always dryrun for now
dryrun = True
if live:
dryrun = False
LOG.info("Setting database name to %s", mongo_config["mongodb"])
LOG.debug("Setting host to %s", mongo_config["host"])
LOG.debug("Setting port to %s", mongo_config["port"])
valid_connection = check_connection(
host=mongo_config["host"],
port=mongo_config["port"],
username=mongo_config["username"],
password=mongo_config["password"],
authdb=mongo_config["authdb"],
)
LOG.info("Test if mongod is running")
if not valid_connection:
LOG.warning("Connection could not be established")
context.abort()
try:
client = get_connection(**mongo_config)
except ConnectionFailure:
context.abort()
database = client[mongo_config["mongodb"]]
LOG.info("Setting up a mongo adapter")
mongo_config["client"] = client
adapter = MongoAdapter(database)
## First, create all operation requests that would be needed in a live run.
user_requests = []
event_requests = []
acmg_requests = []
clinvar_requests = []
clinvar_submission_requests = []
case_requests = []
for oi_user_obj in adapter.user_collection.find({"_id": {"$type": "objectId"}}):
if not oi_user_obj.get("email"):
continue
LOG.info("===USER===")
oi_user_id = oi_user_obj.get("_id")
oi_user_email = oi_user_obj.get("email")
LOG.info("user: {}".format(oi_user_obj))
create_alt = False
alt_user_obj = adapter.user_collection.find_one({"_id": oi_user_email})
if not alt_user_obj:
create_alt = True
alt_user_obj = copy.deepcopy(oi_user_obj)
alt_user_obj["_id"] = oi_user_email
else:
LOG.info("alt user: {}".format(alt_user_obj))
merged_institutes = set()
merged_institutes.update(
alt_user_obj.get("institutes", []) + oi_user_obj.get("institutes", [])
)
LOG.info("merged institutes: {}".format(merged_institutes))
alt_user_obj["institutes"] = list(merged_institutes)
merged_roles = set()
merged_roles.update(
alt_user_obj.get("roles", []) + oi_user_obj.get("roles", [])
)
LOG.info("merged roles: {}".format(merged_roles))
alt_user_obj["roles"] = list(merged_roles)
created_at = oi_user_obj.get("created_at")
alt_created_at = alt_user_obj.get("created_at")
if (alt_created_at and created_at) and alt_created_at < created_at:
created_at = alt_created_at
if created_at:
alt_user_obj["created_at"] = created_at
accessed_at = alt_user_obj.get("accessed_at")
oi_accessed_at = oi_user_obj.get("accessed_at")
if (oi_accessed_at and accessed_at) and oi_accessed_at > accessed_at:
accessed_at = oi_accessed_at
if accessed_at:
alt_user_obj["accessed_at"] = accessed_at
if create_alt:
LOG.info("create user: {}".format(alt_user_obj))
operation = pymongo.InsertOne(alt_user_obj)
user_requests.append(operation)
else:
LOG.info("update user: {}".format(alt_user_obj))
alt_user_id = alt_user_obj.pop("_id")
operation = pymongo.UpdateOne({"_id": alt_user_id}, {"$set": alt_user_obj})
user_requests.append(operation)
# finally, delete the oi user
operation = pymongo.DeleteOne({"_id": ObjectId(str(oi_user_id))})
user_requests.append(operation)
###
### events
###
LOG.info("searching for events for user id {}".format(oi_user_id))
oi_user_events = adapter.event_collection.find(
{"user_id": ObjectId(str(oi_user_id))}
)
if oi_user_events.count() > 0:
LOG.info("===EVENTS===")
for event in oi_user_events:
LOG.info("user event: {}".format(event))
event_id = event.get("_id")
operation = pymongo.UpdateOne(
{"_id": event_id},
{
"$set": {
"user_id": oi_user_email,
"user_name": alt_user_obj.get("name"),
}
},
)
event_requests.append(operation)
###
### ACMG classifications
###
LOG.info("searching for acmg for user id {}".format(oi_user_id))
oi_user_acmg = adapter.acmg_collection.find(
{"user_id": ObjectId(str(oi_user_id))}
)
if oi_user_acmg.count() > 0:
LOG.info("===ACMG===")
for acmg in oi_user_acmg:
LOG.info("acmg: {}".format(acmg))
operation = pymongo.UpdateOne(
{"_id": acmg.get("_id")},
{
"$set": {
"user_id": oi_user_email,
"user_name": alt_user_obj.get("name"),
}
},
)
acmg_requests.append(operation)
# Clinvar
LOG.info("searching for ClinVar for user id {}".format(oi_user_id))
oi_user_clinvar = adapter.clinvar_collection.find(
{"user": ObjectId(str(oi_user_id))}
)
if oi_user_clinvar.count() > 0:
LOG.info("=== ClinVar ===")
for clinvar in oi_user_clinvar:
LOG.info("acmg: {}".format(clinvar))
operation = pymongo.UpdateOne(
{"_id": clinvar.get("_id")}, {"$set": {"user": oi_user_email}}
)
clinvar_requests.append(operation)
# clinvar_submission
LOG.info("searching for clinvar submissions for user id {}".format(oi_user_id))
oi_user_clinvars = adapter.clinvar_submission_collection.find(
{"user_id": ObjectId(str(oi_user_id))}
)
if oi_user_clinvars.count() > 0:
LOG.info("=== ClinVar submission ===")
for clinvars in oi_user_clinvars:
LOG.info("acmg: {}".format(clinvars))
operation = pymongo.UpdateOne(
{"_id": clinvars.get("_id")}, {"$set": {"user_id": oi_user_email}}
)
clinvar_submission_requests.append(operation)
###
### cases
###
LOG.info("searching for cases assigned to user id {}".format(oi_user_id))
oi_user_cases = adapter.case_collection.find(
{"assignees": ObjectId(str(oi_user_id))}
)
if oi_user_cases.count() > 0:
LOG.info("=== Case assignees ===")
for case in oi_user_cases:
LOG.info("case {} assignees: {}".format(case["_id"], case["assignees"]))
operation = pymongo.UpdateOne(
{"_id": case.get("_id"), "assignees": ObjectId(str(oi_user_id))},
{"$set": {"assignees.$": oi_user_email}},
)
case_requests.append(operation)
else:
LOG.info("No ObjectId ID user IDs found - nothing more to do.")
# if everything worked out ok with dryrun, and after getting this far on a live run,
# bulk write all proposed changes.
if event_requests:
LOG.info("event requests to execute: {}".format(event_requests))
if not dryrun:
result = adapter.event_collection.bulk_write(event_requests, ordered=False)
if acmg_requests:
LOG.info("acmg requests to execute: {}".format(acmg_requests))
if not dryrun:
result = adapter.acmg_collection.bulk_write(acmg_requests, ordered=False)
LOG.info("Modified {} ACMG.".format(result.modified_count))
if clinvar_requests:
LOG.info("clinvar requests to execute: {}".format(clinvar_requests))
if not dryrun:
result = adapter.clinvar_collection.bulk_write(
clinvar_requests, ordered=False
)
LOG.info("Modified {} ClinVar.".format(result.modified_count))
if clinvar_submission_requests:
LOG.info(
"clinvar sub requests to execute: {}".format(clinvar_submission_requests)
)
if not dryrun:
result = adapter.clinvar_submission_collection.bulk_write(
clinvar_submission_requests, ordered=False
)
LOG.info("Modified {} ClinVar submissions.".format(result.modified_count))
if case_requests:
LOG.info("case requests to execute: {}".format(case_requests))
if not dryrun:
result = adapter.case_collection.bulk_write(case_requests, ordered=False)
LOG.info("Modified {} case submissions.".format(result.modified_count))
# now delete oi user, and actually update/create alt user
if user_requests:
LOG.info("user requests to execute: {}".format(user_requests))
if not dryrun:
result = adapter.user_collection.bulk_write(user_requests, ordered=False)
LOG.info(
"Modified users with the following: {}".format(result.bulk_api_result)
)