def cli(context, mongodb, username, password, authdb, host, port, loglevel,
config, demo):
"""scout: manage interactions with a scout instance."""
# log_format = "%(message)s" if sys.stdout.isatty() else None
log_format = None
coloredlogs.install(level=loglevel, fmt=log_format)
LOG.info("Running scout version %s", __version__)
LOG.debug("Debug logging enabled.")
mongo_config = {}
cli_config = {}
if config:
LOG.debug("Use config file %s", config)
with open(config, 'r') as in_handle:
cli_config = yaml.load(in_handle)
mongo_config['mongodb'] = (mongodb or cli_config.get('mongodb') or 'scout')
if demo:
mongo_config['mongodb'] = 'scout-demo'
mongo_config['host'] = (host or cli_config.get('host') or 'localhost')
mongo_config['port'] = (port or cli_config.get('port') or 27017)
mongo_config['username'] = username or cli_config.get('username')
mongo_config['password'] = password or cli_config.get('password')
mongo_config['authdb'] = authdb or cli_config.get(
'authdb') or mongo_config['mongodb']
mongo_config['omim_api_key'] = cli_config.get('omim_api_key')
if context.invoked_subcommand in ('setup', 'serve'):
mongo_config['adapter'] = None
else:
LOG.info("Setting database name to %s", mongo_config['mongodb'])
LOG.debug("Setting host to %s", mongo_config['host'])
LOG.debug("Setting port to %s", mongo_config['port'])
try:
client = get_connection(**mongo_config)
except ConnectionFailure:
context.abort()
database = client[mongo_config['mongodb']]
LOG.info("Setting up a mongo adapter")
mongo_config['client'] = client
adapter = MongoAdapter(database)
mongo_config['adapter'] = adapter
LOG.info("Check if authenticated...")
try:
for ins_obj in adapter.institutes():
pass
except OperationFailure as err:
LOG.info("User not authenticated")
context.abort()
context.obj = mongo_config
def adapter(request, pymongo_client):
"""Get an adapter connected to mongo database"""
LOG.info("Connecting to database...")
mongo_client = pymongo_client
database = mongo_client[DATABASE]
mongo_adapter = PymongoAdapter(database)
mongo_adapter.setup(database)
LOG.info("Connected to database")
return mongo_adapter
def real_adapter(request, real_pymongo_client):
"""Get an adapter connected to mongo database"""
mongo_client = real_pymongo_client
LOG.info("Connecting to database %s", REAL_DATABASE)
database = mongo_client[REAL_DATABASE]
mongo_adapter = PymongoAdapter(database)
mongo_adapter.load_indexes()
LOG.info("Connected to database")
return mongo_adapter
def cli(context, mongodb, username, password, authdb, host, port, loglevel,
config, demo):
"""scout: manage interactions with a scout instance."""
coloredlogs.install(level=loglevel)
log.info("Running scout version %s", __version__)
log.debug("Debug logging enabled.")
mongo_config = {}
cli_config = {}
if config:
log.debug("Use config file %s", config)
with open(config, 'r') as in_handle:
cli_config = yaml.load(in_handle)
mongo_config['mongodb'] = (mongodb or cli_config.get('mongodb') or 'scout')
if demo:
mongo_config['mongodb'] = 'scout-demo'
mongo_config['host'] = (host or cli_config.get('host') or 'localhost')
mongo_config['port'] = (port or cli_config.get('port') or 27017)
mongo_config['username'] = username or cli_config.get('username')
mongo_config['password'] = password or cli_config.get('password')
mongo_config['authdb'] = authdb or cli_config.get(
'authdb') or mongo_config['mongodb']
# mongo uri looks like:
# mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]]
if context.invoked_subcommand in ('setup', 'serve'):
mongo_config['adapter'] = None
else:
log.info("Setting database name to %s", mongo_config['mongodb'])
log.debug("Setting host to %s", mongo_config['host'])
log.debug("Setting port to %s", mongo_config['port'])
valid_connection = check_connection(
host=mongo_config['host'],
port=mongo_config['port'],
username=mongo_config['username'],
password=mongo_config['password'],
authdb=mongo_config['authdb'],
)
log.info("Test if mongod is running")
if not valid_connection:
log.warning("Connection could not be established")
context.abort()
try:
client = get_connection(**mongo_config)
except ConnectionFailure:
context.abort()
database = client[mongo_config['mongodb']]
log.info("Setting up a mongo adapter")
mongo_config['adapter'] = MongoAdapter(database)
context.obj = mongo_config
def cli(context, mongodb, username, password, host, port, logfile, loglevel,
config, demo):
"""scout: manage interactions with a scout instance."""
coloredlogs.install(log_level=loglevel)
log.info("Running scout version %s", __version__)
mongo_configs = {}
configs = {}
if config:
log.debug("Use config file {0}".format(config))
with open(config, 'r') as in_handle:
configs = yaml.load(in_handle)
mongo_configs['mongodb'] = (mongodb or configs.get('mongodb') or 'scout')
if demo:
mongo_configs['mongodb'] = 'scout-demo'
mongo_configs['host'] = (host or configs.get('host') or 'localhost')
mongo_configs['port'] = (port or configs.get('port') or 27017)
mongo_configs['username'] = username or configs.get('username')
mongo_configs['password'] = password or configs.get('password')
# mongo uri looks like:
# mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]]
if context.invoked_subcommand not in ('setup', 'serve'):
log.info("Setting database name to %s", mongo_configs['mongodb'])
log.debug("Setting host to {0}".format(mongo_configs['host']))
log.debug("Setting port to {0}".format(mongo_configs['port']))
try:
client = get_connection(**mongo_configs)
except ConnectionFailure:
context.abort()
database = client[mongo_configs['mongodb']]
log.info("Test if mongod is running")
try:
database.test.find_one()
except ServerSelectionTimeoutError as err:
log.warning("Connection could not be established")
context.abort()
log.info("Setting up a mongo adapter")
mongo_adapter = MongoAdapter(database)
mongo_configs['adapter'] = mongo_adapter
else:
mongo_configs['adapter'] = None
context.obj = mongo_configs
def setup(context):
"""
Setup scout instances.
"""
context.obj['institute_name'] = 'cust000'
context.obj['user_name'] = 'Clark Kent'
context.obj['user_mail'] = '*****@*****.**'
if context.invoked_subcommand == 'demo':
# Update context.obj settings here
LOG.info("Change database name to scout-demo")
context.obj['mongodb'] = 'scout-demo'
LOG.info("Setting database name to %s", context.obj['mongodb'])
LOG.debug("Setting host to %s", context.obj['host'])
LOG.debug("Setting port to %s", context.obj['port'])
try:
client = get_connection(
host=context.obj['host'],
port=context.obj['port'],
username=context.obj['username'],
password=context.obj['password'],
mongodb = context.obj['mongodb']
)
except ConnectionFailure:
context.abort()
LOG.info("connecting to database %s", context.obj['mongodb'])
database = client[context.obj['mongodb']]
LOG.info("Test if mongod is running")
try:
database.test.find_one()
except ServerSelectionTimeoutError as err:
LOG.warning("Connection could not be established")
LOG.warning("Please check if mongod is running")
context.abort()
LOG.info("Setting up a mongo adapter")
mongo_adapter = MongoAdapter(database)
context.obj['adapter'] = mongo_adapter
def update_panels(context, mongodb, username, password, authdb, host, port,
loglevel, config):
"""scout: manage interactions with a scout instance."""
coloredlogs.install(level=loglevel)
LOG.info("Running scout version %s", __version__)
LOG.debug("Debug logging enabled.")
mongo_config = {}
cli_config = {}
if config:
LOG.debug("Use config file %s", config)
with open(config, "r") as in_handle:
cli_config = yaml.load(in_handle, Loader=yaml.FullLoader)
mongo_config["mongodb"] = mongodb or cli_config.get("mongodb") or "scout"
mongo_config["host"] = host or cli_config.get("host") or "localhost"
mongo_config["port"] = port or cli_config.get("port") or 27017
mongo_config["username"] = username or cli_config.get("username")
mongo_config["password"] = password or cli_config.get("password")
mongo_config["authdb"] = (authdb or cli_config.get("authdb")
or mongo_config["mongodb"])
mongo_config["omim_api_key"] = cli_config.get("omim_api_key")
LOG.info("Setting database name to %s", mongo_config["mongodb"])
LOG.debug("Setting host to %s", mongo_config["host"])
LOG.debug("Setting port to %s", mongo_config["port"])
valid_connection = check_connection(
host=mongo_config["host"],
port=mongo_config["port"],
username=mongo_config["username"],
password=mongo_config["password"],
authdb=mongo_config["authdb"],
)
LOG.info("Test if mongod is running")
if not valid_connection:
LOG.warning("Connection could not be established")
context.abort()
try:
client = get_connection(**mongo_config)
except ConnectionFailure:
context.abort()
database = client[mongo_config["mongodb"]]
LOG.info("Setting up a mongo adapter")
mongo_config["client"] = client
adapter = MongoAdapter(database)
requests = []
for case_obj in adapter.case_collection.find():
# pp(case_obj)
gene_to_panels = adapter.gene_to_panels(case_obj)
variants = adapter.variant_collection.find({
"case_id": case_obj["_id"],
"category": "snv",
"variant_type": "clinical"
})
for variant_obj in variants:
panel_names = set()
for hgnc_id in variant_obj["hgnc_ids"]:
gene_panels = gene_to_panels.get(hgnc_id, set())
panel_names = panel_names.union(gene_panels)
if panel_names:
operation = pymongo.UpdateOne(
{"_id": variant_obj["_id"]},
{"$set": {
"panels": list(panel_names)
}})
requests.append(operation)
if len(requests) > 5000:
adapter.variant_collection.bulk_write(requests, ordered=False)
requests = []
if requests:
adapter.variant_collection.bulk_write(requests, ordered=False)
requests = []
def update_panels(context, mongodb, username, password, authdb, host, port, loglevel, config):
"""scout: manage interactions with a scout instance."""
coloredlogs.install(level=loglevel)
LOG.info("Running scout version %s", __version__)
LOG.debug("Debug logging enabled.")
mongo_config = {}
cli_config = {}
if config:
LOG.debug("Use config file %s", config)
with open(config, 'r') as in_handle:
cli_config = yaml.load(in_handle, Loader=yaml.FullLoader)
mongo_config['mongodb'] = (mongodb or cli_config.get('mongodb') or 'scout')
mongo_config['host'] = (host or cli_config.get('host') or 'localhost')
mongo_config['port'] = (port or cli_config.get('port') or 27017)
mongo_config['username'] = username or cli_config.get('username')
mongo_config['password'] = password or cli_config.get('password')
mongo_config['authdb'] = authdb or cli_config.get('authdb') or mongo_config['mongodb']
mongo_config['omim_api_key'] = cli_config.get('omim_api_key')
LOG.info("Setting database name to %s", mongo_config['mongodb'])
LOG.debug("Setting host to %s", mongo_config['host'])
LOG.debug("Setting port to %s", mongo_config['port'])
valid_connection = check_connection(
host=mongo_config['host'],
port=mongo_config['port'],
username=mongo_config['username'],
password=mongo_config['password'],
authdb=mongo_config['authdb'],
)
LOG.info("Test if mongod is running")
if not valid_connection:
LOG.warning("Connection could not be established")
context.abort()
try:
client = get_connection(**mongo_config)
except ConnectionFailure:
context.abort()
database = client[mongo_config['mongodb']]
LOG.info("Setting up a mongo adapter")
mongo_config['client'] = client
adapter = MongoAdapter(database)
requests = []
for case_obj in adapter.case_collection.find():
# pp(case_obj)
gene_to_panels = adapter.gene_to_panels(case_obj)
variants = adapter.variant_collection.find({
'case_id': case_obj['_id'],
'category': 'snv',
'variant_type': 'clinical',
})
for variant_obj in variants:
panel_names = set()
for hgnc_id in variant_obj['hgnc_ids']:
gene_panels = gene_to_panels.get(hgnc_id, set())
panel_names = panel_names.union(gene_panels)
if panel_names:
operation = pymongo.UpdateOne(
{'_id': variant_obj['_id']},
{
'$set': {
'panels': list(panel_names)
}
})
requests.append(operation)
if len(requests) > 5000:
adapter.variant_collection.bulk_write(requests, ordered=False)
requests = []
if requests:
adapter.variant_collection.bulk_write(requests, ordered=False)
requests = []
def update_panels(context, mongodb, username, password, authdb, host, port,
loglevel, config):
"""scout: manage interactions with a scout instance."""
coloredlogs.install(level=loglevel)
LOG.info("Running scout version %s", __version__)
LOG.debug("Debug logging enabled.")
mongo_config = {}
cli_config = {}
if config:
LOG.debug("Use config file %s", config)
with open(config, 'r') as in_handle:
cli_config = yaml.load(in_handle)
mongo_config['mongodb'] = (mongodb or cli_config.get('mongodb') or 'scout')
mongo_config['host'] = (host or cli_config.get('host') or 'localhost')
mongo_config['port'] = (port or cli_config.get('port') or 27017)
mongo_config['username'] = username or cli_config.get('username')
mongo_config['password'] = password or cli_config.get('password')
mongo_config['authdb'] = authdb or cli_config.get(
'authdb') or mongo_config['mongodb']
mongo_config['omim_api_key'] = cli_config.get('omim_api_key')
LOG.info("Setting database name to %s", mongo_config['mongodb'])
LOG.debug("Setting host to %s", mongo_config['host'])
LOG.debug("Setting port to %s", mongo_config['port'])
valid_connection = check_connection(
host=mongo_config['host'],
port=mongo_config['port'],
username=mongo_config['username'],
password=mongo_config['password'],
authdb=mongo_config['authdb'],
)
LOG.info("Test if mongod is running")
if not valid_connection:
LOG.warning("Connection could not be established")
context.abort()
try:
client = get_connection(**mongo_config)
except ConnectionFailure:
context.abort()
database = client[mongo_config['mongodb']]
LOG.info("Setting up a mongo adapter")
mongo_config['client'] = client
adapter = MongoAdapter(database)
requests = []
for case_obj in adapter.case_collection.find():
# pp(case_obj)
gene_to_panels = adapter.gene_to_panels(case_obj)
variants = adapter.variant_collection.find({
'case_id': case_obj['_id'],
'category': 'snv',
'variant_type': 'clinical',
})
for variant_obj in variants:
panel_names = set()
for hgnc_id in variant_obj['hgnc_ids']:
gene_panels = gene_to_panels.get(hgnc_id, set())
panel_names = panel_names.union(gene_panels)
if panel_names:
operation = pymongo.UpdateOne(
{'_id': variant_obj['_id']},
{'$set': {
'panels': list(panel_names)
}})
requests.append(operation)
if len(requests) > 5000:
adapter.variant_collection.bulk_write(requests, ordered=False)
requests = []
if requests:
adapter.variant_collection.bulk_write(requests, ordered=False)
requests = []
def setup(context):
"""
Setup scout instances.
"""
context.obj['institute_name'] = 'cust000'
context.obj['user_name'] = 'Clark Kent'
context.obj['user_mail'] = '*****@*****.**'
if context.invoked_subcommand == 'demo':
# Update context.obj settings here
log.info("Change database name to scout-demo")
context.obj['mongodb'] = 'scout-demo'
log.info("Loading hgnc genes from %s", hgnc_reduced_path)
context.obj['hgnc'] = get_file_handle(hgnc_reduced_path)
context.obj['hgnc38'] = get_file_handle(hgnc_reduced_path)
log.info("Loading exac genes from %s", exac_reduced_path)
context.obj['exac'] = get_file_handle(exac_reduced_path)
context.obj['exac38'] = get_file_handle(exac_reduced_path)
log.info("Loading mim2gene info from %s", mim2gene_reduced_path)
context.obj['mim2gene'] = get_file_handle(mim2gene_reduced_path)
context.obj['mim2gene38'] = get_file_handle(mim2gene_reduced_path)
log.info("Loading genemap info from %s", genemap2_reduced_path)
context.obj['genemap2'] = get_file_handle(genemap2_reduced_path)
context.obj['genemap2_38'] = get_file_handle(genemap2_reduced_path)
log.info("Loading hpo gene info from %s", hpogenes_reduced_path)
context.obj['hpogenes'] = get_file_handle(hpogenes_reduced_path)
context.obj['hpogenes_38'] = get_file_handle(hpogenes_reduced_path)
log.info("Loading hpo disease info from %s", hpo_phenotype_to_terms_reduced_path)
context.obj['hpodiseases'] = get_file_handle(hpo_phenotype_to_terms_reduced_path)
log.info("Loading hpo terms from %s", hpoterms_reduced_path)
context.obj['hpo_terms'] = get_file_handle(hpoterms_reduced_path)
log.info("Loading omim disease info from %s", genemap2_reduced_path)
context.obj['disease_terms'] = get_file_handle(genemap2_reduced_path)
log.info("Loading transcripts build 37 info from %s", transcripts37_reduced_path)
context.obj['transcripts37'] = get_file_handle(transcripts37_reduced_path)
# log.info("Loading transcripts build 38 info from %s", transcripts38_path)
# context.obj['transcripts38'] = get_file_handle(transcripts38_path)
else:
log.info("Loading hgnc genes from %s", hgnc_path)
context.obj['hgnc'] = get_file_handle(hgnc_path)
context.obj['hgnc38'] = get_file_handle(hgnc_path)
log.info("Loading exac genes from %s", exac_path)
context.obj['exac'] = get_file_handle(exac_path)
context.obj['exac38'] = get_file_handle(exac_path)
log.info("Loading mim2gene info from %s", mim2gene_path)
context.obj['mim2gene'] = get_file_handle(mim2gene_path)
context.obj['mim2gene38'] = get_file_handle(mim2gene_path)
log.info("Loading genemap info from %s", genemap2_path)
context.obj['genemap2'] = get_file_handle(genemap2_path)
context.obj['genemap2_38'] = get_file_handle(genemap2_path)
log.info("Loading hpo gene info from %s", hpogenes_path)
context.obj['hpogenes'] = get_file_handle(hpogenes_path)
context.obj['hpogenes_38'] = get_file_handle(hpogenes_path)
log.info("Loading hpo disease info from %s", hpo_phenotype_to_terms_path)
context.obj['hpodiseases'] = get_file_handle(hpo_phenotype_to_terms_path)
log.info("Loading hpo terms from %s", hpoterms_path)
context.obj['hpo_terms'] = get_file_handle(hpoterms_path)
log.info("Loading omim disease info from %s", genemap2_path)
context.obj['disease_terms'] = get_file_handle(genemap2_path)
log.info("Loading transcripts build 37 info from %s", transcripts37_path)
context.obj['transcripts37'] = get_file_handle(transcripts37_path)
log.info("Loading transcripts build 38 info from %s", transcripts38_path)
context.obj['transcripts38'] = get_file_handle(transcripts38_path)
log.info("Setting database name to %s", context.obj['mongodb'])
log.debug("Setting host to %s", context.obj['host'])
log.debug("Setting port to %s", context.obj['port'])
try:
client = get_connection(
host=context.obj['host'],
port=context.obj['port'],
username=context.obj['username'],
password=context.obj['password'],
)
except ConnectionFailure:
context.abort()
log.info("connecting to database %s", context.obj['mongodb'])
database = client[context.obj['mongodb']]
log.info("Test if mongod is running")
try:
database.test.find_one()
except ServerSelectionTimeoutError as err:
log.warning("Connection could not be established")
log.warning("Please check if mongod is running")
context.abort()
log.info("Setting up a mongo adapter")
mongo_adapter = MongoAdapter(database)
context.obj['adapter'] = mongo_adapter
def merge_users(
context, mongodb, username, password, authdb, host, port, loglevel, config, live
):
"""scout: manage interactions with a scout instance."""
coloredlogs.install(level=loglevel)
LOG.info("Running scout version %s", __version__)
LOG.debug("Debug logging enabled.")
mongo_config = {}
cli_config = {}
if config:
LOG.debug("Use config file %s", config)
with open(config, "r") as in_handle:
cli_config = yaml.load(in_handle, Loader=yaml.FullLoader)
mongo_config["mongodb"] = mongodb or cli_config.get("mongodb") or "scout"
mongo_config["host"] = host or cli_config.get("host") or "localhost"
mongo_config["port"] = port or cli_config.get("port") or 27017
mongo_config["username"] = username or cli_config.get("username")
mongo_config["password"] = password or cli_config.get("password")
mongo_config["authdb"] = (
authdb or cli_config.get("authdb") or mongo_config["mongodb"]
)
mongo_config["omim_api_key"] = cli_config.get("omim_api_key")
# always dryrun for now
dryrun = True
if live:
dryrun = False
LOG.info("Setting database name to %s", mongo_config["mongodb"])
LOG.debug("Setting host to %s", mongo_config["host"])
LOG.debug("Setting port to %s", mongo_config["port"])
valid_connection = check_connection(
host=mongo_config["host"],
port=mongo_config["port"],
username=mongo_config["username"],
password=mongo_config["password"],
authdb=mongo_config["authdb"],
)
LOG.info("Test if mongod is running")
if not valid_connection:
LOG.warning("Connection could not be established")
context.abort()
try:
client = get_connection(**mongo_config)
except ConnectionFailure:
context.abort()
database = client[mongo_config["mongodb"]]
LOG.info("Setting up a mongo adapter")
mongo_config["client"] = client
adapter = MongoAdapter(database)
## First, create all operation requests that would be needed in a live run.
user_requests = []
event_requests = []
acmg_requests = []
clinvar_requests = []
clinvar_submission_requests = []
case_requests = []
for oi_user_obj in adapter.user_collection.find({"_id": {"$type": "objectId"}}):
if not oi_user_obj.get("email"):
continue
LOG.info("===USER===")
oi_user_id = oi_user_obj.get("_id")
oi_user_email = oi_user_obj.get("email")
LOG.info("user: {}".format(oi_user_obj))
create_alt = False
alt_user_obj = adapter.user_collection.find_one({"_id": oi_user_email})
if not alt_user_obj:
create_alt = True
alt_user_obj = copy.deepcopy(oi_user_obj)
alt_user_obj["_id"] = oi_user_email
else:
LOG.info("alt user: {}".format(alt_user_obj))
merged_institutes = set()
merged_institutes.update(
alt_user_obj.get("institutes", []) + oi_user_obj.get("institutes", [])
)
LOG.info("merged institutes: {}".format(merged_institutes))
alt_user_obj["institutes"] = list(merged_institutes)
merged_roles = set()
merged_roles.update(
alt_user_obj.get("roles", []) + oi_user_obj.get("roles", [])
)
LOG.info("merged roles: {}".format(merged_roles))
alt_user_obj["roles"] = list(merged_roles)
created_at = oi_user_obj.get("created_at")
alt_created_at = alt_user_obj.get("created_at")
if (alt_created_at and created_at) and alt_created_at < created_at:
created_at = alt_created_at
if created_at:
alt_user_obj["created_at"] = created_at
accessed_at = alt_user_obj.get("accessed_at")
oi_accessed_at = oi_user_obj.get("accessed_at")
if (oi_accessed_at and accessed_at) and oi_accessed_at > accessed_at:
accessed_at = oi_accessed_at
if accessed_at:
alt_user_obj["accessed_at"] = accessed_at
if create_alt:
LOG.info("create user: {}".format(alt_user_obj))
operation = pymongo.InsertOne(alt_user_obj)
user_requests.append(operation)
else:
LOG.info("update user: {}".format(alt_user_obj))
alt_user_id = alt_user_obj.pop("_id")
operation = pymongo.UpdateOne({"_id": alt_user_id}, {"$set": alt_user_obj})
user_requests.append(operation)
# finally, delete the oi user
operation = pymongo.DeleteOne({"_id": ObjectId(str(oi_user_id))})
user_requests.append(operation)
###
### events
###
LOG.info("searching for events for user id {}".format(oi_user_id))
oi_user_events = adapter.event_collection.find(
{"user_id": ObjectId(str(oi_user_id))}
)
if oi_user_events.count() > 0:
LOG.info("===EVENTS===")
for event in oi_user_events:
LOG.info("user event: {}".format(event))
event_id = event.get("_id")
operation = pymongo.UpdateOne(
{"_id": event_id},
{
"$set": {
"user_id": oi_user_email,
"user_name": alt_user_obj.get("name"),
}
},
)
event_requests.append(operation)
###
### ACMG classifications
###
LOG.info("searching for acmg for user id {}".format(oi_user_id))
oi_user_acmg = adapter.acmg_collection.find(
{"user_id": ObjectId(str(oi_user_id))}
)
if oi_user_acmg.count() > 0:
LOG.info("===ACMG===")
for acmg in oi_user_acmg:
LOG.info("acmg: {}".format(acmg))
operation = pymongo.UpdateOne(
{"_id": acmg.get("_id")},
{
"$set": {
"user_id": oi_user_email,
"user_name": alt_user_obj.get("name"),
}
},
)
acmg_requests.append(operation)
# Clinvar
LOG.info("searching for ClinVar for user id {}".format(oi_user_id))
oi_user_clinvar = adapter.clinvar_collection.find(
{"user": ObjectId(str(oi_user_id))}
)
if oi_user_clinvar.count() > 0:
LOG.info("=== ClinVar ===")
for clinvar in oi_user_clinvar:
LOG.info("acmg: {}".format(clinvar))
operation = pymongo.UpdateOne(
{"_id": clinvar.get("_id")}, {"$set": {"user": oi_user_email}}
)
clinvar_requests.append(operation)
# clinvar_submission
LOG.info("searching for clinvar submissions for user id {}".format(oi_user_id))
oi_user_clinvars = adapter.clinvar_submission_collection.find(
{"user_id": ObjectId(str(oi_user_id))}
)
if oi_user_clinvars.count() > 0:
LOG.info("=== ClinVar submission ===")
for clinvars in oi_user_clinvars:
LOG.info("acmg: {}".format(clinvars))
operation = pymongo.UpdateOne(
{"_id": clinvars.get("_id")}, {"$set": {"user_id": oi_user_email}}
)
clinvar_submission_requests.append(operation)
###
### cases
###
LOG.info("searching for cases assigned to user id {}".format(oi_user_id))
oi_user_cases = adapter.case_collection.find(
{"assignees": ObjectId(str(oi_user_id))}
)
if oi_user_cases.count() > 0:
LOG.info("=== Case assignees ===")
for case in oi_user_cases:
LOG.info("case {} assignees: {}".format(case["_id"], case["assignees"]))
operation = pymongo.UpdateOne(
{"_id": case.get("_id"), "assignees": ObjectId(str(oi_user_id))},
{"$set": {"assignees.$": oi_user_email}},
)
case_requests.append(operation)
else:
LOG.info("No ObjectId ID user IDs found - nothing more to do.")
# if everything worked out ok with dryrun, and after getting this far on a live run,
# bulk write all proposed changes.
if event_requests:
LOG.info("event requests to execute: {}".format(event_requests))
if not dryrun:
result = adapter.event_collection.bulk_write(event_requests, ordered=False)
if acmg_requests:
LOG.info("acmg requests to execute: {}".format(acmg_requests))
if not dryrun:
result = adapter.acmg_collection.bulk_write(acmg_requests, ordered=False)
LOG.info("Modified {} ACMG.".format(result.modified_count))
if clinvar_requests:
LOG.info("clinvar requests to execute: {}".format(clinvar_requests))
if not dryrun:
result = adapter.clinvar_collection.bulk_write(
clinvar_requests, ordered=False
)
LOG.info("Modified {} ClinVar.".format(result.modified_count))
if clinvar_submission_requests:
LOG.info(
"clinvar sub requests to execute: {}".format(clinvar_submission_requests)
)
if not dryrun:
result = adapter.clinvar_submission_collection.bulk_write(
clinvar_submission_requests, ordered=False
)
LOG.info("Modified {} ClinVar submissions.".format(result.modified_count))
if case_requests:
LOG.info("case requests to execute: {}".format(case_requests))
if not dryrun:
result = adapter.case_collection.bulk_write(case_requests, ordered=False)
LOG.info("Modified {} case submissions.".format(result.modified_count))
# now delete oi user, and actually update/create alt user
if user_requests:
LOG.info("user requests to execute: {}".format(user_requests))
if not dryrun:
result = adapter.user_collection.bulk_write(user_requests, ordered=False)
LOG.info(
"Modified users with the following: {}".format(result.bulk_api_result)
)
def check_panels(context, mongodb, username, password, authdb, host, port,
loglevel, config):
"""scout: manage interactions with a scout instance."""
coloredlogs.install(level=loglevel)
LOG.info("Running scout version %s", __version__)
LOG.debug("Debug logging enabled.")
mongo_config = {}
cli_config = {}
if config:
LOG.debug("Use config file %s", config)
with open(config, "r") as in_handle:
cli_config = yaml.load(in_handle, Loader=yaml.SafeLoader)
mongo_config["mongodb"] = mongodb or cli_config.get("mongodb") or "scout"
mongo_config["host"] = host or cli_config.get("host") or "localhost"
mongo_config["port"] = port or cli_config.get("port") or 27017
mongo_config["username"] = username or cli_config.get("username")
mongo_config["password"] = password or cli_config.get("password")
mongo_config["authdb"] = authdb or cli_config.get(
"authdb") or mongo_config["mongodb"]
mongo_config["omim_api_key"] = cli_config.get("omim_api_key")
LOG.info("Setting database name to %s", mongo_config["mongodb"])
LOG.debug("Setting host to %s", mongo_config["host"])
LOG.debug("Setting port to %s", mongo_config["port"])
# Connection validity will be checked in adapter.client
client = get_connection(**mongo_config)
database = client[mongo_config["mongodb"]]
LOG.info("Setting up a mongo adapter")
mongo_config["client"] = client
adapter = MongoAdapter(database)
cases_compromised = 0
compromised_cases = []
cases = 0
panel_genes_incorrect = {}
variants_found = ""
panel_issue_found = ""
for case_obj in adapter.case_collection.find(no_cursor_timeout=True).sort(
"updated_at", -1):
if case_obj.get("updated_at") < datetime.datetime(2020, 9, 8):
break
# if cases_compromised >= 20:
# break
case_compromised = False
for panel_info in case_obj.get("panels", []):
panel_name = panel_info["panel_name"]
if not panel_info.get("is_default", False):
continue
if panel_genes_incorrect.get(panel_name, None) is not None:
if len(panel_genes_incorrect[panel_name]) > 0:
case_compromised = True
else:
panel_obj = adapter.gene_panel(panel_name)
if not panel_obj:
LOG.warning("Panel: {0} does not exist in database".format(
panel_name))
continue
symbols_compromised = []
for panel_gene in panel_obj["genes"]:
gene_hgnc_id = panel_gene["hgnc_id"]
hgnc_symbol_panel = panel_gene["symbol"]
hgnc_gene_from_id = adapter.hgnc_gene(gene_hgnc_id)
hgnc_symbol_hgnc = hgnc_gene_from_id.get("hgnc_symbol")
if hgnc_symbol_panel != hgnc_symbol_hgnc:
case_compromised = True
symbols_compromised.append(hgnc_symbol_hgnc)
panel_issue_found += "Case {} panel {} gene symbol {} differes from current hgnc symbol {}\n".format(
case_obj["display_name"],
panel_name,
hgnc_symbol_panel,
hgnc_symbol_hgnc,
)
panel_genes_incorrect[panel_name] = symbols_compromised
if panel_genes_incorrect.get(panel_name, None):
for variant in adapter.variants(
case_obj["_id"],
query={
"hgnc_symbols": panel_genes_incorrect[panel_name],
},
nr_of_variants=-1,
sort_key="rank_score",
):
if variant.get("rank_score") > 15:
variants_found += "{} case {} {} variant {} {} {}.\n".format(
variant["institute"],
case_obj["_id"],
case_obj["display_name"],
variant["display_name"],
variant["rank_score"],
variant["hgnc_symbols"],
)
cases += 1
if case_compromised:
cases_compromised += 1
compromised_cases.append(case_obj["_id"])
LOG.debug("Case {} compromised.".format(
case_obj["display_name"], ))
else:
LOG.debug("Case {} ok.".format(case_obj["display_name"], ))
LOG.warning("{} cases out of {} were compromised.".format(
cases_compromised, cases))
for panel, symbols in panel_genes_incorrect.items():
LOG.warning("Panel {} had {} symbols compromised".format(
panel, len(symbols)))
LOG.warning("Panel issues:\n{}".format(panel_issue_found))
LOG.warning("Variants of interest: {} ".format(variants_found))
