def cli(context, mongodb, username, password, authdb, host, port, loglevel, config, demo): """scout: manage interactions with a scout instance.""" coloredlogs.install(level=loglevel) log.info("Running scout version %s", __version__) log.debug("Debug logging enabled.") mongo_config = {} cli_config = {} if config: log.debug("Use config file %s", config) with open(config, 'r') as in_handle: cli_config = yaml.load(in_handle) mongo_config['mongodb'] = (mongodb or cli_config.get('mongodb') or 'scout') if demo: mongo_config['mongodb'] = 'scout-demo' mongo_config['host'] = (host or cli_config.get('host') or 'localhost') mongo_config['port'] = (port or cli_config.get('port') or 27017) mongo_config['username'] = username or cli_config.get('username') mongo_config['password'] = password or cli_config.get('password') mongo_config['authdb'] = authdb or cli_config.get( 'authdb') or mongo_config['mongodb'] # mongo uri looks like: # mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]] if context.invoked_subcommand in ('setup', 'serve'): mongo_config['adapter'] = None else: log.info("Setting database name to %s", mongo_config['mongodb']) log.debug("Setting host to %s", mongo_config['host']) log.debug("Setting port to %s", mongo_config['port']) valid_connection = check_connection( host=mongo_config['host'], port=mongo_config['port'], username=mongo_config['username'], password=mongo_config['password'], authdb=mongo_config['authdb'], ) log.info("Test if mongod is running") if not valid_connection: log.warning("Connection could not be established") context.abort() try: client = get_connection(**mongo_config) except ConnectionFailure: context.abort() database = client[mongo_config['mongodb']] log.info("Setting up a mongo adapter") mongo_config['adapter'] = MongoAdapter(database) context.obj = mongo_config
def cli(context, mongodb, username, password, authdb, host, port, loglevel, config, demo): """scout: manage interactions with a scout instance.""" # log_format = "%(message)s" if sys.stdout.isatty() else None log_format = None coloredlogs.install(level=loglevel, fmt=log_format) LOG.info("Running scout version %s", __version__) LOG.debug("Debug logging enabled.") mongo_config = {} cli_config = {} if config: LOG.debug("Use config file %s", config) with open(config, 'r') as in_handle: cli_config = yaml.load(in_handle) mongo_config['mongodb'] = (mongodb or cli_config.get('mongodb') or 'scout') if demo: mongo_config['mongodb'] = 'scout-demo' mongo_config['host'] = (host or cli_config.get('host') or 'localhost') mongo_config['port'] = (port or cli_config.get('port') or 27017) mongo_config['username'] = username or cli_config.get('username') mongo_config['password'] = password or cli_config.get('password') mongo_config['authdb'] = authdb or cli_config.get( 'authdb') or mongo_config['mongodb'] mongo_config['omim_api_key'] = cli_config.get('omim_api_key') if context.invoked_subcommand in ('setup', 'serve'): mongo_config['adapter'] = None else: LOG.info("Setting database name to %s", mongo_config['mongodb']) LOG.debug("Setting host to %s", mongo_config['host']) LOG.debug("Setting port to %s", mongo_config['port']) try: client = get_connection(**mongo_config) except ConnectionFailure: context.abort() database = client[mongo_config['mongodb']] LOG.info("Setting up a mongo adapter") mongo_config['client'] = client adapter = MongoAdapter(database) mongo_config['adapter'] = adapter LOG.info("Check if authenticated...") try: for ins_obj in adapter.institutes(): pass except OperationFailure as err: LOG.info("User not authenticated") context.abort() context.obj = mongo_config
def cli(context, mongodb, username, password, host, port, logfile, loglevel, config, demo): """scout: manage interactions with a scout instance.""" coloredlogs.install(log_level=loglevel) log.info("Running scout version %s", __version__) mongo_configs = {} configs = {} if config: log.debug("Use config file {0}".format(config)) with open(config, 'r') as in_handle: configs = yaml.load(in_handle) mongo_configs['mongodb'] = (mongodb or configs.get('mongodb') or 'scout') if demo: mongo_configs['mongodb'] = 'scout-demo' mongo_configs['host'] = (host or configs.get('host') or 'localhost') mongo_configs['port'] = (port or configs.get('port') or 27017) mongo_configs['username'] = username or configs.get('username') mongo_configs['password'] = password or configs.get('password') # mongo uri looks like: # mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]] if context.invoked_subcommand not in ('setup', 'serve'): log.info("Setting database name to %s", mongo_configs['mongodb']) log.debug("Setting host to {0}".format(mongo_configs['host'])) log.debug("Setting port to {0}".format(mongo_configs['port'])) try: client = get_connection(**mongo_configs) except ConnectionFailure: context.abort() database = client[mongo_configs['mongodb']] log.info("Test if mongod is running") try: database.test.find_one() except ServerSelectionTimeoutError as err: log.warning("Connection could not be established") context.abort() log.info("Setting up a mongo adapter") mongo_adapter = MongoAdapter(database) mongo_configs['adapter'] = mongo_adapter else: mongo_configs['adapter'] = None context.obj = mongo_configs
def init_app(app): """Initialize from flask""" uri = app.config.get("MONGO_URI", None) db_name = app.config.get("MONGO_DBNAME", "scout") client = get_connection( host=app.config.get("MONGO_HOST", "localhost"), port=app.config.get("MONGO_PORT", 27017), username=app.config.get("MONGO_USERNAME", None), password=app.config.get("MONGO_PASSWORD", None), uri=uri, mongodb=db_name, ) app.config["MONGO_DATABASE"] = client[db_name] app.config["MONGO_CLIENT"] = client
def setup(context, institute, user_mail, user_name): """ Setup scout instances. """ context.obj['institute_name'] = institute context.obj['user_name'] = user_name context.obj['user_mail'] = user_mail if context.invoked_subcommand == 'demo': # Update context.obj settings here LOG.debug("Change database name to scout-demo") context.obj['mongodb'] = 'scout-demo' LOG.info("Setting database name to %s", context.obj['mongodb']) LOG.debug("Setting host to %s", context.obj['host']) LOG.debug("Setting port to %s", context.obj['port']) try: client = get_connection( host=context.obj['host'], port=context.obj['port'], username=context.obj['username'], password=context.obj['password'], mongodb = context.obj['mongodb'] ) except ConnectionFailure: context.abort() LOG.info("connecting to database %s", context.obj['mongodb']) database = client[context.obj['mongodb']] LOG.info("Test if mongod is running") try: LOG.info("Test if mongod is running") database.test.find_one() except ServerSelectionTimeoutError as err: LOG.warning("Connection could not be established") LOG.warning("Please check if mongod is running") context.abort() LOG.info("Setting up a mongo adapter") mongo_adapter = MongoAdapter(database) context.obj['adapter'] = mongo_adapter
def init_app(self, app): """Initialize from flask""" uri = app.config.get("MONGO_URI", None) db_name = app.config.get("MONGO_DBNAME", 'scout') try: client = get_connection( host=app.config.get("MONGO_HOST", 'localhost'), port=app.config.get("MONGO_PORT", 27017), username=app.config.get("MONGO_USERNAME", None), password=app.config.get("MONGO_PASSWORD", None), uri=uri, mongodb=db_name) except ConnectionFailure: context.abort() app.config["MONGO_DATABASE"] = client[db_name] app.config['MONGO_CLIENT'] = client
def setup(context): """ Setup scout instances. """ context.obj['institute_name'] = 'cust000' context.obj['user_name'] = 'Clark Kent' context.obj['user_mail'] = '*****@*****.**' if context.invoked_subcommand == 'demo': # Update context.obj settings here LOG.info("Change database name to scout-demo") context.obj['mongodb'] = 'scout-demo' LOG.info("Setting database name to %s", context.obj['mongodb']) LOG.debug("Setting host to %s", context.obj['host']) LOG.debug("Setting port to %s", context.obj['port']) try: client = get_connection( host=context.obj['host'], port=context.obj['port'], username=context.obj['username'], password=context.obj['password'], mongodb = context.obj['mongodb'] ) except ConnectionFailure: context.abort() LOG.info("connecting to database %s", context.obj['mongodb']) database = client[context.obj['mongodb']] LOG.info("Test if mongod is running") try: database.test.find_one() except ServerSelectionTimeoutError as err: LOG.warning("Connection could not be established") LOG.warning("Please check if mongod is running") context.abort() LOG.info("Setting up a mongo adapter") mongo_adapter = MongoAdapter(database) context.obj['adapter'] = mongo_adapter
def init_app(self, app): """Initialize from flask""" uri = app.config.get("MONGO_URI", None) db_name = app.config.get("MONGO_DBNAME", 'scout') try: client = get_connection( host = app.config.get("MONGO_HOST", 'localhost'), port=app.config.get("MONGO_PORT", 27017), username=app.config.get("MONGO_USERNAME", None), password=app.config.get("MONGO_PASSWORD", None), uri=uri, mongodb= db_name ) except ConnectionFailure: context.abort() app.config["MONGO_DATABASE"] = client[db_name] app.config['MONGO_CLIENT'] = client
def update_panels(context, mongodb, username, password, authdb, host, port, loglevel, config): """scout: manage interactions with a scout instance.""" coloredlogs.install(level=loglevel) LOG.info("Running scout version %s", __version__) LOG.debug("Debug logging enabled.") mongo_config = {} cli_config = {} if config: LOG.debug("Use config file %s", config) with open(config, "r") as in_handle: cli_config = yaml.load(in_handle, Loader=yaml.FullLoader) mongo_config["mongodb"] = mongodb or cli_config.get("mongodb") or "scout" mongo_config["host"] = host or cli_config.get("host") or "localhost" mongo_config["port"] = port or cli_config.get("port") or 27017 mongo_config["username"] = username or cli_config.get("username") mongo_config["password"] = password or cli_config.get("password") mongo_config["authdb"] = (authdb or cli_config.get("authdb") or mongo_config["mongodb"]) mongo_config["omim_api_key"] = cli_config.get("omim_api_key") LOG.info("Setting database name to %s", mongo_config["mongodb"]) LOG.debug("Setting host to %s", mongo_config["host"]) LOG.debug("Setting port to %s", mongo_config["port"]) valid_connection = check_connection( host=mongo_config["host"], port=mongo_config["port"], username=mongo_config["username"], password=mongo_config["password"], authdb=mongo_config["authdb"], ) LOG.info("Test if mongod is running") if not valid_connection: LOG.warning("Connection could not be established") context.abort() try: client = get_connection(**mongo_config) except ConnectionFailure: context.abort() database = client[mongo_config["mongodb"]] LOG.info("Setting up a mongo adapter") mongo_config["client"] = client adapter = MongoAdapter(database) requests = [] for case_obj in adapter.case_collection.find(): # pp(case_obj) gene_to_panels = adapter.gene_to_panels(case_obj) variants = adapter.variant_collection.find({ "case_id": case_obj["_id"], "category": "snv", "variant_type": "clinical" }) for variant_obj in variants: panel_names = set() for hgnc_id in variant_obj["hgnc_ids"]: gene_panels = gene_to_panels.get(hgnc_id, set()) panel_names = panel_names.union(gene_panels) if panel_names: operation = pymongo.UpdateOne( {"_id": variant_obj["_id"]}, {"$set": { "panels": list(panel_names) }}) requests.append(operation) if len(requests) > 5000: adapter.variant_collection.bulk_write(requests, ordered=False) requests = [] if requests: adapter.variant_collection.bulk_write(requests, ordered=False) requests = []
def update_panels(context, mongodb, username, password, authdb, host, port, loglevel, config): """scout: manage interactions with a scout instance.""" coloredlogs.install(level=loglevel) LOG.info("Running scout version %s", __version__) LOG.debug("Debug logging enabled.") mongo_config = {} cli_config = {} if config: LOG.debug("Use config file %s", config) with open(config, 'r') as in_handle: cli_config = yaml.load(in_handle, Loader=yaml.FullLoader) mongo_config['mongodb'] = (mongodb or cli_config.get('mongodb') or 'scout') mongo_config['host'] = (host or cli_config.get('host') or 'localhost') mongo_config['port'] = (port or cli_config.get('port') or 27017) mongo_config['username'] = username or cli_config.get('username') mongo_config['password'] = password or cli_config.get('password') mongo_config['authdb'] = authdb or cli_config.get('authdb') or mongo_config['mongodb'] mongo_config['omim_api_key'] = cli_config.get('omim_api_key') LOG.info("Setting database name to %s", mongo_config['mongodb']) LOG.debug("Setting host to %s", mongo_config['host']) LOG.debug("Setting port to %s", mongo_config['port']) valid_connection = check_connection( host=mongo_config['host'], port=mongo_config['port'], username=mongo_config['username'], password=mongo_config['password'], authdb=mongo_config['authdb'], ) LOG.info("Test if mongod is running") if not valid_connection: LOG.warning("Connection could not be established") context.abort() try: client = get_connection(**mongo_config) except ConnectionFailure: context.abort() database = client[mongo_config['mongodb']] LOG.info("Setting up a mongo adapter") mongo_config['client'] = client adapter = MongoAdapter(database) requests = [] for case_obj in adapter.case_collection.find(): # pp(case_obj) gene_to_panels = adapter.gene_to_panels(case_obj) variants = adapter.variant_collection.find({ 'case_id': case_obj['_id'], 'category': 'snv', 'variant_type': 'clinical', }) for variant_obj in variants: panel_names = set() for hgnc_id in variant_obj['hgnc_ids']: gene_panels = gene_to_panels.get(hgnc_id, set()) panel_names = panel_names.union(gene_panels) if panel_names: operation = pymongo.UpdateOne( {'_id': variant_obj['_id']}, { '$set': { 'panels': list(panel_names) } }) requests.append(operation) if len(requests) > 5000: adapter.variant_collection.bulk_write(requests, ordered=False) requests = [] if requests: adapter.variant_collection.bulk_write(requests, ordered=False) requests = []
def update_panels(context, mongodb, username, password, authdb, host, port, loglevel, config): """scout: manage interactions with a scout instance.""" coloredlogs.install(level=loglevel) LOG.info("Running scout version %s", __version__) LOG.debug("Debug logging enabled.") mongo_config = {} cli_config = {} if config: LOG.debug("Use config file %s", config) with open(config, 'r') as in_handle: cli_config = yaml.load(in_handle) mongo_config['mongodb'] = (mongodb or cli_config.get('mongodb') or 'scout') mongo_config['host'] = (host or cli_config.get('host') or 'localhost') mongo_config['port'] = (port or cli_config.get('port') or 27017) mongo_config['username'] = username or cli_config.get('username') mongo_config['password'] = password or cli_config.get('password') mongo_config['authdb'] = authdb or cli_config.get( 'authdb') or mongo_config['mongodb'] mongo_config['omim_api_key'] = cli_config.get('omim_api_key') LOG.info("Setting database name to %s", mongo_config['mongodb']) LOG.debug("Setting host to %s", mongo_config['host']) LOG.debug("Setting port to %s", mongo_config['port']) valid_connection = check_connection( host=mongo_config['host'], port=mongo_config['port'], username=mongo_config['username'], password=mongo_config['password'], authdb=mongo_config['authdb'], ) LOG.info("Test if mongod is running") if not valid_connection: LOG.warning("Connection could not be established") context.abort() try: client = get_connection(**mongo_config) except ConnectionFailure: context.abort() database = client[mongo_config['mongodb']] LOG.info("Setting up a mongo adapter") mongo_config['client'] = client adapter = MongoAdapter(database) requests = [] for case_obj in adapter.case_collection.find(): # pp(case_obj) gene_to_panels = adapter.gene_to_panels(case_obj) variants = adapter.variant_collection.find({ 'case_id': case_obj['_id'], 'category': 'snv', 'variant_type': 'clinical', }) for variant_obj in variants: panel_names = set() for hgnc_id in variant_obj['hgnc_ids']: gene_panels = gene_to_panels.get(hgnc_id, set()) panel_names = panel_names.union(gene_panels) if panel_names: operation = pymongo.UpdateOne( {'_id': variant_obj['_id']}, {'$set': { 'panels': list(panel_names) }}) requests.append(operation) if len(requests) > 5000: adapter.variant_collection.bulk_write(requests, ordered=False) requests = [] if requests: adapter.variant_collection.bulk_write(requests, ordered=False) requests = []
def setup(context): """ Setup scout instances. """ context.obj['institute_name'] = 'cust000' context.obj['user_name'] = 'Clark Kent' context.obj['user_mail'] = '*****@*****.**' if context.invoked_subcommand == 'demo': # Update context.obj settings here log.info("Change database name to scout-demo") context.obj['mongodb'] = 'scout-demo' log.info("Loading hgnc genes from %s", hgnc_reduced_path) context.obj['hgnc'] = get_file_handle(hgnc_reduced_path) context.obj['hgnc38'] = get_file_handle(hgnc_reduced_path) log.info("Loading exac genes from %s", exac_reduced_path) context.obj['exac'] = get_file_handle(exac_reduced_path) context.obj['exac38'] = get_file_handle(exac_reduced_path) log.info("Loading mim2gene info from %s", mim2gene_reduced_path) context.obj['mim2gene'] = get_file_handle(mim2gene_reduced_path) context.obj['mim2gene38'] = get_file_handle(mim2gene_reduced_path) log.info("Loading genemap info from %s", genemap2_reduced_path) context.obj['genemap2'] = get_file_handle(genemap2_reduced_path) context.obj['genemap2_38'] = get_file_handle(genemap2_reduced_path) log.info("Loading hpo gene info from %s", hpogenes_reduced_path) context.obj['hpogenes'] = get_file_handle(hpogenes_reduced_path) context.obj['hpogenes_38'] = get_file_handle(hpogenes_reduced_path) log.info("Loading hpo disease info from %s", hpo_phenotype_to_terms_reduced_path) context.obj['hpodiseases'] = get_file_handle(hpo_phenotype_to_terms_reduced_path) log.info("Loading hpo terms from %s", hpoterms_reduced_path) context.obj['hpo_terms'] = get_file_handle(hpoterms_reduced_path) log.info("Loading omim disease info from %s", genemap2_reduced_path) context.obj['disease_terms'] = get_file_handle(genemap2_reduced_path) log.info("Loading transcripts build 37 info from %s", transcripts37_reduced_path) context.obj['transcripts37'] = get_file_handle(transcripts37_reduced_path) # log.info("Loading transcripts build 38 info from %s", transcripts38_path) # context.obj['transcripts38'] = get_file_handle(transcripts38_path) else: log.info("Loading hgnc genes from %s", hgnc_path) context.obj['hgnc'] = get_file_handle(hgnc_path) context.obj['hgnc38'] = get_file_handle(hgnc_path) log.info("Loading exac genes from %s", exac_path) context.obj['exac'] = get_file_handle(exac_path) context.obj['exac38'] = get_file_handle(exac_path) log.info("Loading mim2gene info from %s", mim2gene_path) context.obj['mim2gene'] = get_file_handle(mim2gene_path) context.obj['mim2gene38'] = get_file_handle(mim2gene_path) log.info("Loading genemap info from %s", genemap2_path) context.obj['genemap2'] = get_file_handle(genemap2_path) context.obj['genemap2_38'] = get_file_handle(genemap2_path) log.info("Loading hpo gene info from %s", hpogenes_path) context.obj['hpogenes'] = get_file_handle(hpogenes_path) context.obj['hpogenes_38'] = get_file_handle(hpogenes_path) log.info("Loading hpo disease info from %s", hpo_phenotype_to_terms_path) context.obj['hpodiseases'] = get_file_handle(hpo_phenotype_to_terms_path) log.info("Loading hpo terms from %s", hpoterms_path) context.obj['hpo_terms'] = get_file_handle(hpoterms_path) log.info("Loading omim disease info from %s", genemap2_path) context.obj['disease_terms'] = get_file_handle(genemap2_path) log.info("Loading transcripts build 37 info from %s", transcripts37_path) context.obj['transcripts37'] = get_file_handle(transcripts37_path) log.info("Loading transcripts build 38 info from %s", transcripts38_path) context.obj['transcripts38'] = get_file_handle(transcripts38_path) log.info("Setting database name to %s", context.obj['mongodb']) log.debug("Setting host to %s", context.obj['host']) log.debug("Setting port to %s", context.obj['port']) try: client = get_connection( host=context.obj['host'], port=context.obj['port'], username=context.obj['username'], password=context.obj['password'], ) except ConnectionFailure: context.abort() log.info("connecting to database %s", context.obj['mongodb']) database = client[context.obj['mongodb']] log.info("Test if mongod is running") try: database.test.find_one() except ServerSelectionTimeoutError as err: log.warning("Connection could not be established") log.warning("Please check if mongod is running") context.abort() log.info("Setting up a mongo adapter") mongo_adapter = MongoAdapter(database) context.obj['adapter'] = mongo_adapter
def merge_users( context, mongodb, username, password, authdb, host, port, loglevel, config, live ): """scout: manage interactions with a scout instance.""" coloredlogs.install(level=loglevel) LOG.info("Running scout version %s", __version__) LOG.debug("Debug logging enabled.") mongo_config = {} cli_config = {} if config: LOG.debug("Use config file %s", config) with open(config, "r") as in_handle: cli_config = yaml.load(in_handle, Loader=yaml.FullLoader) mongo_config["mongodb"] = mongodb or cli_config.get("mongodb") or "scout" mongo_config["host"] = host or cli_config.get("host") or "localhost" mongo_config["port"] = port or cli_config.get("port") or 27017 mongo_config["username"] = username or cli_config.get("username") mongo_config["password"] = password or cli_config.get("password") mongo_config["authdb"] = ( authdb or cli_config.get("authdb") or mongo_config["mongodb"] ) mongo_config["omim_api_key"] = cli_config.get("omim_api_key") # always dryrun for now dryrun = True if live: dryrun = False LOG.info("Setting database name to %s", mongo_config["mongodb"]) LOG.debug("Setting host to %s", mongo_config["host"]) LOG.debug("Setting port to %s", mongo_config["port"]) valid_connection = check_connection( host=mongo_config["host"], port=mongo_config["port"], username=mongo_config["username"], password=mongo_config["password"], authdb=mongo_config["authdb"], ) LOG.info("Test if mongod is running") if not valid_connection: LOG.warning("Connection could not be established") context.abort() try: client = get_connection(**mongo_config) except ConnectionFailure: context.abort() database = client[mongo_config["mongodb"]] LOG.info("Setting up a mongo adapter") mongo_config["client"] = client adapter = MongoAdapter(database) ## First, create all operation requests that would be needed in a live run. user_requests = [] event_requests = [] acmg_requests = [] clinvar_requests = [] clinvar_submission_requests = [] case_requests = [] for oi_user_obj in adapter.user_collection.find({"_id": {"$type": "objectId"}}): if not oi_user_obj.get("email"): continue LOG.info("===USER===") oi_user_id = oi_user_obj.get("_id") oi_user_email = oi_user_obj.get("email") LOG.info("user: {}".format(oi_user_obj)) create_alt = False alt_user_obj = adapter.user_collection.find_one({"_id": oi_user_email}) if not alt_user_obj: create_alt = True alt_user_obj = copy.deepcopy(oi_user_obj) alt_user_obj["_id"] = oi_user_email else: LOG.info("alt user: {}".format(alt_user_obj)) merged_institutes = set() merged_institutes.update( alt_user_obj.get("institutes", []) + oi_user_obj.get("institutes", []) ) LOG.info("merged institutes: {}".format(merged_institutes)) alt_user_obj["institutes"] = list(merged_institutes) merged_roles = set() merged_roles.update( alt_user_obj.get("roles", []) + oi_user_obj.get("roles", []) ) LOG.info("merged roles: {}".format(merged_roles)) alt_user_obj["roles"] = list(merged_roles) created_at = oi_user_obj.get("created_at") alt_created_at = alt_user_obj.get("created_at") if (alt_created_at and created_at) and alt_created_at < created_at: created_at = alt_created_at if created_at: alt_user_obj["created_at"] = created_at accessed_at = alt_user_obj.get("accessed_at") oi_accessed_at = oi_user_obj.get("accessed_at") if (oi_accessed_at and accessed_at) and oi_accessed_at > accessed_at: accessed_at = oi_accessed_at if accessed_at: alt_user_obj["accessed_at"] = accessed_at if create_alt: LOG.info("create user: {}".format(alt_user_obj)) operation = pymongo.InsertOne(alt_user_obj) user_requests.append(operation) else: LOG.info("update user: {}".format(alt_user_obj)) alt_user_id = alt_user_obj.pop("_id") operation = pymongo.UpdateOne({"_id": alt_user_id}, {"$set": alt_user_obj}) user_requests.append(operation) # finally, delete the oi user operation = pymongo.DeleteOne({"_id": ObjectId(str(oi_user_id))}) user_requests.append(operation) ### ### events ### LOG.info("searching for events for user id {}".format(oi_user_id)) oi_user_events = adapter.event_collection.find( {"user_id": ObjectId(str(oi_user_id))} ) if oi_user_events.count() > 0: LOG.info("===EVENTS===") for event in oi_user_events: LOG.info("user event: {}".format(event)) event_id = event.get("_id") operation = pymongo.UpdateOne( {"_id": event_id}, { "$set": { "user_id": oi_user_email, "user_name": alt_user_obj.get("name"), } }, ) event_requests.append(operation) ### ### ACMG classifications ### LOG.info("searching for acmg for user id {}".format(oi_user_id)) oi_user_acmg = adapter.acmg_collection.find( {"user_id": ObjectId(str(oi_user_id))} ) if oi_user_acmg.count() > 0: LOG.info("===ACMG===") for acmg in oi_user_acmg: LOG.info("acmg: {}".format(acmg)) operation = pymongo.UpdateOne( {"_id": acmg.get("_id")}, { "$set": { "user_id": oi_user_email, "user_name": alt_user_obj.get("name"), } }, ) acmg_requests.append(operation) # Clinvar LOG.info("searching for ClinVar for user id {}".format(oi_user_id)) oi_user_clinvar = adapter.clinvar_collection.find( {"user": ObjectId(str(oi_user_id))} ) if oi_user_clinvar.count() > 0: LOG.info("=== ClinVar ===") for clinvar in oi_user_clinvar: LOG.info("acmg: {}".format(clinvar)) operation = pymongo.UpdateOne( {"_id": clinvar.get("_id")}, {"$set": {"user": oi_user_email}} ) clinvar_requests.append(operation) # clinvar_submission LOG.info("searching for clinvar submissions for user id {}".format(oi_user_id)) oi_user_clinvars = adapter.clinvar_submission_collection.find( {"user_id": ObjectId(str(oi_user_id))} ) if oi_user_clinvars.count() > 0: LOG.info("=== ClinVar submission ===") for clinvars in oi_user_clinvars: LOG.info("acmg: {}".format(clinvars)) operation = pymongo.UpdateOne( {"_id": clinvars.get("_id")}, {"$set": {"user_id": oi_user_email}} ) clinvar_submission_requests.append(operation) ### ### cases ### LOG.info("searching for cases assigned to user id {}".format(oi_user_id)) oi_user_cases = adapter.case_collection.find( {"assignees": ObjectId(str(oi_user_id))} ) if oi_user_cases.count() > 0: LOG.info("=== Case assignees ===") for case in oi_user_cases: LOG.info("case {} assignees: {}".format(case["_id"], case["assignees"])) operation = pymongo.UpdateOne( {"_id": case.get("_id"), "assignees": ObjectId(str(oi_user_id))}, {"$set": {"assignees.$": oi_user_email}}, ) case_requests.append(operation) else: LOG.info("No ObjectId ID user IDs found - nothing more to do.") # if everything worked out ok with dryrun, and after getting this far on a live run, # bulk write all proposed changes. if event_requests: LOG.info("event requests to execute: {}".format(event_requests)) if not dryrun: result = adapter.event_collection.bulk_write(event_requests, ordered=False) if acmg_requests: LOG.info("acmg requests to execute: {}".format(acmg_requests)) if not dryrun: result = adapter.acmg_collection.bulk_write(acmg_requests, ordered=False) LOG.info("Modified {} ACMG.".format(result.modified_count)) if clinvar_requests: LOG.info("clinvar requests to execute: {}".format(clinvar_requests)) if not dryrun: result = adapter.clinvar_collection.bulk_write( clinvar_requests, ordered=False ) LOG.info("Modified {} ClinVar.".format(result.modified_count)) if clinvar_submission_requests: LOG.info( "clinvar sub requests to execute: {}".format(clinvar_submission_requests) ) if not dryrun: result = adapter.clinvar_submission_collection.bulk_write( clinvar_submission_requests, ordered=False ) LOG.info("Modified {} ClinVar submissions.".format(result.modified_count)) if case_requests: LOG.info("case requests to execute: {}".format(case_requests)) if not dryrun: result = adapter.case_collection.bulk_write(case_requests, ordered=False) LOG.info("Modified {} case submissions.".format(result.modified_count)) # now delete oi user, and actually update/create alt user if user_requests: LOG.info("user requests to execute: {}".format(user_requests)) if not dryrun: result = adapter.user_collection.bulk_write(user_requests, ordered=False) LOG.info( "Modified users with the following: {}".format(result.bulk_api_result) )
def check_panels(context, mongodb, username, password, authdb, host, port, loglevel, config): """scout: manage interactions with a scout instance.""" coloredlogs.install(level=loglevel) LOG.info("Running scout version %s", __version__) LOG.debug("Debug logging enabled.") mongo_config = {} cli_config = {} if config: LOG.debug("Use config file %s", config) with open(config, "r") as in_handle: cli_config = yaml.load(in_handle, Loader=yaml.SafeLoader) mongo_config["mongodb"] = mongodb or cli_config.get("mongodb") or "scout" mongo_config["host"] = host or cli_config.get("host") or "localhost" mongo_config["port"] = port or cli_config.get("port") or 27017 mongo_config["username"] = username or cli_config.get("username") mongo_config["password"] = password or cli_config.get("password") mongo_config["authdb"] = authdb or cli_config.get( "authdb") or mongo_config["mongodb"] mongo_config["omim_api_key"] = cli_config.get("omim_api_key") LOG.info("Setting database name to %s", mongo_config["mongodb"]) LOG.debug("Setting host to %s", mongo_config["host"]) LOG.debug("Setting port to %s", mongo_config["port"]) # Connection validity will be checked in adapter.client client = get_connection(**mongo_config) database = client[mongo_config["mongodb"]] LOG.info("Setting up a mongo adapter") mongo_config["client"] = client adapter = MongoAdapter(database) cases_compromised = 0 compromised_cases = [] cases = 0 panel_genes_incorrect = {} variants_found = "" panel_issue_found = "" for case_obj in adapter.case_collection.find(no_cursor_timeout=True).sort( "updated_at", -1): if case_obj.get("updated_at") < datetime.datetime(2020, 9, 8): break # if cases_compromised >= 20: # break case_compromised = False for panel_info in case_obj.get("panels", []): panel_name = panel_info["panel_name"] if not panel_info.get("is_default", False): continue if panel_genes_incorrect.get(panel_name, None) is not None: if len(panel_genes_incorrect[panel_name]) > 0: case_compromised = True else: panel_obj = adapter.gene_panel(panel_name) if not panel_obj: LOG.warning("Panel: {0} does not exist in database".format( panel_name)) continue symbols_compromised = [] for panel_gene in panel_obj["genes"]: gene_hgnc_id = panel_gene["hgnc_id"] hgnc_symbol_panel = panel_gene["symbol"] hgnc_gene_from_id = adapter.hgnc_gene(gene_hgnc_id) hgnc_symbol_hgnc = hgnc_gene_from_id.get("hgnc_symbol") if hgnc_symbol_panel != hgnc_symbol_hgnc: case_compromised = True symbols_compromised.append(hgnc_symbol_hgnc) panel_issue_found += "Case {} panel {} gene symbol {} differes from current hgnc symbol {}\n".format( case_obj["display_name"], panel_name, hgnc_symbol_panel, hgnc_symbol_hgnc, ) panel_genes_incorrect[panel_name] = symbols_compromised if panel_genes_incorrect.get(panel_name, None): for variant in adapter.variants( case_obj["_id"], query={ "hgnc_symbols": panel_genes_incorrect[panel_name], }, nr_of_variants=-1, sort_key="rank_score", ): if variant.get("rank_score") > 15: variants_found += "{} case {} {} variant {} {} {}.\n".format( variant["institute"], case_obj["_id"], case_obj["display_name"], variant["display_name"], variant["rank_score"], variant["hgnc_symbols"], ) cases += 1 if case_compromised: cases_compromised += 1 compromised_cases.append(case_obj["_id"]) LOG.debug("Case {} compromised.".format( case_obj["display_name"], )) else: LOG.debug("Case {} ok.".format(case_obj["display_name"], )) LOG.warning("{} cases out of {} were compromised.".format( cases_compromised, cases)) for panel, symbols in panel_genes_incorrect.items(): LOG.warning("Panel {} had {} symbols compromised".format( panel, len(symbols))) LOG.warning("Panel issues:\n{}".format(panel_issue_found)) LOG.warning("Variants of interest: {} ".format(variants_found))