def test_human_only(): im = IncrementalModel() stmt1 = Phosphorylation(None, Agent('BRAF', db_refs={'UP': 'P15056'})) stmt2 = Phosphorylation(None, Agent('BRAF', db_refs={'UP': 'P28028'})) stmt3 = Phosphorylation(None, Agent('BRAF', db_refs={'HGNC': 'BRAF'})) stmt4 = Phosphorylation(None, Agent('BRAF', db_refs={})) im.add_statements('12345', [stmt1], filters=['human_only']) assert(len(im.get_statements()) == 1) im.add_statements('12346', [stmt2], filters=['human_only']) assert(len(im.get_statements()) == 1) im.add_statements('12346', [stmt3], filters=['human_only']) assert (len(im.get_statements()) == 2) im.add_statements('12346', [stmt4], filters=['human_only']) assert (len(im.get_statements()) == 3)
def summarize_helper(model_path): logger.info(time.strftime('%c')) logger.info('Loading original model.') inc_model_file = os.path.join(model_path, 'model.pkl') model = IncrementalModel(inc_model_file) stmts = model.get_statements() click.echo('Number of statements: {}'.format(len(stmts))) agents = model.get_model_agents() click.echo('Number of agents: {}'.format(len(agents)))
def test_grounding_none_agent(): im = IncrementalModel() im.add_statements('12345', stmts2, filters=['grounding']) assert(len(im.get_statements()) == 1)
def test_grounding_all(): im = IncrementalModel() stmt = Complex([Agent('A', db_refs={'UP': 'ABCD'}), Agent('B', db_refs={'HGNC': '1234'})]) im.add_statements('12345', [stmt], filters=['grounding']) assert(len(im.get_statements()) == 1)
def test_add_stmts_prior_all(): im = IncrementalModel() im.stmts['prior'] = [stmts[0]] im.add_statements('12345', [stmts[1]], filters=['prior_all']) assert(len(im.get_statements()) == 1)
def test_add_stmts_model_all(): im = IncrementalModel() im.add_statements('12345', [stmts[0]]) im.add_statements('23456', [stmts[1]], filters=['model_all']) assert(len(im.get_statements()) == 1)
def test_add_stmts_blank_noprior2(): im = IncrementalModel() im.add_statements('12345', stmts, filters=['prior_all']) assert(len(im.get_statements()) == 2)
def test_add_stmts_blank_emptyfilter(): im = IncrementalModel() im.add_statements('12345', stmts, filters=[]) assert(len(im.get_statements()) == 2)
def test_add_stmts_blank(): im = IncrementalModel() im.add_statements('12345', stmts) assert(len(im.get_statements()) == 2)
def run_machine(model_path, pmids, belief_threshold, search_genes=None, ndex_cred=None, twitter_cred=None): start_time_local = datetime.datetime.now(tzlocal.get_localzone()) date_str = make_date_str() # Save PMIDs in file and send for remote reading if aws_available: pmid_fname = 'pmids-%s.txt' % date_str all_pmids = [] for v in pmids.values(): all_pmids += v all_pmids = list(set(all_pmids)) with open(pmid_fname, 'wt') as fh: for pmid in all_pmids: fh.write('%s\n' % pmid) # Submit reading job_list = submit_reading('rasmachine', pmid_fname, ['reach']) # Wait for reading to complete wait_for_complete('run_reach_queue', job_list, idle_log_timeout=600, kill_on_log_timeout=True) # Load the model logger.info(time.strftime('%c')) logger.info('Loading original model.') inc_model_file = os.path.join(model_path, 'model.pkl') model = IncrementalModel(inc_model_file) # Include search genes as prior genes if search_genes: model.prior_genes = search_genes stats = {} logger.info(time.strftime('%c')) logger.info('Preassembling original model.') model.preassemble(filters=global_filters) logger.info(time.strftime('%c')) # Original statistics stats['orig_stmts'] = len(model.get_statements()) stats['orig_assembled'] = len(model.assembled_stmts) orig_stmts = filter_db_highbelief(model.assembled_stmts, ['bel', 'biopax'], belief_threshold) orig_stmts = ac.filter_top_level(orig_stmts) stats['orig_final'] = len(orig_stmts) logger.info('%d final statements' % len(orig_stmts)) # Extend the model with PMIDs logger.info('----------------') logger.info(time.strftime('%c')) logger.info('Extending model.') stats['new_papers'], stats['new_abstracts'], stats['existing'] = \ extend_model(model_path, model, pmids, start_time_local) # Having added new statements, we preassemble the model model.preassemble(filters=global_filters) # New statistics stats['new_stmts'] = len(model.get_statements()) stats['new_assembled'] = len(model.assembled_stmts) new_stmts = filter_db_highbelief(model.assembled_stmts, ['bel', 'biopax'], belief_threshold) new_stmts = ac.filter_top_level(new_stmts) stats['new_final'] = len(new_stmts) logger.info('%d final statements' % len(new_stmts)) check_pmids(model.get_statements()) # Save model logger.info(time.strftime('%c')) logger.info('Saving model') model.save(inc_model_file) logger.info(time.strftime('%c')) # Save a time stamped version of the pickle for backup/diagnostic purposes if not aws_available: inc_model_bkp_file = os.path.join(model_path, 'model-%s.pkl' % date_str) model.save(inc_model_bkp_file) else: key = 'rasmachine/%s/model-%s.pkl' % (model_path.replace( '/', '_'), date_str) s3 = boto3.client('s3') s3.upload_file(inc_model_file, 'bigmech', key) # Upload the new, final statements to NDEx if ndex_cred: upload_new_ndex(model_path, new_stmts, ndex_cred) # Print and tweet the status message logger.info('--- Final statistics ---') for k, v in sorted(stats.items(), key=lambda x: x[0]): logger.info('%s: %s' % (k, v)) logger.info('------------------------') msg_str = make_status_message(stats) if msg_str is not None: logger.info('Status message: %s' % msg_str) if twitter_cred: logger.info('Now tweeting: %s' % msg_str) twitter_client.update_status(msg_str, twitter_cred)
def test_add_stmts_blank(): im = IncrementalModel() im.add_statements('12345', stmts) assert len(im.get_statements()) == 2 im.preassemble() assert len(im.assembled_stmts) == 2
def run_machine(model_path, pmids, belief_threshold, search_genes=None, ndex_cred=None, twitter_cred=None, grounding_map=None): start_time_local = datetime.datetime.now(tzlocal.get_localzone()) date_str = make_date_str() # Save PMIDs in file and send for remote reading if aws_available: pmid_fname = 'pmids-%s.txt' % date_str all_pmids = [] for v in pmids.values(): all_pmids += v all_pmids = list(set(all_pmids)) with open(pmid_fname, 'wt') as fh: for pmid in all_pmids: fh.write('%s\n' % pmid) # Submit reading job_list = submit_reading('rasmachine', pmid_fname, ['reach']) # Wait for reading to complete wait_for_complete('run_reach_queue', job_list, idle_log_timeout=600, kill_on_log_timeout=True) # Load the model logger.info(time.strftime('%c')) logger.info('Loading original model.') inc_model_file = os.path.join(model_path, 'model.pkl') model = IncrementalModel(inc_model_file) # Include search genes as prior genes if search_genes: model.prior_genes = search_genes stats = {} logger.info(time.strftime('%c')) logger.info('Preassembling original model.') model.preassemble(filters=global_filters, grounding_map=grounding_map) logger.info(time.strftime('%c')) # Original statistics stats['orig_stmts'] = len(model.get_statements()) stats['orig_assembled'] = len(model.assembled_stmts) orig_stmts = filter_db_highbelief(model.assembled_stmts, ['bel', 'biopax'], belief_threshold) orig_stmts = ac.filter_top_level(orig_stmts) stats['orig_final'] = len(orig_stmts) logger.info('%d final statements' % len(orig_stmts)) # Extend the model with PMIDs logger.info('----------------') logger.info(time.strftime('%c')) logger.info('Extending model.') stats['new_papers'], stats['new_abstracts'], stats['existing'] = \ extend_model(model_path, model, pmids, start_time_local) # Having added new statements, we preassemble the model model.preassemble(filters=global_filters, grounding_map=grounding_map) # New statistics stats['new_stmts'] = len(model.get_statements()) stats['new_assembled'] = len(model.assembled_stmts) new_stmts = filter_db_highbelief(model.assembled_stmts, ['bel', 'biopax'], belief_threshold) new_stmts = ac.filter_top_level(new_stmts) stats['new_final'] = len(new_stmts) logger.info('%d final statements' % len(new_stmts)) check_pmids(model.get_statements()) # Save model logger.info(time.strftime('%c')) logger.info('Saving model') model.save(inc_model_file) logger.info(time.strftime('%c')) # Save a time stamped version of the pickle for backup/diagnostic purposes if not aws_available: inc_model_bkp_file = os.path.join(model_path, 'model-%s.pkl' % date_str) model.save(inc_model_bkp_file) else: key = 'rasmachine/%s/model-%s.pkl' % (model_path.replace('/', '_'), date_str) s3 = boto3.client('s3') s3.upload_file(inc_model_file, 'bigmech', key) # Upload the new, final statements to NDEx if ndex_cred: upload_new_ndex(model_path, new_stmts, ndex_cred) # Print and tweet the status message logger.info('--- Final statistics ---') for k, v in sorted(stats.items(), key=lambda x: x[0]): logger.info('%s: %s' % (k, v)) logger.info('------------------------') msg_str = make_status_message(stats) if msg_str is not None: logger.info('Status message: %s' % msg_str) if twitter_cred: logger.info('Now tweeting: %s' % msg_str) twitter_client.update_status(msg_str, twitter_cred)
# Load the model logger.info(time.strftime('%c')) logger.info('Loading original model.') inc_model_file = os.path.join(model_path, model_name, 'model.pkl') model = IncrementalModel(inc_model_file) # Include search genes as prior genes model.prior_genes = search_genes stats = {} logger.info(time.strftime('%c')) logger.info('Preassembling original model.') model.preassemble(filters=global_filters) logger.info(time.strftime('%c')) # Original statistics stats['orig_stmts'] = len(model.get_statements()) stats['orig_assembled'] = len(model.assembled_stmts) db_stmts = ac.filter_evidence_source(model.assembled_stmts, ['biopax', 'bel'], policy='one') no_db_stmts = ac.filter_evidence_source(model.assembled_stmts, ['biopax', 'bel'], policy='none') no_db_stmts = ac.filter_belief(no_db_stmts, belief_threshold) orig_stmts = db_stmts + no_db_stmts stats['orig_final'] = len(orig_stmts) logger.info('%d final statements' % len(orig_stmts)) # Extend the model with PMIDs logger.info('----------------') logger.info(time.strftime('%c'))