def read_pmids(pmids, date): """Return extracted INDRA Statements per PMID after running reading on AWS. Parameters ---------- pmids : list[str] A list of PMIDs to read. Returns ------- dict[str, list[indra.statements.Statement]] A dict of PMIDs and the list of Statements extracted for the given PMID by reading. """ pmid_fname = 'pmids-%s.txt' % date_str with open(pmid_fname, 'wt') as fh: fh.write('\n'.join(pmids)) job_list = submit_reading('emmaa', pmid_fname, ['reach']) date_str = date.strftime('%Y-%m-%d-%H-%M-%S') wait_for_complete('run_reach_queue', job_list, idle_log_timeout=600, kill_on_log_timeout=True) pmid_stmts = {} for pmid in pmids: reach_json_str = get_reader_json_str('reach', pmid) rp = reach.process_json_str(reach_json_str) if not rp: pmid_stmts[pmid] = [] else: pmid_stmts[pmid] = rp.statements return pmid_stmts
def process_paper_aws(pmid, start_time_local): try: metadata, content_type = get_full_text(pmid, metadata=True) except Exception as e: logger.error('Could not get content from S3: %s' % e) return None, None logger.info('Downloading %s output from AWS' % pmid) reach_json_str = get_reader_json_str('reach', pmid) if not reach_json_str: logger.info('Could not get output.') return None, content_type rp = reach.process_json_str(reach_json_str) current_time_local = datetime.datetime.now(tzlocal.get_localzone()) dt_script = current_time_local - start_time_local last_mod_remote = metadata['LastModified'] dt = (current_time_local - last_mod_remote) # If it was not modified since the script started if dt > dt_script: content_type = 'existing_json' return rp, content_type
def process_paper_aws(pmid, start_time_local): try: metadata, content_type = get_full_text(pmid, metadata=True) except Exception as e: logger.error('Could not get content from S3: %s' % e) return None, None logger.info('Downloading %s output from AWS' % pmid) reach_json_str = get_reader_json_str('reach', pmid) if not reach_json_str: logger.info('Could not get output.') return None, content_type rp = reach.process_json_str(reach_json_str) current_time_local = datetime.datetime.now(tzlocal.get_localzone()) dt_script = current_time_local - start_time_local last_mod_remote = metadata['LastModified'] dt = (current_time_local - last_mod_remote) # If it was not modified since the script started if dt > dt_script: content_type = 'existing_json' return rp, content_type
def read_pmids(pmids, date): """Return extracted INDRA Statements per PMID after running reading on AWS. Parameters ---------- pmids : list[str] A list of PMIDs to read. date : datetime The date and time associated with the reading, typically the current time. Returns ------- dict[str, list[indra.statements.Statement] A dict of PMIDs and the list of Statements extracted for the given PMID by reading. """ date_str = date.strftime('%Y-%m-%d-%H-%M-%S') pmid_fname = 'pmids-%s.txt' % date_str with open(pmid_fname, 'wt') as fh: fh.write('\n'.join(pmids)) job_list = submit_reading('emmaa', pmid_fname, ['reach']) monitor = BatchMonitor('run_reach_queue', job_list) monitor.watch_and_wait(idle_log_timeout=600, kill_on_log_timeout=True) pmid_stmts = {} for pmid in pmids: reach_json_str = get_reader_json_str('reach', pmid) if reach_json_str is None: pmid_stmts[pmid] = [] continue rp = reach.process_json_str(reach_json_str) if not rp: pmid_stmts[pmid] = [] else: pmid_stmts[pmid] = rp.statements return pmid_stmts
logger = logging.getLogger('indra.tools.reading.starcluster_reading.' 'process_reach_from_s3') pmid_list_file = sys.argv[1] start_ix = int(sys.argv[2]) end_ix = int(sys.argv[3]) # Load the list of PMIDs from the given file with open(pmid_list_file) as f: pmid_list = [line.strip('\n') for line in f.readlines()] if end_ix > len(pmid_list): end_ix = len(pmid_list) stmts = {} for ix, pmid in enumerate(pmid_list[start_ix:end_ix]): reach_json_str = s3_client.get_reader_json_str('reach', pmid) # Logging message will have been produced by get_reach_output if reach_json_str is None: continue # Run the REACH processor on the JSON try: logger.info('%d: Processing %s' % (ix, pmid)) reach_proc = reach.process_json_str(reach_json_str, citation=pmid) # If there's a problem, skip it except Exception as e: print("Exception processing %s" % pmid) print(e) continue stmts[pmid] = reach_proc.statements
def process_reach_from_s3(pmid): reach_json_str = s3_client.get_reader_json_str('reach', pmid) if reach_json_str is None: return [] else: return {pmid: process_reach_str(reach_json_str, pmid)}
def get_stmts_from_cache(pmid): json_str = s3_client.get_reader_json_str('sparser', pmid) stmts = [] if json_str is not None: stmts = sparser.process_json_dict(json.loads(json_str)).statements return {pmid: stmts}
logger = logging.getLogger('indra.tools.reading.starcluster_reading.' 'process_reach_from_s3') pmid_list_file = sys.argv[1] start_ix = int(sys.argv[2]) end_ix = int(sys.argv[3]) # Load the list of PMIDs from the given file with open(pmid_list_file) as f: pmid_list = [line.strip('\n') for line in f.readlines()] if end_ix > len(pmid_list): end_ix = len(pmid_list) stmts = {} for ix, pmid in enumerate(pmid_list[start_ix:end_ix]): reach_json_str = s3_client.get_reader_json_str('reach', pmid) # Logging message will have been produced by get_reach_output if reach_json_str is None: continue # Run the REACH processor on the JSON try: logger.info('%d: Processing %s' % (ix, pmid)) reach_proc = reach.process_json_str(reach_json_str, citation=pmid) # If there's a problem, skip it except Exception as e: print("Exception processing %s" % pmid) print(e) continue stmts[pmid] = reach_proc.statements