def main(args): file_exists_or_exit(args.medline_paragraph_path) file_exists_or_exit(args.pmid_file) ensure_path_exists(args.output_directory) pmids_of_interest = get_pmid_set(args) print('Reading input files...') all_files = (f for f in Path(args.medline_paragraph_path).glob('**/*') if f.is_file()) found_count = 0 for medline_file in tqdm(all_files): if medline_file.name in pmids_of_interest: if found_count % 1000 == 0: current_subdir = '{0:0>4}'.format(found_count) ensure_path_exists( str(Path(args.output_directory) / current_subdir)) found_count += 1 new_file_path = str( Path(args.output_directory) / current_subdir / medline_file.name) shutil.copyfile(str(medline_file), new_file_path) print('Copied {} files (by PMID) to directory {}'.format( found_count, args.output_directory))
def test_file_exists_or_exit(self): # silence STDERR output from this test sys.stderr = StringIO() with self.assertRaises(SystemExit) as e: util.file_exists_or_exit('/this/file/doesnt/exist') self.assertEqual(e.exception.code, 1)
def test_file_exists_or_exit(self): # silence STDERR output from this test sys.stderr = StringIO() with self.assertRaises(SystemExit) as e: util.file_exists_or_exit('/this/file/doesnt/exist') self.assertEqual(e.exception.code, 1)
def main(args): file_exists_or_exit(os.path.join(args.tmchem, 'tmChem.pl')) args.paragraph_path = os.path.abspath(args.paragraph_path) # glob.glob doesn't support double star expressions in Python 3.4, so using this: print('Reading input files...') all_files = [ str(f) for f in tqdm(Path(args.paragraph_path).glob('**/*'), disable=args.notqdm) if f.is_file() ] filelist_with_sublists = create_n_sublists(all_files, mp.cpu_count() * 1000) # check if save_directory exists and create if necessary if not os.path.isdir(args.output_directory): os.makedirs(args.output_directory) log_filename = os.path.join(args.logdir, 'chem_ner.log') logging_format = '%(asctime)s %(name)-15s %(levelname)-8s %(processName)-10s %(message)s' logging.basicConfig(filename=log_filename, format=logging_format, level=logging.INFO, filemode='w') print('Using {} cores to process {} files...'.format( args.poolsize, len(all_files))) with mp.Pool(args.poolsize) as pool: mgr = mp.Manager() q = mgr.Queue() log_thread = threading.Thread(target=logging_thread, args=(q, )) log_thread.start() imap_gen = pool.imap_unordered( process_and_run_chunk, zip(filelist_with_sublists, repeat(args), repeat(q))) for i in tqdm(imap_gen, total=len(filelist_with_sublists), disable=args.notqdm): pass logging.info('Done processing {} files'.format(len(all_files))) # reorganize a directory with a huge number of files into a bunch of # subdirectories containing those same files, with a max of 10k files per # subdirectory reorganize_directory(args.output_directory, max_files_per_subdir=10000, quiet=args.notqdm) # end logging_thread q.put(None) log_thread.join()
def main(args): file_exists_or_exit(os.path.join(args.dnorm, 'ApplyDNorm.sh')) all_files = glob(os.path.join(args.paragraph_path, '*')) filelist_with_sublists = create_n_sublists(all_files, mp.cpu_count()*10) # check if save_directory exists and create if necessary if not os.path.isdir(args.output_directory): os.makedirs(args.output_directory) log_filename = os.path.join(args.logdir, 'disease_ner.log') logging_format = '%(asctime)s %(name)-15s %(levelname)-8s %(processName)-10s %(message)s' logging.basicConfig(filename=log_filename, format=logging_format, level=logging.INFO, filemode='w') # calculate how many Java/DNorm processes to start, given available RAM # (DNorm requires 10GB RAM per process) num_java_processes = calc_dnorm_num_processes(num_cores=args.poolsize, ram_gb=None) print('Using {} cores to process {} files...'.format(num_java_processes, len(all_files))) print('Allocating {} GB RAM'.format(num_java_processes*10)) with mp.Pool(num_java_processes) as pool: mgr = mp.Manager() q = mgr.Queue() log_thread = threading.Thread(target=logging_thread, args=(q,)) log_thread.start() imap_gen = pool.imap_unordered(process_and_run_chunk, zip(filelist_with_sublists, repeat(args), repeat(q))) for i in tqdm(imap_gen, total=len(filelist_with_sublists), disable=args.notqdm): pass logging.info('Done processing {} files'.format(len(all_files))) # reorganize a directory with a huge number of files into a bunch of # subdirectories containing those same files, with a max of 10k files per # subdirectory reorganize_directory(args.output_directory, max_files_per_subdir=10000, quiet=args.notqdm) # end logging_thread q.put(None) log_thread.join()
def main(args): file_exists_or_exit(os.path.join(args.tmchem,'tmChem.pl')) args.paragraph_path = os.path.abspath(args.paragraph_path) # glob.glob doesn't support double star expressions in Python 3.4, so using this: print('Reading input files...') all_files = [str(f) for f in tqdm(Path(args.paragraph_path).glob('**/*'), disable=args.notqdm) if f.is_file()] filelist_with_sublists = create_n_sublists(all_files, mp.cpu_count()*1000) # check if save_directory exists and create if necessary if not os.path.isdir(args.output_directory): os.makedirs(args.output_directory) log_filename = os.path.join(args.logdir, 'chem_ner.log') logging_format = '%(asctime)s %(name)-15s %(levelname)-8s %(processName)-10s %(message)s' logging.basicConfig(filename=log_filename, format=logging_format, level=logging.INFO, filemode='w') print('Using {} cores to process {} files...'.format(args.poolsize, len(all_files))) with mp.Pool(args.poolsize) as pool: mgr = mp.Manager() q = mgr.Queue() log_thread = threading.Thread(target=logging_thread, args=(q,)) log_thread.start() imap_gen = pool.imap_unordered(process_and_run_chunk, zip(filelist_with_sublists, repeat(args), repeat(q))) for i in tqdm(imap_gen, total=len(filelist_with_sublists), disable=args.notqdm): pass logging.info('Done processing {} files'.format(len(all_files))) # reorganize a directory with a huge number of files into a bunch of # subdirectories containing those same files, with a max of 10k files per # subdirectory reorganize_directory(args.output_directory, max_files_per_subdir=10000, quiet=args.notqdm) # end logging_thread q.put(None) log_thread.join()
def main(args): file_exists_or_exit(os.path.join(args.dnorm, 'ApplyDNorm.sh')) all_files = glob(os.path.join(args.paragraph_path, '*')) filelist_with_sublists = create_n_sublists(all_files, mp.cpu_count() * 10) # check if save_directory exists and create if necessary if not os.path.isdir(args.output_directory): os.makedirs(args.output_directory) log_filename = os.path.join(args.logdir, 'disease_ner.log') logging_format = '%(asctime)s %(name)-15s %(levelname)-8s %(processName)-10s %(message)s' logging.basicConfig(filename=log_filename, format=logging_format, level=logging.INFO, filemode='w') # calculate how many Java/DNorm processes to start, given available RAM # (DNorm requires 10GB RAM per process) num_java_processes = calc_dnorm_num_processes(num_cores=args.poolsize, ram_gb=None) print('Using {} cores to process {} files...'.format( num_java_processes, len(all_files))) print('Allocating {} GB RAM'.format(num_java_processes * 10)) with mp.Pool(num_java_processes) as pool: mgr = mp.Manager() q = mgr.Queue() log_thread = threading.Thread(target=logging_thread, args=(q, )) log_thread.start() imap_gen = pool.imap_unordered( process_and_run_chunk, zip(filelist_with_sublists, repeat(args), repeat(q))) for i in tqdm(imap_gen, total=len(filelist_with_sublists), disable=args.notqdm): pass logging.info('Done processing {} files'.format(len(all_files))) # reorganize a directory with a huge number of files into a bunch of # subdirectories containing those same files, with a max of 10k files per # subdirectory reorganize_directory(args.output_directory, max_files_per_subdir=10000, quiet=args.notqdm) # end logging_thread q.put(None) log_thread.join()
def main(args): # ensure that `qsub` command exists on this machine... shell_command_exists_or_exit('qsub') # make paths absolute so that all other paths are also absolute args.paragraph_path = os.path.abspath(args.paragraph_path) args.output_directory = os.path.abspath(args.output_directory) args.corenlp = os.path.abspath(args.corenlp) file_exists_or_exit(os.path.join(args.corenlp, 'corenlp.sh')) input_dir = os.path.join(args.output_directory, 'input_files') job_dir = os.path.join(args.output_directory, 'job_files') output_dir = os.path.join(args.output_directory, 'output_files') for dirpath in (args.output_directory, input_dir, job_dir, output_dir): ensure_path_exists(dirpath) all_files = glob(os.path.join(args.paragraph_path, '*')) number_of_chunks = len(all_files) // 100 filelist_with_sublists = create_n_sublists(all_files, number_of_chunks) jobs_submitted_success = 0 job_file_paths_failed = [] print('Creating {} input files in {} groups with matched job files...'. format(len(all_files), number_of_chunks)) if args.submit: print('and submitting jobs to queue \'{}\'...'.format(args.queue)) for chunk_num, subfilelist in enumerate(tqdm(filelist_with_sublists)): new_input_files = create_corenlp_input_files(subfilelist, input_dir) new_job_file_path = create_job_file(new_input_files, job_dir, output_dir, chunk_num, args) if args.submit: result = submit_pbs_job(new_job_file_path, queue=args.queue) if result: jobs_submitted_success += 1 else: job_file_paths_failed.append(new_job_file_path) print('Created {} PBS job files in {}'.format(chunk_num + 1, job_dir)) if args.submit: print( 'Successfully submitted {} jobs with {} failed submissions'.format( jobs_submitted_success, len(job_file_paths_failed))) for path in job_file_paths_failed: print('FAILED TO SUBMIT', path)
def main(args): file_exists_or_exit(args.pmid_file) file_exists_or_exit(args.bioconcepts_file) pmids_set = get_file_lines_set(args.pmid_file) records = produce_records(args.bioconcepts_file) print('Found {} distinct PMIDs in file {}'.format(len(pmids_set), args.bioconcepts_file)) if not args.c: # create output subdirectories ensure_path_exists(args.output_directory) pubtator_outdir = os.path.join(args.output_directory, 'pubtator') ensure_path_exists(pubtator_outdir) abstract_outdir = os.path.join(args.output_directory, 'abstracts') ensure_path_exists(abstract_outdir) found_count = 0 not_found_count = 0 for record in tqdm(records): title, abstract, *ner_lines = record pmid = title.split('|')[0] if pmid in pmids_set: found_count += 1 if not args.c: save_file(pmid, [ line + '\n' for line in (title, abstract, '\n'.join(ner_lines)) ], pubtator_outdir) _, parform_output = pubtator_to_parform(title, abstract, newlines=True) save_file(pmid, parform_output, abstract_outdir) else: not_found_count += 1 print('Out of {} abstracts...'.format(found_count + not_found_count)) print('- {} records in {}'.format(found_count, args.pmid_file)) print('- {} records NOT in {}'.format(not_found_count, args.pmid_file)) if not args.c: print('Parsed/paragraph abstracts saved to {}'.format(abstract_outdir)) print('NER-annotated abstracts saved to {}'.format(pubtator_outdir))
def main(args): file_exists_or_exit(args.pmid_file) file_exists_or_exit(args.bioconcepts_file) pmids_set = get_file_lines_set(args.pmid_file) records = produce_records(args.bioconcepts_file) print('Found {} distinct PMIDs in file {}'.format(len(pmids_set), args.bioconcepts_file)) if not args.c: # create output subdirectories ensure_path_exists(args.output_directory) pubtator_outdir = os.path.join(args.output_directory, 'pubtator') ensure_path_exists(pubtator_outdir) abstract_outdir = os.path.join(args.output_directory, 'abstracts') ensure_path_exists(abstract_outdir) found_count = 0 not_found_count = 0 for record in tqdm(records): title, abstract, *ner_lines = record pmid = title.split('|')[0] if pmid in pmids_set: found_count += 1 if not args.c: save_file(pmid, [line+'\n' for line in (title, abstract, '\n'.join(ner_lines))], pubtator_outdir) _, parform_output = pubtator_to_parform(title, abstract, newlines=True) save_file(pmid, parform_output, abstract_outdir) else: not_found_count += 1 print('Out of {} abstracts...'.format(found_count+not_found_count)) print('- {} records in {}'.format(found_count, args.pmid_file)) print('- {} records NOT in {}'.format(not_found_count, args.pmid_file)) if not args.c: print('Parsed/paragraph abstracts saved to {}'.format(abstract_outdir)) print('NER-annotated abstracts saved to {}'.format(pubtator_outdir))
def main(args): file_exists_or_exit(args.medline_paragraph_path) file_exists_or_exit(args.pmid_file) ensure_path_exists(args.output_directory) pmids_of_interest = get_pmid_set(args) print('Reading input files...') all_files = (f for f in Path(args.medline_paragraph_path).glob('**/*') if f.is_file()) found_count = 0 for medline_file in tqdm(all_files): if medline_file.name in pmids_of_interest: if found_count % 1000 == 0: current_subdir = '{0:0>4}'.format(found_count) ensure_path_exists(str(Path(args.output_directory) / current_subdir)) found_count += 1 new_file_path = str(Path(args.output_directory) / current_subdir / medline_file.name) shutil.copyfile(str(medline_file), new_file_path) print('Copied {} files (by PMID) to directory {}'.format(found_count, args.output_directory))
def main(args): file_exists_or_exit(os.path.join(args.gnormplus, 'GNormPlus.pl')) all_files = glob(os.path.join(args.paragraph_path, '*')) filelist_with_sublists = create_n_sublists(all_files, mp.cpu_count()*10) # check if save_directory exists and create if necessary if not os.path.isdir(args.output_directory): os.makedirs(args.output_directory) log_filename = os.path.join(args.logdir, 'gene_ner.log') logging_format = '%(asctime)s %(name)-15s %(levelname)-8s %(processName)-10s %(message)s' logging.basicConfig(filename=log_filename, format=logging_format, level=logging.INFO, filemode='w') print('Using {} cores to process {} files...'.format(mp.cpu_count(), len(all_files))) with mp.Pool() as pool: mgr = mp.Manager() q = mgr.Queue() log_thread = threading.Thread(target=logging_thread, args=(q,)) log_thread.start() imap_gen = pool.imap_unordered(process_and_run_chunk, zip(filelist_with_sublists, repeat(args), repeat(q))) for i in tqdm(imap_gen, total=len(filelist_with_sublists)): pass logging.info('Done processing {} files'.format(len(all_files))) # end logging_thread q.put(None) log_thread.join()
def main(args): # make all paths absolute (to simplify things later) args.paragraph_path = os.path.abspath(args.paragraph_path) args.output_directory = os.path.abspath(args.output_directory) args.logdir = os.path.abspath(args.logdir) args.dnorm = os.path.abspath(args.dnorm) args.bioshovel = os.path.abspath(args.bioshovel) file_exists_or_exit(os.path.join(args.dnorm, 'ApplyDNorm.sh')) # calculate how many Java/DNorm processes to start, given available RAM # (DNorm requires 10GB RAM per process) args.poolsize = calc_dnorm_num_processes(num_cores=args.poolsize, ram_gb=args.memgb) # get all files (recursive) print('Organizing input files and submitting jobs...') if args.resume: # filter out filenames that are already done... args.resume = os.path.abspath(args.resume) file_exists_or_exit(args.resume) print('Resuming job submission from path: {}'.format(args.resume)) if os.path.exists(args.output_directory): print('Output directory changed to avoid file conflicts:') print('OLD: '+args.output_directory) args.output_directory += '_resume' print('NEW: '+args.output_directory) print('Reading previously completed files...') # this set may use a lot of RAM if args.resume path contains a ton of files... done_files = set(p.name for p in tqdm(Path(args.resume).glob('**/*')) if p.is_file()) all_files = (str(f) for f in tqdm(Path(args.paragraph_path).glob('**/*')) if f.is_file() and f.name not in done_files) else: all_files = (str(f) for f in tqdm(Path(args.paragraph_path).glob('**/*')) if f.is_file()) # divide list into chunks of size n filelist_with_sublists = create_sublists_sized_n(all_files, args.nfiles) # move job files into the appropriate subdirectories # (don't store more than 1k per sub-subdirectory) # create job file for each subdirectory # submit jobs base_input_directory = os.path.join(args.output_directory, 'input_files') job_dir = os.path.join(args.output_directory, 'job_files') output_directory = os.path.join(args.output_directory, 'output') for path in (base_input_directory, job_dir, output_directory): ensure_path_exists(path) job_file_paths_failed = [] jobs_submitted_success = 0 for sublist_num, sublist in enumerate(filelist_with_sublists): sublist_dir = os.path.join(base_input_directory, 'sublist_{0:0>4}'.format(sublist_num)) ensure_path_exists(sublist_dir) create_sublist_symlinks(sublist, sublist_dir, 1000) job_file_path = create_job_file(job_dir, sublist_dir, output_directory, sublist_num, args) if args.submit: result = submit_pbs_job(job_file_path, queue=args.queue) if result: jobs_submitted_success += 1 else: job_file_paths_failed.append(job_file_path) print('Successfully submitted {} jobs with {} failed submissions'.format(jobs_submitted_success, len(job_file_paths_failed))) for path in job_file_paths_failed: print('FAILED TO SUBMIT', path)
def process_and_run_chunk(filepaths_args_tuple): ''' Generates reformatted files for each file path in list_of_file_paths, saves them to a single temp directory, and calls DNorm on each individual file using subprocess ''' list_of_file_paths, args, q = filepaths_args_tuple if not list_of_file_paths: return qh = logging.handlers.QueueHandler(q) l = logging.getLogger() parsed_files = [parse_parform_file(file_path) for file_path in list_of_file_paths] # filter out files with no title line # (for which parse_parform_file returned None) parsed_files = [f for f in parsed_files if f] reformatted_files = [parform_to_pubtator(escaped_doi, title_line, body) for escaped_doi, title_line, body in parsed_files] reqs = {'banner_ncbidisease': os.path.join(args.dnorm, 'config', 'banner_NCBIDisease_UMLS2013AA_TEST.xml'), 'ctd_diseases': os.path.join(args.dnorm, 'data', 'CTD_diseases.tsv'), 'simmatrix': os.path.join(args.dnorm, 'output', 'simmatrix_NCBIDisease_e4.bin'), 'ab3p_path': os.path.join(args.dnorm, '..', 'Ab3P-v1.5')} for required_file in reqs: file_exists_or_exit(reqs[required_file]) with tempfile.TemporaryDirectory() as input_tempdir, tempfile.TemporaryDirectory() as output_tempdir, tempfile.TemporaryDirectory() as dnorm_tempdir: for doi_filename, file_info in reformatted_files: save_file(doi_filename, file_info, input_tempdir) try: out = subprocess.check_output(['bash', 'ApplyDNorm.sh', reqs['banner_ncbidisease'], reqs['ctd_diseases'], reqs['simmatrix'], reqs['ab3p_path'], dnorm_tempdir, os.path.join(input_tempdir, doi_filename), os.path.join(output_tempdir, doi_filename)], cwd=args.dnorm, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as err: string_error = err.output.decode(encoding='UTF-8').rstrip('\n') l.critical('DNorm error: {}'.format(string_error)) l.critical('DNorm error while processing chunk: {}'.format(list_of_file_paths)) # grab all new output files and copy to args.output_directory all_tempfiles = glob(os.path.join(output_tempdir, '*')) try: subprocess.check_output(['cp', '-t', args.output_directory+'/'] + all_tempfiles) except subprocess.CalledProcessError: l.critical('Copy error, chunk: {}'.format(list_of_file_paths))
def main(args): ensure_path_exists(args.output_directory) # pmid_path = os.path.join(args.output_directory, 'by_pmid') # ensure_path_exists(pmid_path) if args.doiindex: file_exists_or_exit(args.doiindex) # doi_path = os.path.join(args.output_directory, 'by_doi') # ensure_path_exists(doi_path) pmid_doi_map = create_pmid_doi_mapping(args) all_files = glob(os.path.join(args.xml_file_directory, '*')) doi_filenames = [] pmid_filenames = [] skipped_files = 0 total_citation_count = 0 print('Processing {} XML files...'.format(len(all_files))) for xml_filepath in tqdm(all_files): root = get_root_object(xml_filepath) citations = get_element('MedlineCitation', root) for citation in citations: # update save directories every 10k files. # # directory structure will be: # [base_dir]/0000/by_pmid/ # [base_dir]/0000/by_doi/ # [base_dir]/0001/by_pmid/ # [base_dir]/0001/by_doi/ # ... # # (no subdirectory will have > 10000 files) if total_citation_count % 10000 == 0: pmid_path = os.path.join(args.output_directory, '{0:0>4}'.format(total_citation_count//10000), 'by_pmid') ensure_path_exists(pmid_path) if args.doiindex: doi_path = os.path.join(args.output_directory, '{0:0>4}'.format(total_citation_count//10000), 'by_doi') ensure_path_exists(doi_path) total_citation_count += 1 abstracts = get_element('AbstractText', citation) if not abstracts: # not all MedlineCitations have Abstracts... continue # grab one of the abstracts identified in this citation and identify # its parent XML elements first_abstract_section = abstracts[0] d = get_abstract_parent_info(first_abstract_section) pmid = d['pmid'] title = d['title'] if not pmid or not d['title']: # if no PMID, no easy way to identify article (skip) # if no title, nothing to include in PubTator title line (skip) skipped_files += 1 continue combined_abstract = combine_all_abstract_text_tags(d['parent_abstract']) file_lines = create_filelines(title, combined_abstract) doi_file_name = pmid_doi_map.get(pmid) if doi_file_name: # use DOI file name and file path doi_filenames.append(save_file(doi_file_name, file_lines, doi_path)) else: # use PMID file name and file path pmid_filenames.append(save_file(pmid, file_lines, pmid_path)) print('{} files saved by PMID to {}'.format(len(pmid_filenames), pmid_path)) if args.doiindex: print('{} files saved by DOI to {}'.format(len(doi_filenames), doi_path)) print('{} MEDLINE citations skipped'.format(skipped_files))
def process_and_run_chunk(filepaths_args_tuple): ''' Generates reformatted files for each file path in list_of_file_paths, saves them to a single temp directory, and calls DNorm on each individual file using subprocess ''' list_of_file_paths, args, q = filepaths_args_tuple if not list_of_file_paths: return qh = logging.handlers.QueueHandler(q) l = logging.getLogger() parsed_files = [ parse_parform_file(file_path) for file_path in list_of_file_paths ] # filter out files with no title line # (for which parse_parform_file returned None) parsed_files = [f for f in parsed_files if f] reformatted_files = [ parform_to_pubtator(escaped_doi, title_line, body) for escaped_doi, title_line, body in parsed_files ] reqs = { 'banner_ncbidisease': os.path.join(args.dnorm, 'config', 'banner_NCBIDisease_UMLS2013AA_TEST.xml'), 'ctd_diseases': os.path.join(args.dnorm, 'data', 'CTD_diseases.tsv'), 'simmatrix': os.path.join(args.dnorm, 'output', 'simmatrix_NCBIDisease_e4.bin'), 'ab3p_path': os.path.join(args.dnorm, '..', 'Ab3P-v1.5') } for required_file in reqs: file_exists_or_exit(reqs[required_file]) with tempfile.TemporaryDirectory( ) as input_tempdir, tempfile.TemporaryDirectory( ) as output_tempdir, tempfile.TemporaryDirectory() as dnorm_tempdir: for doi_filename, file_info in reformatted_files: save_file(doi_filename, file_info, input_tempdir) try: out = subprocess.check_output([ 'bash', 'ApplyDNorm.sh', reqs['banner_ncbidisease'], reqs['ctd_diseases'], reqs['simmatrix'], reqs['ab3p_path'], dnorm_tempdir, os.path.join(input_tempdir, doi_filename), os.path.join(output_tempdir, doi_filename) ], cwd=args.dnorm, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as err: string_error = err.output.decode(encoding='UTF-8').rstrip('\n') l.critical('DNorm error: {}'.format(string_error)) l.critical('DNorm error while processing chunk: {}'.format( list_of_file_paths)) # grab all new output files and copy to args.output_directory all_tempfiles = glob(os.path.join(output_tempdir, '*')) try: subprocess.check_output(['cp', '-t', args.output_directory + '/'] + all_tempfiles) except subprocess.CalledProcessError: l.critical('Copy error, chunk: {}'.format(list_of_file_paths))
def main(args): ensure_path_exists(args.output_directory) # pmid_path = os.path.join(args.output_directory, 'by_pmid') # ensure_path_exists(pmid_path) if args.doiindex: file_exists_or_exit(args.doiindex) # doi_path = os.path.join(args.output_directory, 'by_doi') # ensure_path_exists(doi_path) pmid_doi_map = create_pmid_doi_mapping(args) all_files = glob(os.path.join(args.xml_file_directory, '*')) doi_filenames = [] pmid_filenames = [] skipped_files = 0 total_citation_count = 0 print('Processing {} XML files...'.format(len(all_files))) for xml_filepath in tqdm(all_files): root = get_root_object(xml_filepath) citations = get_element('MedlineCitation', root) for citation in citations: # update save directories every 10k files. # # directory structure will be: # [base_dir]/0000/by_pmid/ # [base_dir]/0000/by_doi/ # [base_dir]/0001/by_pmid/ # [base_dir]/0001/by_doi/ # ... # # (no subdirectory will have > 10000 files) if total_citation_count % 10000 == 0: pmid_path = os.path.join( args.output_directory, '{0:0>4}'.format(total_citation_count // 10000), 'by_pmid') ensure_path_exists(pmid_path) if args.doiindex: doi_path = os.path.join( args.output_directory, '{0:0>4}'.format(total_citation_count // 10000), 'by_doi') ensure_path_exists(doi_path) total_citation_count += 1 abstracts = get_element('AbstractText', citation) if not abstracts: # not all MedlineCitations have Abstracts... continue # grab one of the abstracts identified in this citation and identify # its parent XML elements first_abstract_section = abstracts[0] d = get_abstract_parent_info(first_abstract_section) pmid = d['pmid'] title = d['title'] if not pmid or not d['title']: # if no PMID, no easy way to identify article (skip) # if no title, nothing to include in PubTator title line (skip) skipped_files += 1 continue combined_abstract = combine_all_abstract_text_tags( d['parent_abstract']) file_lines = create_filelines(title, combined_abstract) doi_file_name = pmid_doi_map.get(pmid) if doi_file_name: # use DOI file name and file path doi_filenames.append( save_file(doi_file_name, file_lines, doi_path)) else: # use PMID file name and file path pmid_filenames.append(save_file(pmid, file_lines, pmid_path)) print('{} files saved by PMID to {}'.format(len(pmid_filenames), pmid_path)) if args.doiindex: print('{} files saved by DOI to {}'.format(len(doi_filenames), doi_path)) print('{} MEDLINE citations skipped'.format(skipped_files))
def main(args): # make all paths absolute (to simplify things later) args.paragraph_path = os.path.abspath(args.paragraph_path) args.output_directory = os.path.abspath(args.output_directory) args.logdir = os.path.abspath(args.logdir) args.tmchem = os.path.abspath(args.tmchem) args.bioshovel = os.path.abspath(args.bioshovel) file_exists_or_exit(os.path.join(args.tmchem, 'tmChem.pl')) # get all files (recursive) print('Organizing input files and submitting jobs...') if args.resume: # filter out filenames that are already done... args.resume = os.path.abspath(args.resume) file_exists_or_exit(args.resume) print('Resuming job submission from path: {}'.format(args.resume)) if os.path.exists(args.output_directory): print('Output directory changed to avoid file conflicts:') print('OLD: ' + args.output_directory) args.output_directory += '_resume' print('NEW: ' + args.output_directory) print('Reading previously completed files...') # this set may use a lot of RAM if args.resume path contains a ton of files... done_files = set( p.name.rstrip('.tmChem') for p in tqdm(Path(args.resume).glob('**/*')) if p.is_file()) all_files = (str(f) for f in tqdm(Path(args.paragraph_path).glob('**/*')) if f.is_file() and f.name not in done_files) else: all_files = (str(f) for f in tqdm(Path(args.paragraph_path).glob('**/*')) if f.is_file()) # divide list into chunks of size n filelist_with_sublists = create_sublists_sized_n(all_files, args.nfiles) # move job files into the appropriate subdirectories # (don't store more than 1k per sub-subdirectory) # create job file for each subdirectory # submit jobs base_input_directory = os.path.join(args.output_directory, 'input_files') job_dir = os.path.join(args.output_directory, 'job_files') output_directory = os.path.join(args.output_directory, 'output') for path in (base_input_directory, job_dir, output_directory): ensure_path_exists(path) job_file_paths_failed = [] jobs_submitted_success = 0 for sublist_num, sublist in enumerate(filelist_with_sublists): sublist_dir = os.path.join(base_input_directory, 'sublist_{0:0>4}'.format(sublist_num)) ensure_path_exists(sublist_dir) create_sublist_symlinks(sublist, sublist_dir, 1000) job_file_path = create_job_file(job_dir, sublist_dir, output_directory, sublist_num, args) if args.submit: result = submit_pbs_job(job_file_path, queue=args.queue) if result: jobs_submitted_success += 1 else: job_file_paths_failed.append(job_file_path) print('Successfully submitted {} jobs with {} failed submissions'.format( jobs_submitted_success, len(job_file_paths_failed))) for path in job_file_paths_failed: print('FAILED TO SUBMIT', path)