print(dbfile) if not os.path.isfile(dbfile): print('No database file found!') quit() logfile = open(os.path.join(test_output_folder, 'log.txt'), 'w') print('Starting. %s' % time.strftime('%d/%m/%y %H:%M:%S')) starttime = time.time() logit('Starting. %s' % time.strftime('%d/%m/%y %H:%M:%S')) logit('Spectral path: %s' % test_spectral_database_path) logit('Chemical Databases Included: %s' % test_chemical_databases) precisions = [0.005] for precision in precisions: specmanager = SpectralManager() logit('Reading test spectra...') #print(test_spectral_database_path); specmanager.import_textfile_spectra_from_folder( test_spectral_database_path) logit('Finshed reading spectra. Total number: %s' % len(specmanager.ms_spectra)) logtime() #selected_InChI=load_inchi_list('e:/Imperial/HMDB/PubChem_Test/InChI_Selected100.list'); #remove_spectra_not_in_inchi_list(selected_InChI,['HMDB','MassBank']); remove_low_precision_spectra(precision)
print(settings.program_description) settings.parse_command_line_args() print(settings.format_parameters()) #Print current parameters print('\nStarting.....') if settings.parameters['no_output'] == 'no': if not os.path.isdir(settings.parameters['output']): os.makedirs(settings.parameters['output']) else: print( 'Error! Output path already exists! Please provide a new path to avoid overwriting of existing files!' ) #sys.exit(-1); with SpectralManager() as spectral_manager: if os.path.isfile(settings.parameters['inputdatapath']): import_spectra_from_file(settings.parameters['inputdatapath'], spectral_manager, settings.parameters) else: flist = get_files_list(settings.parameters['inputdatapath'], settings.parameters['recursive'] == 'yes') for fname in flist: import_spectra_from_file(fname, spectral_manager, settings.parameters) if settings.parameters['no_output'] == 'no': postprocess_spectra(spectral_manager, settings.parameters) spectral_manager.export_textfile_spectra_to_folder( settings.parameters['output'])
os.makedirs(outpath) specfolder = os.path.join(outpath, 'Spectra') if not os.path.exists(specfolder): os.makedirs(specfolder) generate_index(outpath) generate_main(outpath, specmanager) print('Generating reports:\n') for i in range(len(specmanager.ms_spectra)): print_in_the_same_line('\rSpectrum %s of %s' % (i + 1, len(specmanager.ms_spectra))) spectrum = specmanager.ms_spectra[i] generate_spectrum(specfolder, i, spectrum) if hasattr(spectrum, '_title'): del spectrum._title print('\nFinished') if __name__ == '__main__': spectral_input_path = 'e:/Imperial/TestDB/totest' outpath = 'e:/Imperial/TestDB/totestHTMLReport' from chemdistiller.msspectra.manager import SpectralManager specmanager = SpectralManager() specmanager.import_textfile_spectra_from_folder(spectral_input_path) generate_HTML_report(outpath, specmanager) specmanager.close()
def run_annotation(spectral_input_path, db_file, chemical_databases, output_folder, SVMs_path, ncpu, ppm, max_results_per_query, test_mode=False, ignore_peaks_without_spectra=True): global starttime; global logfile; if not os.path.isfile(db_file): print('No database list file found!'); quit(); db_files=[db_file]; if not os.path.exists(output_folder): try: os.makedirs(output_folder); except: print('Cannot create output path %s !'%output_folder); print('Run: python annotate.py -h for help'); sys.exit(2); logfile=open(os.path.join(output_folder,'log.txt'),'w'); starttime=time.time(); logit('Starting. %s'%time.strftime('%d/%m/%y %H:%M:%S')); logit('Input Spectral path: %s'%spectral_input_path); logit('Output path: %s'%output_folder) if chemical_databases: logit('Chemical Databases Included: %s'%chemical_databases); else: logit('Chemical Databases Included: All'); logit('from chemical database paths file %s'%db_file); logit('SVMs from %s'%SVMs_path); logit('nCPUs to use: %s'%ncpu); logit('Default ppm tolerance for MS1: %s'%ppm); specmanager=SpectralManager(); logit('Reading test spectra...'); specmanager.import_textfile_spectra_from_folder(spectral_input_path); logit('Finshed reading spectra. Total number: %s'%len(specmanager.ms_spectra)); logtime(); multiprocessing.freeze_support(); scorers_list=['FingerPrintScorer','FragPrintScorer']; scorers_settings=[[],[]]; total_score=total_multiplicative_score; with PeakAnnotator(db_files, SVMs_path, ncpu) as peak_annotator: logit('Beginning annotation...'); logtime(); total_peaks=0; best_results_neg=[0]*max_results_per_query; worst_results_neg=[0]*max_results_per_query; return_number_neg=[0]*max_results_per_query; neg_mode_count=0; best_results_pos=[0]*max_results_per_query; worst_results_pos=[0]*max_results_per_query; return_number_pos=[0]*max_results_per_query; pos_mode_count=0; peaks=[]; supported_adducts=set(global_supported_adducts); #Collect all supported adducts #if 'FingerPrintScorer' in scorers_list: # supported_adducts=supported_adducts|FingerPrintScorer.supported_adducts; #if 'FragPrintScorer' in scorers_list: # supported_adducts=supported_adducts|FragPrintScorer.supported_adducts; #Leave only adducts supported by all selected filters if 'FingerPrintScorer' in scorers_list: supported_adducts=supported_adducts&FingerPrintScorer.supported_adducts; if 'FragPrintScorer' in scorers_list: supported_adducts=supported_adducts&FragPrintScorer.supported_adducts; logit('Preparing annotations'); if test_mode: logit('Running in test mode.....'); logit('Assuming correct adduct info and 0 isotope.'); logit('No formula assumption. Mass precision: %s ppm'%ppm); logit('Only considering [M+H]+ and [M-H]- adducts for now'); print('\n'); for spectrum_index in range(len(specmanager.ms_spectra)): #print('\r%s of %s spectra preprocessed'%(spectrum_index,len(specmanager.ms_spectra))); spectrum=specmanager.ms_spectra[spectrum_index]; for peak in spectrum.peaks: if (not ignore_peaks_without_spectra) or (hasattr(peak,'ms_spectra') and peak.ms_spectra): if hasattr(peak,'parameters') and ('ion_type' in peak.parameters): if (peak.parameters['ion_type'] in FragPrintScorer.supported_adducts): peak.ppm=ppm; shortinchi=get_short_inchi_from_full_inchi(peak.parent_spectrum.parameters['inchi']); annotation=MSPeakAnnotation(peak, adduct=\ get_adduct_by_name(peak.parameters['ion_type'], \ spectrum.parameters['mode']), isotope=0, \ formula_scorer=None, element_scorer=None, filters=[], scores={}); #testformulascorer=FormulaScorer(); #testformulascorer.setup_scorer(peak.parent_spectrum.parameters['formula'],1.0); #testelementscorer=ElementScorer(); #testelementscorer.setup_scorer({'C':0.8,'O':0.9,'Si':0.1}); #formulafilter=FormulasFilter(formulas=[peak.parent_spectrum.parameters['formula']]); #elementfilter=ElementCompositionFilter(peak.parent_spectrum.parameters['formula'], peak.parent_spectrum.parameters['formula']); #inchifilter=InChIFilter(ref_inchi=peak.parent_spectrum.parameters['inchi'],use_short_inchi=False, match_type=0) #annotation=MSPeakAnnotation(peak, adduct=\ # get_adduct_by_name(peak.parameters['ion_type'], \ # spectrum.parameters['mode']), isotope=0, \ # formula_scorer=testformulascorer, element_scorer=testelementscorer, \ # filters=[formulafilter, inchifilter, elementfilter], scores={'ExtraScore':25.0, 'AllExtra':1.9}); if not (annotation.adduct is None): peak.annotations=[annotation]; peaks.append(peak); else: logit('Unsupported Adduct: %s'%peak.parameters['ion_type']); else: logit('Running in normal mode.....'); logit('Mass precision: %s ppm'%ppm); print('\n'); for spectrum_index in range(len(specmanager.ms_spectra)): #print('\r%s of %s spectra preprocessed'%(spectrum_index,len(specmanager.ms_spectra))); spectrum=specmanager.ms_spectra[spectrum_index]; for peak in spectrum.peaks: if (not ignore_peaks_without_spectra) or (hasattr(peak,'ms_spectra') and peak.ms_spectra): peak.ppm=ppm; generate_annotation=True; if hasattr(peak, 'annotations'): if peak.annotations: generate_annotation=False; peaks.append(peak); if generate_annotation: #Assuming isotope 0 by default if spectrum.parameters['mode']==1: selected_adducts=get_positive_mode_adducts(supported_adducts); else: selected_adducts=get_negative_mode_adducts(supported_adducts); peak.annotations=[]; for adduct in selected_adducts: annotation=MSPeakAnnotation(peak, adduct=adduct, isotope=0, formula_scorer=None, element_scorer=None, filters=[], scores={}); peak.annotations.append(annotation); peaks.append(peak); logit('Peaks to annotate: %s'%len(peaks)); if test_mode: required_fields=set(['ShortInChI','InChI','SMILES','Formula','IDs']); #required_fields=set(['ShortInChI','InChI','SMILES','Formula','IDs', 'InChIKeyValues', 'InChiKey', 'FormulaVector', 'ElementVector', 'Frag', 'FPT']); else: required_fields=set(['InChI','SMILES','Formula','IDs']); peak_annotator.annotate_peaks(peaks, chemical_databases, \ scorers_list=scorers_list, scorers_settings=scorers_settings, total_score=total_score, required_fields=required_fields,\ results_limit=max_results_per_query, save_memory=False, ppm=ppm, overwrite=True); annotation_count=0; total_candidates=0; total_peaks=len(peaks); for peakindex in range(total_peaks): peak=peaks[peakindex]; if peak.annotations: for annotation in peak.annotations: if not (annotation.mol_candidates is None): annotation_count+=1; total_candidates+=annotation.mol_candidates.total_candidate_count; logit('Finished annotating... '); logit('Annotated peaks :%s'%total_peaks); if total_peaks>0: logit('Averaged annotations per peak:%s'%(float(annotation_count)/total_peaks)); logit('Averaged candidate molecules per peak:%s'%(float(total_candidates)/total_peaks)); #Calculating retrieval statistics using test data #===================================================================== if test_mode: #logit('Removing peaks with no annotations found from consideration...'); #for i in reversed(range(len(peaks))): # if not peaks[i].annotations: # del peaks[i]; #logit('Finished removing... Annotated peaks :%s'%len(peaks)); total_peaks=len(peaks); print('Calculating retrieval stats...\n'); for peakindex in range(total_peaks): peak=peaks[peakindex]; peakmode=0; if peak.parent_spectrum.parameters['mode']==1: pos_mode_count+=1; peakmode=1; elif peak.parent_spectrum.parameters['mode']==-1: neg_mode_count+=1; peakmode=-1; #print('\r%s of %s '%(peakindex,total_peaks)); shortinchi=get_short_inchi_from_full_inchi(peak.parent_spectrum.parameters['inchi']); for annotation in peak.annotations: annotation.min_correct=-1; annotation.max_correct=-1; annotation.mean_score=0.0; annotation.max_score=0.0; for index in range(len(annotation.mol_candidates.mol_list)): total_score=annotation.mol_candidates.mol_list[index]['TotalScore']; annotation.mean_score+=total_score; if total_score>annotation.max_score: annotation.max_score=total_score; if len(annotation.mol_candidates.mol_list)>0: annotation.mean_score=annotation.mean_score/len(annotation.mol_candidates.mol_list); if len(annotation.mol_candidates.mol_list)<=max_results_per_query: if peakmode==1: return_number_pos[len(annotation.mol_candidates.mol_list)-1]+=1; elif peakmode==-1: return_number_neg[len(annotation.mol_candidates.mol_list)-1]+=1; for index in reversed(range(1,len(annotation.mol_candidates.mol_list))): if annotation.mol_candidates.mol_list[index-1]['ShortInChI']==annotation.mol_candidates.mol_list[index]['ShortInChI']: del annotation.mol_candidates.mol_list[index]; # For the purpose of statistics condensing sequential identical Short InChI-s for index in range(len(annotation.mol_candidates.mol_list)): if annotation.mol_candidates.mol_list[index]['ShortInChI']==shortinchi: annotation.mol_candidates.mol_list[index]['Annotation']='Correct'; if annotation.min_correct==-1: annotation.min_correct=index; annotation.max_correct=index; else: annotation.mol_candidates.mol_list[index]['Annotation']='Wrong'; if annotation.min_correct>-1: if peakmode==1: for i in range(len(best_results_pos)): if annotation.min_correct<=i: best_results_pos[i]+=1; if annotation.max_correct<=i: worst_results_pos[i]+=1; elif peakmode==-1: for i in range(len(best_results_neg)): if annotation.min_correct<=i: best_results_neg[i]+=1; if annotation.max_correct<=i: worst_results_neg[i]+=1; logit('Finished Calculating Retrieval Stats.'); logit('Positive Mode Count: %s'%pos_mode_count); logit('Negative Mode Count: %s'%neg_mode_count); logit('Total Peaks: %s'%total_peaks); logit('Positive Mode (Total: %s ):'%pos_mode_count); if pos_mode_count>0: for i in range(max_results_per_query): logit('Correct within first\t%s:\tBest:\t%.2f%%\tWorst:\t%.2f%%\tCCount:\t%s'%(i+1,float(best_results_pos[i])*100/pos_mode_count,worst_results_pos[i]*100/pos_mode_count,return_number_pos[i])); logit('Negative Mode (Total: %s ):'%neg_mode_count); if neg_mode_count>0: for i in range(max_results_per_query): logit('Correct within first\t%s:\tBest:\t%.2f%%\tWorst:\t%.2f%%\tCCount:\t%s'%(i+1,float(best_results_neg[i])*100/neg_mode_count,worst_results_neg[i]*100/neg_mode_count,return_number_neg[i])); logit('Both Modes (Total: %s ):'%(pos_mode_count+neg_mode_count)); if neg_mode_count>0 or pos_mode_count>0: for i in range(max_results_per_query): logit('Correct within first\t%s:\tBest:\t%.2f%%\tWorst:\t%.2f%%\tCCount:\t%s'%(i+1,float(best_results_neg[i]+best_results_pos[i])*100/(neg_mode_count+pos_mode_count),float(worst_results_neg[i]+worst_results_pos[i])*100/(neg_mode_count+pos_mode_count),return_number_pos[i]+return_number_neg[i])); for peak in peaks: merge_annotations(peak, remove_old_annotations=False); #Finished calculating retrieval statistics using test data #===================================================================== logtime(); logit('Finished Annotation. Exporting results...'); logtime(); logit('Exporting to JSON...'); spectra_to_json(os.path.join(output_folder,'annotated_spectra.json'), specmanager.ms_spectra); logtime(); logit('Exporting to internal text format...'); specmanager.export_textfile_spectra_to_folder(os.path.join(output_folder,'annotated_spectra')); logit('Preparing HTML report...'); generate_HTML_report(os.path.join(output_folder,'Report'), specmanager); specmanager.close(); logtime(); logit('Finished'); logfile.close();
print('No database file found!'); quit(); logfile=open(os.path.join(test_output_folder,'log.txt'),'w'); print('Starting. %s'%time.strftime('%d/%m/%y %H:%M:%S')); starttime=time.time(); logit('Starting. %s'%time.strftime('%d/%m/%y %H:%M:%S')); logit('Spectral path: %s'%test_spectral_database_path); logit('Chemical Databases Included: %s'%test_chemical_databases); specmanager=SpectralManager(); logit('Reading test spectra...'); #print(test_spectral_database_path); specmanager.import_textfile_spectra_from_folder(test_spectral_database_path); logit('Finshed reading spectra. Total number: %s'%len(specmanager.ms_spectra)); logtime(); selected_InChi=load_inchi_list('e:/Imperial/HMDB/PubChem_Test/InChi_Selected100.list'); #remove_spectra_not_in_inchi_list(selected_InChi,['HMDB','MassBank']); remove_low_precision_spectra(0.001); correct_precursor_mz(specmanager); specmanager.export_textfile_spectra_to_folder('e:/Imperial/TestDB/Precise_0.001');
#from chemdistiller.scorers.totalscore import total_multiplicative_score; #from chemdistiller.scorers.fragprint import FragPrintScorer; trainSVM_spectral_database_path = 'e:/Imperial/TestDB/NIST14_HMDB_MassBank_5fold_CV_FPT3' trainSVM_output_folder = 'e:/Imperial/TestDB/SVM_Train' fpt_mask_file = 'e:/Imperial/TestDB/FPT3_0.05-0.95_2567.txt' if not os.path.exists(trainSVM_output_folder): os.makedirs(trainSVM_output_folder) nbatch = 5 print('Loading spectra....') specmanager = SpectralManager() specmanager.import_textfile_spectra_from_folder( trainSVM_spectral_database_path) print('Finished reading spectra!') #%% print('Positive mode:') maxmz = 0.0 for spectrum in specmanager.ms_spectra: if spectrum.parameters['mode'] == 1: if float(spectrum.parameters['exactmass']) > maxmz: maxmz = float(spectrum.parameters['exactmass']) print(maxmz)