def test_calculatetrans(self): print '\nTesting calculate_transitions' myprecursors = ((500, 'PEPTIDE', 1, 1, 0), (400, 'CEPC[160]IDM[147]E', 2, 2, 0)) st = time.time() LARGENUMBER = 100000 for i in range(LARGENUMBER): tr_new = c_getnonuis.calculate_transitions_ch( myprecursors, [1, 2], self.q3_low, self.q3_high) ctime = time.time() - st st = time.time() coll = collider.SRMcollider() for i in range(LARGENUMBER): tr_old = list( collider.SRMcollider._get_all_collisions_calculate_sub( coll, myprecursors, self.par, self.R, self.q3_low, self.q3_high)) oldtime = time.time() - st tr_new.sort(mysort) tr_old.sort(mysort) self.assertEqual(len(tr_new), len(tr_old)) for o, n in zip(tr_old, tr_new): for oo, nn in zip(o, n): self.assertTrue(oo - nn < 1e-5) print ctime, oldtime print "Speedup:", oldtime / ctime
def test_calculatetrans2(self): print '\nTesting calculate_transitions 2' pep = test_shared.runpep2 transitions = test_shared.runtransitions2 precursors = test_shared.runprecursors2 transitions = tuple([ (t[0], i) for i,t in enumerate(transitions)]) st = time.time() collisions = list(collider.SRMcollider._get_all_collisions_calculate_sub( collider.SRMcollider(), precursors, self.par, self.R, self.q3_low, self.q3_high)) oldtime = time.time() - st st = time.time() tr_new = c_getnonuis.calculate_transitions_ch(tuple(precursors), [1,2], self.q3_low, self.q3_high) ctime = time.time() - st tr_new.sort(mysort) collisions.sort(mysort) self.assertEqual(len(tr_new), len(collisions)) for o,n in zip(collisions, tr_new): for oo,nn in zip(o,n): self.assertTrue(oo - nn < 1e-5) print ctime, oldtime print "Speedup:", oldtime / ctime
def test_calculatetrans_single(self): pep = test_shared.runpep2 transitions = test_shared.runtransitions2 precursors = [p for p in test_shared.runprecursors2 if p[2] == 106423] transitions = tuple([(t[0], i) for i, t in enumerate(transitions)]) st = time.time() collisions = list( collider.SRMcollider._get_all_collisions_calculate_sub( collider.SRMcollider(), precursors, self.par, self.R, self.q3_low, self.q3_high)) oldtime = time.time() - st st = time.time() tr_new = c_getnonuis.calculate_transitions_ch(tuple(precursors), [1, 2], self.q3_low, self.q3_high) ctime = time.time() - st tr_new.sort(mysort) collisions.sort(mysort) self.assertEqual(len(tr_new), len(collisions)) for o, n in zip(collisions, tr_new): for oo, nn in zip(o, n): self.assertTrue(oo - nn < 1e-5)
def test_calculatetrans1(self): print '\nTesting calculate_transitions 1' pep = test_shared.runpep1 transitions = test_shared.runtransitions1 precursors = test_shared.runprecursors1[240:300] #precursors = test_shared.runprecursors1[2], transitions = tuple([(t[0], i) for i, t in enumerate(transitions)]) st = time.time() collisions = list( collider.SRMcollider._get_all_collisions_calculate_sub( collider.SRMcollider(), precursors, self.par, self.R, self.q3_low, self.q3_high)) oldtime = time.time() - st st = time.time() tr_new = c_getnonuis.calculate_transitions_ch(tuple(precursors), [1, 2], self.q3_low, self.q3_high) ctime = time.time() - st tr_new.sort(mysort) collisions.sort(mysort) self.assertEqual(len(tr_new), len(collisions)) for o, n in zip(collisions, tr_new): for oo, nn in zip(o, n): self.assertTrue(oo - nn < 1e-5) print ctime, oldtime print "Speedup:", oldtime / ctime
def test_calculatetrans(self): print '\nTesting calculate_transitions' myprecursors = ((500, 'PEPTIDE', 1, 1, 0), (400, 'CEPC[160]IDM[147]E', 2, 2, 0)) st = time.time() LARGENUMBER = 100000 for i in range(LARGENUMBER): tr_new = c_getnonuis.calculate_transitions_ch(myprecursors, [1,2], self.q3_low, self.q3_high) ctime = time.time() - st st = time.time() coll = collider.SRMcollider() for i in range(LARGENUMBER): tr_old = list(collider.SRMcollider._get_all_collisions_calculate_sub( coll, myprecursors, self.par, self.R, self.q3_low, self.q3_high)) oldtime = time.time() - st tr_new.sort(mysort) tr_old.sort(mysort) self.assertEqual(len(tr_new), len(tr_old)) for o,n in zip(tr_old, tr_new): for oo,nn in zip(o,n): self.assertTrue(oo - nn < 1e-5) print ctime, oldtime print "Speedup:", oldtime / ctime
def calculate_transitions_ch(peptides, charges, q3_low, q3_high): try: import c_getnonuis return c_getnonuis.calculate_transitions_ch(peptides, charges, q3_low, q3_high) except ImportError: return list(_calculate_transitions_ch(peptides, charges, q3_low, q3_high))
def calculate_transitions_ch(peptides, charges, q3_low, q3_high): try: import c_getnonuis return c_getnonuis.calculate_transitions_ch(peptides, charges, q3_low, q3_high) except ImportError: return list( _calculate_transitions_ch(peptides, charges, q3_low, q3_high))
def calculate_transitions(self, q3_low, q3_high, charges=[1]): import c_getnonuis transitions = c_getnonuis.calculate_transitions_ch( ((self.q1, self.modified_sequence, self.parent_id),), charges, q3_low, q3_high ) # fake some srm_id for the transitions, so that the returned transitions will be tuples of (q1, id) return tuple([(t[0], i) for i, t in enumerate(transitions)])
def test_calculate_transitions_inner(self): mypep = self.pep1 transitions = c_getnonuis.calculate_transitions_ch((mypep,), [2], 0, 5000) transitions = [t[0] for t in transitions] self.assertEqual(len(transitions), 12) self.assertEqual(len(transitions), 2 * len(mypep[1]) - 2) y_series = transitions[: len(mypep[1]) - 1] b_series = transitions[len(mypep[1]) - 1 :] self.assertEqual(len(y_series), 6) self.assertEqual(len(b_series), 6) for calc, ref in zip(y_series, self.pep1_yseries): self.assertTrue(abs(calc - ref) < 1e-3) for calc, ref in zip(b_series, self.pep1_bseries): self.assertTrue(abs(calc - ref) < 1e-3)
def test_calculate_transitions_modifcation(self): trans = c_getnonuis.calculate_transitions_ch((self.pep2,), [1, 2], 300, 1500) # TODO check self.assertTrue(abs(trans[0][0]) - 909.333 < 1e-3) self.assertTrue(abs(trans[1][0]) - 780.290 < 1e-3) self.assertTrue(abs(trans[2][0]) - 683.238 < 1e-3) self.assertTrue(abs(trans[3][0]) - 523.207 < 1e-3) self.assertTrue(abs(trans[4][0]) - 410.123 < 1e-3) self.assertTrue(abs(trans[5][0]) - 330.112 < 1e-3) self.assertTrue(abs(trans[6][0]) - 490.143 < 1e-3) self.assertTrue(abs(trans[7][0]) - 603.227 < 1e-3) self.assertTrue(abs(trans[8][0]) - 718.254 < 1e-3) self.assertTrue(abs(trans[9][0]) - 865.289 < 1e-3) self.assertTrue(abs(trans[10][0]) - 455.170 < 1e-3) self.assertTrue(abs(trans[11][0]) - 390.649 < 1e-3) self.assertTrue(abs(trans[12][0]) - 342.122 < 1e-3) self.assertTrue(abs(trans[13][0]) - 302.117 < 1e-3) self.assertTrue(abs(trans[14][0]) - 359.630 < 1e-3) self.assertTrue(abs(trans[15][0]) - 433.148 < 1e-3)
def test_integrated_cpp(self): """ Compare the fully integrated vs the mixed C++ rangetree / Python solution. Here we are comparing the fully integrated solution of storing all precursors in a C++ range tree and never passing them to Python vs the solution where we store the precursors in the rangetree, pass them to Python and then evaluate them. """ verbose = True verbose = False print '\n'*1 print "Comparing fully integrated solution (c_integrated.wrap_all)" par = self.par cursor = self.cursor all_precursors = self.precursors_to_evaluate shuffle(all_precursors) all_precursors = all_precursors[:self.limit_large] self.myprecursors.build_parent_id_lookup() testrange = self.myprecursors.build_rangetree() import c_rangetree r = c_rangetree.ExtendedRangetree_Q1_RT.create() r.new_rangetree() r.create_tree(tuple(self.alltuples_isotope_correction)) #c_integrated.create_tree(tuple(self.alltuples_isotope_correction)) MAX_UIS = 5 newtime = 0; oldtime = 0; ctime = 0 oldsql = 0; newsql = 0 alllocaltime = 0 localprecursor = 0 transitiontime = 0 c_fromprecursortime = 0 prepare = [] self._cursor = False print "Running experiment ", par.get_common_filename() print "calc_trans. = time to calculate the transitions of the precursor" print "old = use rangetree to get precursors, use C++ code to get collperpep" print "new = use rangetree to get precursors, use single C++ code to get collperpep" print "i\tcalc_trans.\tnew\t\told\t\t>>\tspeedup" for kk, precursor in enumerate(all_precursors): ii = kk + 1 q1 = precursor.q1 ssrcalc = precursor.ssrcalc sequence = precursor.modified_sequence peptide_key = precursor.transition_group p_id = precursor.parent_id q3_low, q3_high = par.get_q3range_transitions() #new way to calculate the precursors mystart = time.time() transitions = c_getnonuis.calculate_transitions_ch( ((q1, sequence, p_id),), [1,2], q3_low, q3_high) nr_transitions = len( transitions ) #fake some srm_id for the transitions transitions = tuple([ (t[0], i) for i,t in enumerate(transitions)]) q1_low = q1 - par.q1_window q1_high = q1 + par.q1_window innerstart = time.time() #correct rounding errors, s.t. we get the same results as before! ssrcalc_low = ssrcalc - par.ssrcalc_window + 0.001 ssrcalc_high = ssrcalc + par.ssrcalc_window - 0.001 transitiontime += time.time() - mystart isotope_correction = par.isotopes_up_to * Residues.mass_diffC13 / min(par.parent_charges) ################################### # New way to calculate non_uislist # - start out from transitions, get non_uislist mystart = time.time() newresult = c_integrated.wrap_all_bitwise(transitions, q1_low, ssrcalc_low, q1_high, ssrcalc_high, peptide_key, min(MAX_UIS,nr_transitions) , par.q3_window, #q3_low, q3_high, par.ppm, par.isotopes_up_to, isotope_correction, par, r) newtime += time.time() - mystart ################################### # Old way to calculate non_uislist: # - get collisions per peptide # - get non_uislist mystart = time.time() collisions_per_peptide_obj = self.myprecursors.get_collisions_per_peptide_from_rangetree( precursor, precursor.q1 - par.q1_window, precursor.q1 + par.q1_window, transitions, par, r) ## # if False: ## # precursor_ids = tuple(c_rangetree.query_tree( q1_low, ssrcalc_low, ## # q1_high, ssrcalc_high, par.isotopes_up_to, isotope_correction)) ## # precursors = tuple([parentid_lookup[myid[0]] for myid in precursor_ids ## # #dont select myself ## # if parentid_lookup[myid[0]][2] != peptide_key]) ## # collisions_per_peptide = c_getnonuis.calculate_collisions_per_peptide( ## # transitions, precursors, q3_low, q3_high, par.q3_window, par.ppm) non_uis_list = [{} for i in range(MAX_UIS+1)] for order in range(1,MAX_UIS+1): non_uis_list[order] = c_getnonuis.get_non_uis( collisions_per_peptide_obj, order) oldtime += time.time() - mystart non_uis_list_old_way = [set(kk.keys()) for kk in non_uis_list] non_uis_list_len = [len(kk) for kk in non_uis_list_old_way[1:]] ################################### # Assert equality, print out result self.assertEqual( newresult , non_uis_list_len) mys = "%s\t%0.4fms\t%0.2fms\t\t%0.2fms\t\t>>\t%0.2f" % \ (ii, transitiontime *1000/ ii, newtime*1000/ii, oldtime*1000/ii, oldtime *1.0 / newtime) #start a new line for each output? #mys += '\t%s\t%s' % ( len(precursors), len(precursor_new) ) if False: mys += '\n' self.ESC = chr(27) sys.stdout.write(self.ESC + '[2K') if self._cursor: sys.stdout.write(self.ESC + '[u') self._cursor = True sys.stdout.write(self.ESC + '[s') sys.stdout.write(mys) sys.stdout.flush()
def calculate_transitions(self, q3_low, q3_high, charges=[1]): import c_getnonuis transitions = c_getnonuis.calculate_transitions_ch( ((self.q1, self.modified_sequence, self.parent_id),), charges, q3_low, q3_high) # fake some srm_id for the transitions, so that the returned transitions will be tuples of (q1, id) return tuple([ (t[0], i) for i,t in enumerate(transitions)])
def test_integrated_cpp(self): """ Compare the fully integrated vs the mixed C++ rangetree / Python solution. Here we are comparing the fully integrated solution of storing all precursors in a C++ range tree and never passing them to Python vs the solution where we store the precursors in the rangetree, pass them to Python and then evaluate them. """ verbose = True verbose = False print '\n' * 1 print "Comparing fully integrated solution (c_integrated.wrap_all)" par = self.par cursor = self.cursor all_precursors = self.precursors_to_evaluate shuffle(all_precursors) all_precursors = all_precursors[:self.limit_large] self.myprecursors.build_parent_id_lookup() testrange = self.myprecursors.build_rangetree() import c_rangetree r = c_rangetree.ExtendedRangetree_Q1_RT.create() r.new_rangetree() r.create_tree(tuple(self.alltuples_isotope_correction)) #c_integrated.create_tree(tuple(self.alltuples_isotope_correction)) MAX_UIS = 5 newtime = 0 oldtime = 0 ctime = 0 oldsql = 0 newsql = 0 alllocaltime = 0 localprecursor = 0 transitiontime = 0 c_fromprecursortime = 0 prepare = [] self._cursor = False print "Running experiment ", par.get_common_filename() print "calc_trans. = time to calculate the transitions of the precursor" print "old = use rangetree to get precursors, use C++ code to get collperpep" print "new = use rangetree to get precursors, use single C++ code to get collperpep" print "i\tcalc_trans.\tnew\t\told\t\t>>\tspeedup" for kk, precursor in enumerate(all_precursors): ii = kk + 1 q1 = precursor.q1 ssrcalc = precursor.ssrcalc sequence = precursor.modified_sequence peptide_key = precursor.transition_group p_id = precursor.parent_id q3_low, q3_high = par.get_q3range_transitions() #new way to calculate the precursors mystart = time.time() transitions = c_getnonuis.calculate_transitions_ch( ((q1, sequence, p_id), ), [1, 2], q3_low, q3_high) nr_transitions = len(transitions) #fake some srm_id for the transitions transitions = tuple([(t[0], i) for i, t in enumerate(transitions)]) q1_low = q1 - par.q1_window q1_high = q1 + par.q1_window innerstart = time.time() #correct rounding errors, s.t. we get the same results as before! ssrcalc_low = ssrcalc - par.ssrcalc_window + 0.001 ssrcalc_high = ssrcalc + par.ssrcalc_window - 0.001 transitiontime += time.time() - mystart isotope_correction = par.isotopes_up_to * Residues.mass_diffC13 / min( par.parent_charges) ################################### # New way to calculate non_uislist # - start out from transitions, get non_uislist mystart = time.time() newresult = c_integrated.wrap_all_bitwise( transitions, q1_low, ssrcalc_low, q1_high, ssrcalc_high, peptide_key, min(MAX_UIS, nr_transitions), par.q3_window, #q3_low, q3_high, par.ppm, par.isotopes_up_to, isotope_correction, par, r) newtime += time.time() - mystart ################################### # Old way to calculate non_uislist: # - get collisions per peptide # - get non_uislist mystart = time.time() collisions_per_peptide_obj = self.myprecursors.get_collisions_per_peptide_from_rangetree( precursor, precursor.q1 - par.q1_window, precursor.q1 + par.q1_window, transitions, par, r) ## # if False: ## # precursor_ids = tuple(c_rangetree.query_tree( q1_low, ssrcalc_low, ## # q1_high, ssrcalc_high, par.isotopes_up_to, isotope_correction)) ## # precursors = tuple([parentid_lookup[myid[0]] for myid in precursor_ids ## # #dont select myself ## # if parentid_lookup[myid[0]][2] != peptide_key]) ## # collisions_per_peptide = c_getnonuis.calculate_collisions_per_peptide( ## # transitions, precursors, q3_low, q3_high, par.q3_window, par.ppm) non_uis_list = [{} for i in range(MAX_UIS + 1)] for order in range(1, MAX_UIS + 1): non_uis_list[order] = c_getnonuis.get_non_uis( collisions_per_peptide_obj, order) oldtime += time.time() - mystart non_uis_list_old_way = [set(kk.keys()) for kk in non_uis_list] non_uis_list_len = [len(kk) for kk in non_uis_list_old_way[1:]] ################################### # Assert equality, print out result self.assertEqual(newresult, non_uis_list_len) mys = "%s\t%0.4fms\t%0.2fms\t\t%0.2fms\t\t>>\t%0.2f" % \ (ii, transitiontime *1000/ ii, newtime*1000/ii, oldtime*1000/ii, oldtime *1.0 / newtime) #start a new line for each output? #mys += '\t%s\t%s' % ( len(precursors), len(precursor_new) ) if False: mys += '\n' self.ESC = chr(27) sys.stdout.write(self.ESC + '[2K') if self._cursor: sys.stdout.write(self.ESC + '[u') self._cursor = True sys.stdout.write(self.ESC + '[s') sys.stdout.write(mys) sys.stdout.flush()
self = mycollider self.pepids = self._get_unique_pepids_toptransitions(par, cursor) MAX_UIS = par.max_uis common_filename = par.get_common_filename() self.min_transitions = [] start = time.time() progressm = progress.ProgressMeter(total=len(self.pepids), unit='peptides') for i, pep in enumerate(self.pepids): p_id = pep['parent_id'] q1 = pep['q1'] if not use_randomized_transitions and not use_above_precursor: transitions = self._get_all_transitions_toptransitions(par, pep, cursor) elif use_above_precursor: import c_getnonuis transitions = c_getnonuis.calculate_transitions_ch( ((pep['q1'], pep['mod_sequence'], p_id),), [1], 0, 20000) #get all singly charged y ions above the precursor y_series = [(t[0], 0) for t in transitions[:len(transitions)/2] if t[0] > q1] y_series.reverse() y_series = y_series[:5] transitions = y_series elif use_randomized_transitions: import c_getnonuis transitions = c_getnonuis.calculate_transitions_ch( ((pep['q1'], pep['mod_sequence'], p_id),), [1], 0, 20000) #get random fragment ions from random import shuffle shuffle(transitions) transitions = [ (t[0], 0) for t in transitions[:10] ] nr_transitions = len( transitions )
"qadd": qadd, } print q cursor.execute(q) print "Obtained all peptides" alltuples = list(cursor.fetchall()) progressm = progress.ProgressMeter(total=len(alltuples), unit="peptides") prepare = [] total_transitions = 0 total_assays = 0 for kk, pep in enumerate(alltuples): q3_low, q3_high = [400, 1200] q3_low, q3_high = [400, 1400] q3charges = [1, 2] q3charges = [1] # q3_low, q3_high = [0, 12000] # # new way to calculate the precursors transitions = c_getnonuis.calculate_transitions_ch(((-2, pep[0], -1),), q3charges, q3_low, q3_high) # print pep, len(pep[0]), len(transitions) total_transitions += len(transitions) total_assays += sum([collider.choose(len(transitions), i) for i in range(1, 6) if len(transitions) >= i]) progressm.update(1) print "Total number of precursors is ", len(alltuples) print "Total number of transitions is ", total_transitions print "Total number of assays is ", total_assays
print "Obtained all peptides" alltuples = list(cursor.fetchall() ) progressm = progress.ProgressMeter(total=len(alltuples), unit='peptides') prepare = [] total_transitions = 0 total_assays = 0 for kk, pep in enumerate(alltuples): q3_low, q3_high = [400, 1200] q3_low, q3_high = [400, 1400] q3charges = [1,2] q3charges = [1] #q3_low, q3_high = [0, 12000] # #new way to calculate the precursors transitions = c_getnonuis.calculate_transitions_ch( ((-2, pep[0], -1),), q3charges, q3_low, q3_high) #print pep, len(pep[0]), len(transitions) total_transitions += len(transitions) total_assays += sum([ collider.choose(len(transitions), i) for i in range(1,6) if len(transitions) >= i ] ) progressm.update(1) print "Total number of precursors is ", len(alltuples) print "Total number of transitions is ", total_transitions print "Total number of assays is ", total_assays
sys.exit() self = mycollider self.mysqlnewtime = 0 self.pepids = mypepids MAX_UIS = par.max_uis progressm = progress.ProgressMeter(total=len(self.pepids), unit='peptides') prepare = [] for kk, pep in enumerate(self.pepids): p_id = pep['parent_id'] q1 = pep['q1'] ssrcalc = pep['ssrcalc'] q3_low, q3_high = par.get_q3range_transitions() # #new way to calculate the precursors transitions = c_getnonuis.calculate_transitions_ch( ((q1, pep['sequence'], p_id), ), [1], q3_low, q3_high) #fake some srm_id for the transitions transitions = tuple([(t[0], i) for i, t in enumerate(transitions)]) nr_transitions = len(transitions) # #in SWATH mode, we have a fixed window independent of q1 if swath_mode: q1_low = min_q1 q1_high = max_q1 else: q1_low = q1 - par.q1_window q1_high = q1 + par.q1_window # if use_db and not swath_mode: precursors = self._get_all_precursors(par, pep, cursor) elif use_db and swath_mode:
def test_calculate_transitions_regular(self): trans = c_getnonuis.calculate_transitions_ch((self.pep1,), [1, 2], 300, 1500) self.assertEqual(len(trans), 10) for calc, ref in zip(trans, self.transitions_12_between300_1500): self.assertTrue(abs(calc[0] - ref) < 1e-3)
sys.exit() self = mycollider self.mysqlnewtime = 0 self.pepids = mypepids MAX_UIS = par.max_uis progressm = progress.ProgressMeter(total=len(self.pepids), unit='peptides') prepare = [] for kk, pep in enumerate(self.pepids): p_id = pep['parent_id'] q1 = pep['q1'] ssrcalc = pep['ssrcalc'] q3_low, q3_high = par.get_q3range_transitions() # #new way to calculate the precursors transitions = c_getnonuis.calculate_transitions_ch( ((q1, pep['sequence'], p_id),), [1], q3_low, q3_high) #fake some srm_id for the transitions transitions = tuple([ (t[0], i) for i,t in enumerate(transitions)]) nr_transitions = len(transitions) # #in SWATH mode, we have a fixed window independent of q1 if swath_mode: q1_low = min_q1; q1_high = max_q1 else: q1_low = q1 - par.q1_window ; q1_high = q1 + par.q1_window # if use_db and not swath_mode: precursors = self._get_all_precursors(par, pep, cursor) elif use_db and swath_mode: if par.ssrcalc_window > 1000: precursors = [p for p in allprecursors if p[2] != pep['peptide_key'] ] else: ssrcalc_low = ssrcalc - par.ssrcalc_window