def test_calculatetrans(self):
print '\nTesting calculate_transitions'
myprecursors = ((500, 'PEPTIDE', 1, 1, 0), (400, 'CEPC[160]IDM[147]E',
2, 2, 0))
st = time.time()
LARGENUMBER = 100000
for i in range(LARGENUMBER):
tr_new = c_getnonuis.calculate_transitions_ch(
myprecursors, [1, 2], self.q3_low, self.q3_high)
ctime = time.time() - st
st = time.time()
coll = collider.SRMcollider()
for i in range(LARGENUMBER):
tr_old = list(
collider.SRMcollider._get_all_collisions_calculate_sub(
coll, myprecursors, self.par, self.R, self.q3_low,
self.q3_high))
oldtime = time.time() - st
tr_new.sort(mysort)
tr_old.sort(mysort)
self.assertEqual(len(tr_new), len(tr_old))
for o, n in zip(tr_old, tr_new):
for oo, nn in zip(o, n):
self.assertTrue(oo - nn < 1e-5)
print ctime, oldtime
print "Speedup:", oldtime / ctime
def test_calculatetrans2(self):
print '\nTesting calculate_transitions 2'
pep = test_shared.runpep2
transitions = test_shared.runtransitions2
precursors = test_shared.runprecursors2
transitions = tuple([ (t[0], i) for i,t in enumerate(transitions)])
st = time.time()
collisions = list(collider.SRMcollider._get_all_collisions_calculate_sub(
collider.SRMcollider(), precursors, self.par, self.R, self.q3_low, self.q3_high))
oldtime = time.time() - st
st = time.time()
tr_new = c_getnonuis.calculate_transitions_ch(tuple(precursors), [1,2], self.q3_low, self.q3_high)
ctime = time.time() - st
tr_new.sort(mysort)
collisions.sort(mysort)
self.assertEqual(len(tr_new), len(collisions))
for o,n in zip(collisions, tr_new):
for oo,nn in zip(o,n):
self.assertTrue(oo - nn < 1e-5)
print ctime, oldtime
print "Speedup:", oldtime / ctime
def test_calculatetrans_single(self):
pep = test_shared.runpep2
transitions = test_shared.runtransitions2
precursors = [p for p in test_shared.runprecursors2 if p[2] == 106423]
transitions = tuple([(t[0], i) for i, t in enumerate(transitions)])
st = time.time()
collisions = list(
collider.SRMcollider._get_all_collisions_calculate_sub(
collider.SRMcollider(), precursors, self.par, self.R,
self.q3_low, self.q3_high))
oldtime = time.time() - st
st = time.time()
tr_new = c_getnonuis.calculate_transitions_ch(tuple(precursors),
[1, 2], self.q3_low,
self.q3_high)
ctime = time.time() - st
tr_new.sort(mysort)
collisions.sort(mysort)
self.assertEqual(len(tr_new), len(collisions))
for o, n in zip(collisions, tr_new):
for oo, nn in zip(o, n):
self.assertTrue(oo - nn < 1e-5)
def test_calculatetrans1(self):
print '\nTesting calculate_transitions 1'
pep = test_shared.runpep1
transitions = test_shared.runtransitions1
precursors = test_shared.runprecursors1[240:300]
#precursors = test_shared.runprecursors1[2],
transitions = tuple([(t[0], i) for i, t in enumerate(transitions)])
st = time.time()
collisions = list(
collider.SRMcollider._get_all_collisions_calculate_sub(
collider.SRMcollider(), precursors, self.par, self.R,
self.q3_low, self.q3_high))
oldtime = time.time() - st
st = time.time()
tr_new = c_getnonuis.calculate_transitions_ch(tuple(precursors),
[1, 2], self.q3_low,
self.q3_high)
ctime = time.time() - st
tr_new.sort(mysort)
collisions.sort(mysort)
self.assertEqual(len(tr_new), len(collisions))
for o, n in zip(collisions, tr_new):
for oo, nn in zip(o, n):
self.assertTrue(oo - nn < 1e-5)
print ctime, oldtime
print "Speedup:", oldtime / ctime
def test_calculatetrans(self):
print '\nTesting calculate_transitions'
myprecursors = ((500, 'PEPTIDE', 1, 1, 0), (400, 'CEPC[160]IDM[147]E', 2, 2, 0))
st = time.time()
LARGENUMBER = 100000
for i in range(LARGENUMBER):
tr_new = c_getnonuis.calculate_transitions_ch(myprecursors, [1,2], self.q3_low, self.q3_high)
ctime = time.time() - st
st = time.time()
coll = collider.SRMcollider()
for i in range(LARGENUMBER):
tr_old = list(collider.SRMcollider._get_all_collisions_calculate_sub(
coll, myprecursors, self.par, self.R, self.q3_low, self.q3_high))
oldtime = time.time() - st
tr_new.sort(mysort)
tr_old.sort(mysort)
self.assertEqual(len(tr_new), len(tr_old))
for o,n in zip(tr_old, tr_new):
for oo,nn in zip(o,n):
self.assertTrue(oo - nn < 1e-5)
print ctime, oldtime
print "Speedup:", oldtime / ctime
def calculate_transitions_ch(peptides, charges, q3_low, q3_high):
try:
import c_getnonuis
return c_getnonuis.calculate_transitions_ch(peptides, charges, q3_low, q3_high)
except ImportError:
return list(_calculate_transitions_ch(peptides, charges, q3_low, q3_high))
def calculate_transitions_ch(peptides, charges, q3_low, q3_high):
try:
import c_getnonuis
return c_getnonuis.calculate_transitions_ch(peptides, charges, q3_low,
q3_high)
except ImportError:
return list(
_calculate_transitions_ch(peptides, charges, q3_low, q3_high))
def calculate_transitions(self, q3_low, q3_high, charges=[1]):
import c_getnonuis
transitions = c_getnonuis.calculate_transitions_ch(
((self.q1, self.modified_sequence, self.parent_id),), charges, q3_low, q3_high
)
# fake some srm_id for the transitions, so that the returned transitions will be tuples of (q1, id)
return tuple([(t[0], i) for i, t in enumerate(transitions)])
def test_calculate_transitions_inner(self):
mypep = self.pep1
transitions = c_getnonuis.calculate_transitions_ch((mypep,), [2], 0, 5000)
transitions = [t[0] for t in transitions]
self.assertEqual(len(transitions), 12)
self.assertEqual(len(transitions), 2 * len(mypep[1]) - 2)
y_series = transitions[: len(mypep[1]) - 1]
b_series = transitions[len(mypep[1]) - 1 :]
self.assertEqual(len(y_series), 6)
self.assertEqual(len(b_series), 6)
for calc, ref in zip(y_series, self.pep1_yseries):
self.assertTrue(abs(calc - ref) < 1e-3)
for calc, ref in zip(b_series, self.pep1_bseries):
self.assertTrue(abs(calc - ref) < 1e-3)
def test_calculate_transitions_modifcation(self):
trans = c_getnonuis.calculate_transitions_ch((self.pep2,), [1, 2], 300, 1500)
# TODO check
self.assertTrue(abs(trans[0][0]) - 909.333 < 1e-3)
self.assertTrue(abs(trans[1][0]) - 780.290 < 1e-3)
self.assertTrue(abs(trans[2][0]) - 683.238 < 1e-3)
self.assertTrue(abs(trans[3][0]) - 523.207 < 1e-3)
self.assertTrue(abs(trans[4][0]) - 410.123 < 1e-3)
self.assertTrue(abs(trans[5][0]) - 330.112 < 1e-3)
self.assertTrue(abs(trans[6][0]) - 490.143 < 1e-3)
self.assertTrue(abs(trans[7][0]) - 603.227 < 1e-3)
self.assertTrue(abs(trans[8][0]) - 718.254 < 1e-3)
self.assertTrue(abs(trans[9][0]) - 865.289 < 1e-3)
self.assertTrue(abs(trans[10][0]) - 455.170 < 1e-3)
self.assertTrue(abs(trans[11][0]) - 390.649 < 1e-3)
self.assertTrue(abs(trans[12][0]) - 342.122 < 1e-3)
self.assertTrue(abs(trans[13][0]) - 302.117 < 1e-3)
self.assertTrue(abs(trans[14][0]) - 359.630 < 1e-3)
self.assertTrue(abs(trans[15][0]) - 433.148 < 1e-3)
def test_integrated_cpp(self):
""" Compare the fully integrated vs the mixed C++ rangetree / Python solution.
Here we are comparing the fully integrated solution of storing all
precursors in a C++ range tree and never passing them to Python vs
the solution where we store the precursors in the rangetree, pass
them to Python and then evaluate them.
"""
verbose = True
verbose = False
print '\n'*1
print "Comparing fully integrated solution (c_integrated.wrap_all)"
par = self.par
cursor = self.cursor
all_precursors = self.precursors_to_evaluate
shuffle(all_precursors)
all_precursors = all_precursors[:self.limit_large]
self.myprecursors.build_parent_id_lookup()
testrange = self.myprecursors.build_rangetree()
import c_rangetree
r = c_rangetree.ExtendedRangetree_Q1_RT.create()
r.new_rangetree()
r.create_tree(tuple(self.alltuples_isotope_correction))
#c_integrated.create_tree(tuple(self.alltuples_isotope_correction))
MAX_UIS = 5
newtime = 0; oldtime = 0; ctime = 0
oldsql = 0; newsql = 0
alllocaltime = 0
localprecursor = 0
transitiontime = 0
c_fromprecursortime = 0
prepare = []
self._cursor = False
print "Running experiment ", par.get_common_filename()
print "calc_trans. = time to calculate the transitions of the precursor"
print "old = use rangetree to get precursors, use C++ code to get collperpep"
print "new = use rangetree to get precursors, use single C++ code to get collperpep"
print "i\tcalc_trans.\tnew\t\told\t\t>>\tspeedup"
for kk, precursor in enumerate(all_precursors):
ii = kk + 1
q1 = precursor.q1
ssrcalc = precursor.ssrcalc
sequence = precursor.modified_sequence
peptide_key = precursor.transition_group
p_id = precursor.parent_id
q3_low, q3_high = par.get_q3range_transitions()
#new way to calculate the precursors
mystart = time.time()
transitions = c_getnonuis.calculate_transitions_ch(
((q1, sequence, p_id),), [1,2], q3_low, q3_high)
nr_transitions = len( transitions )
#fake some srm_id for the transitions
transitions = tuple([ (t[0], i) for i,t in enumerate(transitions)])
q1_low = q1 - par.q1_window
q1_high = q1 + par.q1_window
innerstart = time.time()
#correct rounding errors, s.t. we get the same results as before!
ssrcalc_low = ssrcalc - par.ssrcalc_window + 0.001
ssrcalc_high = ssrcalc + par.ssrcalc_window - 0.001
transitiontime += time.time() - mystart
isotope_correction = par.isotopes_up_to * Residues.mass_diffC13 / min(par.parent_charges)
###################################
# New way to calculate non_uislist
# - start out from transitions, get non_uislist
mystart = time.time()
newresult = c_integrated.wrap_all_bitwise(transitions, q1_low, ssrcalc_low,
q1_high, ssrcalc_high, peptide_key,
min(MAX_UIS,nr_transitions) , par.q3_window, #q3_low, q3_high,
par.ppm, par.isotopes_up_to, isotope_correction, par, r)
newtime += time.time() - mystart
###################################
# Old way to calculate non_uislist:
# - get collisions per peptide
# - get non_uislist
mystart = time.time()
collisions_per_peptide_obj = self.myprecursors.get_collisions_per_peptide_from_rangetree(
precursor, precursor.q1 - par.q1_window, precursor.q1 + par.q1_window,
transitions, par, r)
## # if False:
## # precursor_ids = tuple(c_rangetree.query_tree( q1_low, ssrcalc_low,
## # q1_high, ssrcalc_high, par.isotopes_up_to, isotope_correction))
## # precursors = tuple([parentid_lookup[myid[0]] for myid in precursor_ids
## # #dont select myself
## # if parentid_lookup[myid[0]][2] != peptide_key])
## # collisions_per_peptide = c_getnonuis.calculate_collisions_per_peptide(
## # transitions, precursors, q3_low, q3_high, par.q3_window, par.ppm)
non_uis_list = [{} for i in range(MAX_UIS+1)]
for order in range(1,MAX_UIS+1):
non_uis_list[order] = c_getnonuis.get_non_uis(
collisions_per_peptide_obj, order)
oldtime += time.time() - mystart
non_uis_list_old_way = [set(kk.keys()) for kk in non_uis_list]
non_uis_list_len = [len(kk) for kk in non_uis_list_old_way[1:]]
###################################
# Assert equality, print out result
self.assertEqual( newresult , non_uis_list_len)
mys = "%s\t%0.4fms\t%0.2fms\t\t%0.2fms\t\t>>\t%0.2f" % \
(ii, transitiontime *1000/ ii,
newtime*1000/ii, oldtime*1000/ii,
oldtime *1.0 / newtime)
#start a new line for each output?
#mys += '\t%s\t%s' % ( len(precursors), len(precursor_new) )
if False: mys += '\n'
self.ESC = chr(27)
sys.stdout.write(self.ESC + '[2K')
if self._cursor:
sys.stdout.write(self.ESC + '[u')
self._cursor = True
sys.stdout.write(self.ESC + '[s')
sys.stdout.write(mys)
sys.stdout.flush()
def calculate_transitions(self, q3_low, q3_high, charges=[1]):
import c_getnonuis
transitions = c_getnonuis.calculate_transitions_ch(
((self.q1, self.modified_sequence, self.parent_id),), charges, q3_low, q3_high)
# fake some srm_id for the transitions, so that the returned transitions will be tuples of (q1, id)
return tuple([ (t[0], i) for i,t in enumerate(transitions)])
def test_integrated_cpp(self):
""" Compare the fully integrated vs the mixed C++ rangetree / Python solution.
Here we are comparing the fully integrated solution of storing all
precursors in a C++ range tree and never passing them to Python vs
the solution where we store the precursors in the rangetree, pass
them to Python and then evaluate them.
"""
verbose = True
verbose = False
print '\n' * 1
print "Comparing fully integrated solution (c_integrated.wrap_all)"
par = self.par
cursor = self.cursor
all_precursors = self.precursors_to_evaluate
shuffle(all_precursors)
all_precursors = all_precursors[:self.limit_large]
self.myprecursors.build_parent_id_lookup()
testrange = self.myprecursors.build_rangetree()
import c_rangetree
r = c_rangetree.ExtendedRangetree_Q1_RT.create()
r.new_rangetree()
r.create_tree(tuple(self.alltuples_isotope_correction))
#c_integrated.create_tree(tuple(self.alltuples_isotope_correction))
MAX_UIS = 5
newtime = 0
oldtime = 0
ctime = 0
oldsql = 0
newsql = 0
alllocaltime = 0
localprecursor = 0
transitiontime = 0
c_fromprecursortime = 0
prepare = []
self._cursor = False
print "Running experiment ", par.get_common_filename()
print "calc_trans. = time to calculate the transitions of the precursor"
print "old = use rangetree to get precursors, use C++ code to get collperpep"
print "new = use rangetree to get precursors, use single C++ code to get collperpep"
print "i\tcalc_trans.\tnew\t\told\t\t>>\tspeedup"
for kk, precursor in enumerate(all_precursors):
ii = kk + 1
q1 = precursor.q1
ssrcalc = precursor.ssrcalc
sequence = precursor.modified_sequence
peptide_key = precursor.transition_group
p_id = precursor.parent_id
q3_low, q3_high = par.get_q3range_transitions()
#new way to calculate the precursors
mystart = time.time()
transitions = c_getnonuis.calculate_transitions_ch(
((q1, sequence, p_id), ), [1, 2], q3_low, q3_high)
nr_transitions = len(transitions)
#fake some srm_id for the transitions
transitions = tuple([(t[0], i) for i, t in enumerate(transitions)])
q1_low = q1 - par.q1_window
q1_high = q1 + par.q1_window
innerstart = time.time()
#correct rounding errors, s.t. we get the same results as before!
ssrcalc_low = ssrcalc - par.ssrcalc_window + 0.001
ssrcalc_high = ssrcalc + par.ssrcalc_window - 0.001
transitiontime += time.time() - mystart
isotope_correction = par.isotopes_up_to * Residues.mass_diffC13 / min(
par.parent_charges)
###################################
# New way to calculate non_uislist
# - start out from transitions, get non_uislist
mystart = time.time()
newresult = c_integrated.wrap_all_bitwise(
transitions,
q1_low,
ssrcalc_low,
q1_high,
ssrcalc_high,
peptide_key,
min(MAX_UIS, nr_transitions),
par.q3_window, #q3_low, q3_high,
par.ppm,
par.isotopes_up_to,
isotope_correction,
par,
r)
newtime += time.time() - mystart
###################################
# Old way to calculate non_uislist:
# - get collisions per peptide
# - get non_uislist
mystart = time.time()
collisions_per_peptide_obj = self.myprecursors.get_collisions_per_peptide_from_rangetree(
precursor, precursor.q1 - par.q1_window,
precursor.q1 + par.q1_window, transitions, par, r)
## # if False:
## # precursor_ids = tuple(c_rangetree.query_tree( q1_low, ssrcalc_low,
## # q1_high, ssrcalc_high, par.isotopes_up_to, isotope_correction))
## # precursors = tuple([parentid_lookup[myid[0]] for myid in precursor_ids
## # #dont select myself
## # if parentid_lookup[myid[0]][2] != peptide_key])
## # collisions_per_peptide = c_getnonuis.calculate_collisions_per_peptide(
## # transitions, precursors, q3_low, q3_high, par.q3_window, par.ppm)
non_uis_list = [{} for i in range(MAX_UIS + 1)]
for order in range(1, MAX_UIS + 1):
non_uis_list[order] = c_getnonuis.get_non_uis(
collisions_per_peptide_obj, order)
oldtime += time.time() - mystart
non_uis_list_old_way = [set(kk.keys()) for kk in non_uis_list]
non_uis_list_len = [len(kk) for kk in non_uis_list_old_way[1:]]
###################################
# Assert equality, print out result
self.assertEqual(newresult, non_uis_list_len)
mys = "%s\t%0.4fms\t%0.2fms\t\t%0.2fms\t\t>>\t%0.2f" % \
(ii, transitiontime *1000/ ii,
newtime*1000/ii, oldtime*1000/ii,
oldtime *1.0 / newtime)
#start a new line for each output?
#mys += '\t%s\t%s' % ( len(precursors), len(precursor_new) )
if False: mys += '\n'
self.ESC = chr(27)
sys.stdout.write(self.ESC + '[2K')
if self._cursor:
sys.stdout.write(self.ESC + '[u')
self._cursor = True
sys.stdout.write(self.ESC + '[s')
sys.stdout.write(mys)
sys.stdout.flush()
self = mycollider
self.pepids = self._get_unique_pepids_toptransitions(par, cursor)
MAX_UIS = par.max_uis
common_filename = par.get_common_filename()
self.min_transitions = []
start = time.time()
progressm = progress.ProgressMeter(total=len(self.pepids), unit='peptides')
for i, pep in enumerate(self.pepids):
p_id = pep['parent_id']
q1 = pep['q1']
if not use_randomized_transitions and not use_above_precursor:
transitions = self._get_all_transitions_toptransitions(par, pep, cursor)
elif use_above_precursor:
import c_getnonuis
transitions = c_getnonuis.calculate_transitions_ch(
((pep['q1'], pep['mod_sequence'], p_id),), [1], 0, 20000)
#get all singly charged y ions above the precursor
y_series = [(t[0], 0) for t in transitions[:len(transitions)/2] if t[0] > q1]
y_series.reverse()
y_series = y_series[:5]
transitions = y_series
elif use_randomized_transitions:
import c_getnonuis
transitions = c_getnonuis.calculate_transitions_ch(
((pep['q1'], pep['mod_sequence'], p_id),), [1], 0, 20000)
#get random fragment ions
from random import shuffle
shuffle(transitions)
transitions = [ (t[0], 0) for t in transitions[:10] ]
nr_transitions = len( transitions )
"qadd": qadd,
}
print q
cursor.execute(q)
print "Obtained all peptides"
alltuples = list(cursor.fetchall())
progressm = progress.ProgressMeter(total=len(alltuples), unit="peptides")
prepare = []
total_transitions = 0
total_assays = 0
for kk, pep in enumerate(alltuples):
q3_low, q3_high = [400, 1200]
q3_low, q3_high = [400, 1400]
q3charges = [1, 2]
q3charges = [1]
# q3_low, q3_high = [0, 12000]
#
# new way to calculate the precursors
transitions = c_getnonuis.calculate_transitions_ch(((-2, pep[0], -1),), q3charges, q3_low, q3_high)
# print pep, len(pep[0]), len(transitions)
total_transitions += len(transitions)
total_assays += sum([collider.choose(len(transitions), i) for i in range(1, 6) if len(transitions) >= i])
progressm.update(1)
print "Total number of precursors is ", len(alltuples)
print "Total number of transitions is ", total_transitions
print "Total number of assays is ", total_assays
print "Obtained all peptides"
alltuples = list(cursor.fetchall() )
progressm = progress.ProgressMeter(total=len(alltuples), unit='peptides')
prepare = []
total_transitions = 0
total_assays = 0
for kk, pep in enumerate(alltuples):
q3_low, q3_high = [400, 1200]
q3_low, q3_high = [400, 1400]
q3charges = [1,2]
q3charges = [1]
#q3_low, q3_high = [0, 12000]
#
#new way to calculate the precursors
transitions = c_getnonuis.calculate_transitions_ch(
((-2, pep[0], -1),), q3charges, q3_low, q3_high)
#print pep, len(pep[0]), len(transitions)
total_transitions += len(transitions)
total_assays += sum([ collider.choose(len(transitions), i) for i in range(1,6)
if len(transitions) >= i ] )
progressm.update(1)
print "Total number of precursors is ", len(alltuples)
print "Total number of transitions is ", total_transitions
print "Total number of assays is ", total_assays
sys.exit()
self = mycollider
self.mysqlnewtime = 0
self.pepids = mypepids
MAX_UIS = par.max_uis
progressm = progress.ProgressMeter(total=len(self.pepids), unit='peptides')
prepare = []
for kk, pep in enumerate(self.pepids):
p_id = pep['parent_id']
q1 = pep['q1']
ssrcalc = pep['ssrcalc']
q3_low, q3_high = par.get_q3range_transitions()
#
#new way to calculate the precursors
transitions = c_getnonuis.calculate_transitions_ch(
((q1, pep['sequence'], p_id), ), [1], q3_low, q3_high)
#fake some srm_id for the transitions
transitions = tuple([(t[0], i) for i, t in enumerate(transitions)])
nr_transitions = len(transitions)
#
#in SWATH mode, we have a fixed window independent of q1
if swath_mode:
q1_low = min_q1
q1_high = max_q1
else:
q1_low = q1 - par.q1_window
q1_high = q1 + par.q1_window
#
if use_db and not swath_mode:
precursors = self._get_all_precursors(par, pep, cursor)
elif use_db and swath_mode:
def test_calculate_transitions_regular(self):
trans = c_getnonuis.calculate_transitions_ch((self.pep1,), [1, 2], 300, 1500)
self.assertEqual(len(trans), 10)
for calc, ref in zip(trans, self.transitions_12_between300_1500):
self.assertTrue(abs(calc[0] - ref) < 1e-3)
sys.exit()
self = mycollider
self.mysqlnewtime = 0
self.pepids = mypepids
MAX_UIS = par.max_uis
progressm = progress.ProgressMeter(total=len(self.pepids), unit='peptides')
prepare = []
for kk, pep in enumerate(self.pepids):
p_id = pep['parent_id']
q1 = pep['q1']
ssrcalc = pep['ssrcalc']
q3_low, q3_high = par.get_q3range_transitions()
#
#new way to calculate the precursors
transitions = c_getnonuis.calculate_transitions_ch(
((q1, pep['sequence'], p_id),), [1], q3_low, q3_high)
#fake some srm_id for the transitions
transitions = tuple([ (t[0], i) for i,t in enumerate(transitions)])
nr_transitions = len(transitions)
#
#in SWATH mode, we have a fixed window independent of q1
if swath_mode: q1_low = min_q1; q1_high = max_q1
else: q1_low = q1 - par.q1_window ; q1_high = q1 + par.q1_window
#
if use_db and not swath_mode:
precursors = self._get_all_precursors(par, pep, cursor)
elif use_db and swath_mode:
if par.ssrcalc_window > 1000:
precursors = [p for p in allprecursors if p[2] != pep['peptide_key'] ]
else:
ssrcalc_low = ssrcalc - par.ssrcalc_window