def test_main(self, peak, im_i):
area_sum, area_dict = peak_sum_area(im_i,
peak,
single_ion=True,
max_bound=5)
assert isinstance(area_sum, float)
assert isinstance(area_dict, dict)
assert area_sum == 10025814.0
assert area_dict[51] == 3299.0
assert isinstance(area_dict[51], float)
area_sum = peak_sum_area(im_i, peak, single_ion=False, max_bound=5)
assert area_sum == 10025814.0
def Peak_detector(pp_im):
"""
Peak detection and filtering and selection
:param pp_im:
:return:
"""
peakz = []
UID_list = []
counter = 1
for im in list(pp_im):
poss_peaks = BillerBiemann(im, points=9, scans=2) #increase scan #
pi = rel_threshold(poss_peaks, percent=2)
nin = num_ions_threshold(pi, n=5, cutoff=10000)
for peak in nin:
area = peak_sum_area(im, peak)
peak.set_area(area)
peakz.append(nin)
print("...", counter)
counter += 1
for pkz in peakz:pi
print("Peaks detected: ", len(pkz))
uid = pkz.get_UID()
UID_list.append(uid)
def expr_list(pyms_datadir):
with tempfile.TemporaryDirectory() as tmpdir:
outputdir = pathlib.Path(tmpdir)
# Create experiment files
for jcamp_file in eley_codes:
im = build_intensity_matrix_i(
JCAMP_reader(pyms_datadir / f"{jcamp_file}.JDX"))
# Intensity matrix size (scans, masses)
n_scan, n_mz = im.size
# noise filter and baseline correct
for ii in range(n_mz):
ic = im.get_ic_at_index(ii)
ic_smooth = savitzky_golay(ic)
ic_bc = tophat(ic_smooth, struct="1.5m")
im.set_ic_at_index(ii, ic_bc)
peak_list = BillerBiemann(im, points=9, scans=2)
print('#')
apl = rel_threshold(peak_list, 2)
new_peak_list = num_ions_threshold(apl, 3, 3000)
print('#')
# ignore TMS ions and set mass range
for peak in new_peak_list:
peak.crop_mass(50, 400)
peak.null_mass(73)
peak.null_mass(147)
# find area
area = peak_sum_area(im, peak)
peak.area = area
area_dict = peak_top_ion_areas(im, peak)
peak.ion_areas = area_dict
expr = Experiment(jcamp_file, new_peak_list)
# set time range for all experiments
expr.sele_rt_range(["6.5m", "21m"])
print('#')
expr.dump(outputdir / f"{jcamp_file}.expr")
print('#')
# Load experiments
expr_list = []
for expr_code in eley_codes:
expr = load_expr(outputdir / f"{expr_code}.expr")
assert isinstance(expr, Experiment)
expr_list.append(expr)
yield expr_list
def call_peaks(im, tic, smooth, args):
print "calling peaks"
if smooth:
print "Smoothing IM first..."
im.crop_mass(args.lowmass, args.highmass)
print "cropped masses..."
# get the size of the intensity matrix
n_scan, n_mz = im.get_size()
print "# masses in intensity matrix: ", n_mz
# smooth data
for ii in range(n_mz):
ic = im.get_ic_at_index(ii)
#print "got ic for mass ", ii
# ic1 = savitzky_golay(ic)
ic_smooth = savitzky_golay(ic, window=args.window,
degree=4) #JT: changed to 4 from 2
#print "savitky golay ran "
ic_base = tophat(ic_smooth, struct="1.5m")
#print "tophat ran "
im.set_ic_at_index(ii, ic_base)
#print "smoothed mass ", ii
print "smoothed IM..."
# noise level calc
tic1 = savitzky_golay(tic)
tic2 = tophat(tic1, struct="1.5m") #JT: How does struct size work?
noise_level = window_analyzer(tic2)
print "Noise level in TIC: ", noise_level
# get the list of Peak objects using BB peak detection / deconv
pl = BillerBiemann(im, args.window, args.scans)
print "Initial number of Peaks found:", len(pl)
# filter down the peaks.
# - First: remove any masses from each peak that have intensity less than r percent of the max intensity in that peak
# - Second: remove any peak where there are less than n ions with intensity above the cutoff
pl2 = rel_threshold(pl, percent=args.minintensity)
pl3 = num_ions_threshold(
pl2, n=args.minions, cutoff=100000
) #100000 for pegBT #200 for peg3 #minions maybe 3 instead of 4?
#JT: Was getting very different noise cutoff values so just made it 10^5
# Which was decided on by looking at chromatograms to find baseline noise lvl
print "Peaks remaining after filtering:", len(pl3)
for peak in pl3:
#peak.null_mass(73)
#peak.null_mass(207) # column bleed
#peak.null_mass(84) # solvent tailing
area = peak_sum_area(im, peak) # get the TIC area for this peak
peak.set_area(area)
area_dict = peak_top_ion_areas(
im, peak, args.topions) # get top n ion areas for this peak
peak.set_ion_areas(area_dict)
return pl3
def call_peaks(im, tic, smooth, args):
print "calling peaks"
if smooth:
print "Smoothing IM first..."
im.crop_mass(args.lowmass, args.highmass)
print "cropped masses..."
# get the size of the intensity matrix
n_scan, n_mz = im.get_size()
print "# masses in intensity matrix: ", n_mz
# smooth data
for ii in range(n_mz):
ic = im.get_ic_at_index(ii)
#print "got ic for mass ", ii
# ic1 = savitzky_golay(ic)
ic_smooth = savitzky_golay(ic, window=args.window, degree=2)
#print "savitky golay ran "
ic_base = tophat(ic_smooth, struct="1.5m")
#print "tophat ran "
im.set_ic_at_index(ii, ic_base)
#print "smoothed mass ", ii
print "smoothed IM..."
# noise level calc
tic1 = savitzky_golay(tic)
tic2 = tophat(tic1, struct="1.5m")
noise_level = window_analyzer(tic2)
print "Noise level in TIC: ", noise_level
# get the list of Peak objects using BB peak detection / deconv
pl = BillerBiemann(im, args.window, args.scans)
print "Initial number of Peaks found:", len(pl)
# filter down the peaks.
# - First: remove any masses from each peak that have intensity less than r percent of the max intensity in that peak
# - Second: remove any peak where there are less than n ions with intensity above the cutoff
pl2 = rel_threshold(pl, percent=args.minintensity)
pl3 = num_ions_threshold(pl2, n=args.minions, cutoff=noise_level * args.noisemult)
print "Peaks remaining after filtering:", len(pl3)
for peak in pl3:
# peak.null_mass(73)
peak.null_mass(207) # column bleed
peak.null_mass(84) # solvent tailing
area = peak_sum_area(im, peak) # get the TIC area for this peak
peak.set_area(area)
area_dict = peak_top_ion_areas(im, peak, args.topions) # get top n ion areas for this peak
peak.set_ion_areas(area_dict)
return pl3
def test_area(im_i, peak):
peak = copy.deepcopy(peak)
# determine and set area
area = peak_sum_area(im_i, peak)
assert isinstance(area, float)
peak.area = area
assert peak.area == area
assert isinstance(peak.area, float)
scan_i = im_i.get_index_at_time(31.17 * 60.0)
ms = im_i.get_ms_at_index(scan_i)
for obj in [test_string, test_dict, test_list_strs, test_list_ints]:
with pytest.raises(TypeError):
Peak(test_float, ms).area = obj
with pytest.raises(ValueError):
Peak(test_float, ms).area = -1
def _filtered_peak_list(im_i, _peak_list):
peak_list = copy.deepcopy(_peak_list)
# do peak detection on pre-trimmed data
# trim by relative intensity
apl = rel_threshold(peak_list, 2, copy_peaks=False)
# trim by threshold
new_peak_list = num_ions_threshold(apl, 3, 3000, copy_peaks=False)
# ignore TMS ions and set mass range
for peak in new_peak_list:
peak.crop_mass(50, 400)
peak.null_mass(73)
peak.null_mass(147)
# find area
area = peak_sum_area(im_i, peak)
peak.area = area
area_dict = peak_top_ion_areas(im_i, peak)
peak.ion_areas = area_dict
return new_peak_list
def Peak_detector(pp_im):
# Peak detection and filtering and selection
peakz = []
counter = 1
for im in list(pp_im):
poss_peaks = BillerBiemann(im, points=9, scans=2)
pi = rel_threshold(poss_peaks, percent=2)
nin = num_ions_threshold(pi, n=5, cutoff=10000)
for peak in nin:
area = peak_sum_area(im, peak)
peak.set_area(area)
peakz.append(nin)
print("...", counter)
counter += 1
for pkz in peakz:
print("Peaks detected: ", len(pkz))
return (peakz)
def test_peak_errors(self, im_i, obj):
with pytest.raises(TypeError):
peak_sum_area(im_i, obj)
def test_im_errors(self, peak, obj):
with pytest.raises(TypeError):
peak_sum_area(obj, peak)
def Peak_detector(pp_im, noise, name, points, scans, percent, ni, name_tag,
sdir):
# Peak detection and filtering and selection
peakz = []
# counter = 1
savePath = sdir
ms_data_files = []
print("len pp_im", len(list(pp_im)))
print("len noise", len(noise))
print("len name", len(name), name)
for im, n, na in itertools.izip(list(pp_im), noise, name):
ms_data = []
# print(na)
poss_peaks = BillerBiemann(im, points=points,
scans=scans) # increase scan #
pi = rel_threshold(poss_peaks, percent=percent)
nin = num_ions_threshold(pi, n=ni, cutoff=n)
completeName = os.path.join(savePath, na + name_tag + "ms_data.csv")
with open(completeName, 'w') as f:
w = csv.writer(f)
# head = [35.0, 36.0, 37.0, 38.0, 39.0, 40.0, 41.0, 42.0, 43.0, 44.0, 45.0, 46.0, 47.0, 48.0, 49.0, 50.0, 51.0, 52.0, 53.0, 54.0, 55.0, 56.0, 57.0, 58.0, 59.0, 60.0, 61.0, 62.0, 63.0, 64.0, 65.0, 66.0, 67.0, 68.0, 69.0, 70.0, 71.0, 72.0, 73.0, 74.0, 75.0, 76.0, 77.0, 78.0, 79.0, 80.0, 81.0, 82.0, 83.0, 84.0, 85.0, 86.0, 87.0, 88.0, 89.0, 90.0, 91.0, 92.0, 93.0, 94.0, 95.0, 96.0, 97.0, 98.0, 99.0, 100.0, 101.0, 102.0, 103.0, 104.0, 105.0, 106.0, 107.0, 108.0, 109.0, 110.0, 111.0, 112.0, 113.0, 114.0, 115.0, 116.0, 117.0, 118.0, 119.0, 120.0, 121.0, 122.0, 123.0, 124.0, 125.0, 126.0, 127.0, 128.0, 129.0, 130.0, 131.0, 132.0, 133.0, 134.0, 135.0, 136.0, 137.0, 138.0, 139.0, 140.0, 141.0, 142.0, 143.0, 144.0, 145.0, 146.0, 147.0, 148.0, 149.0, 150.0, 151.0, 152.0, 153.0, 154.0, 155.0, 156.0, 157.0, 158.0, 159.0, 160.0, 161.0, 162.0, 163.0, 164.0, 165.0, 166.0, 167.0, 168.0, 169.0, 170.0, 171.0, 172.0, 173.0, 174.0, 175.0, 176.0, 177.0, 178.0, 179.0, 180.0, 181.0, 182.0, 183.0, 184.0, 185.0, 186.0, 187.0, 188.0, 189.0, 190.0, 191.0, 192.0, 193.0, 194.0, 195.0, 196.0, 197.0, 198.0, 199.0, 200.0, 201.0, 202.0, 203.0, 204.0, 205.0, 206.0, 207.0, 208.0, 209.0, 210.0, 211.0, 212.0, 213.0, 214.0, 215.0, 216.0, 217.0, 218.0, 219.0, 220.0]
head = [
'Area', 'RTs', 35.0, 36.0, 37.0, 38.0, 39.0, 40.0, 41.0, 42.0,
43.0, 44.0, 45.0, 46.0, 47.0, 48.0, 49.0, 50.0, 51.0, 52.0,
53.0, 54.0, 55.0, 56.0, 57.0, 58.0, 59.0, 60.0, 61.0, 62.0,
63.0, 64.0, 65.0, 66.0, 67.0, 68.0, 69.0, 70.0, 71.0, 72.0,
73.0, 74.0, 75.0, 76.0, 77.0, 78.0, 79.0, 80.0, 81.0, 82.0,
83.0, 84.0, 85.0, 86.0, 87.0, 88.0, 89.0, 90.0, 91.0, 92.0,
93.0, 94.0, 95.0, 96.0, 97.0, 98.0, 99.0, 100.0, 101.0, 102.0,
103.0, 104.0, 105.0, 106.0, 107.0, 108.0, 109.0, 110.0, 111.0,
112.0, 113.0, 114.0, 115.0, 116.0, 117.0, 118.0, 119.0, 120.0,
121.0, 122.0, 123.0, 124.0, 125.0, 126.0, 127.0, 128.0, 129.0,
130.0, 131.0, 132.0, 133.0, 134.0, 135.0, 136.0, 137.0, 138.0,
139.0, 140.0, 141.0, 142.0, 143.0, 144.0, 145.0, 146.0, 147.0,
148.0, 149.0, 150.0, 151.0, 152.0, 153.0, 154.0, 155.0, 156.0,
157.0, 158.0, 159.0, 160.0, 161.0, 162.0, 163.0, 164.0, 165.0,
166.0, 167.0, 168.0, 169.0, 170.0, 171.0, 172.0, 173.0, 174.0,
175.0, 176.0, 177.0, 178.0, 179.0, 180.0, 181.0, 182.0, 183.0,
184.0, 185.0, 186.0, 187.0, 188.0, 189.0, 190.0, 191.0, 192.0,
193.0, 194.0, 195.0, 196.0, 197.0, 198.0, 199.0, 200.0, 201.0,
202.0, 203.0, 204.0, 205.0, 206.0, 207.0, 208.0, 209.0, 210.0,
211.0, 212.0, 213.0, 214.0, 215.0, 216.0, 217.0, 218.0, 219.0,
220.0
]
w.writerow(head)
for peak in nin:
area = peak_sum_area(im, peak)
# print('area:', area)
peak.set_area(area)
ms = peak.get_mass_spectrum()
# print("Peaks rt: ", peak.get_rt())
# print("Peaks ms_list: ", ms.mass_list)
# print("Peaks ms_spec: ", list(ms.mass_spec))
p_rt = peak.get_rt()
its = []
items = list(ms.mass_spec)
for i in items:
x = float(i)
its.append(x)
ms_d = ([area] + [p_rt] + its)
# ms_d = its
# print('ms_d', ms_d)
w.writerow(ms_d)
f.close()
peakz.append(nin)
# #print("...", counter)
# counter += 1
ms_data_files.append(completeName)
print('ms_data_files:', ms_data_files)
return [peakz, ms_data_files]
def Peak_detector(pp_im, noise, name):
# Peak detection and filtering and selection
peakz = []
counter = 1
savePath = '/home/juicebox/utils/easyGC/MS_peak_data'
for im, n, na in itertools.izip(list(pp_im), noise, name):
ms_data = []
#print(na)
poss_peaks = BillerBiemann(im, points=140, scans=20) #increase scan #
pi = rel_threshold(poss_peaks, percent=2)
nin = num_ions_threshold(pi, n=3, cutoff=n)
completeName = os.path.join(savePath, na + "2y.csv")
with open(completeName, 'w') as f:
w = csv.writer(f)
head = [
'RTs', 35.0, 36.0, 37.0, 38.0, 39.0, 40.0, 41.0, 42.0, 43.0,
44.0, 45.0, 46.0, 47.0, 48.0, 49.0, 50.0, 51.0, 52.0, 53.0,
54.0, 55.0, 56.0, 57.0, 58.0, 59.0, 60.0, 61.0, 62.0, 63.0,
64.0, 65.0, 66.0, 67.0, 68.0, 69.0, 70.0, 71.0, 72.0, 73.0,
74.0, 75.0, 76.0, 77.0, 78.0, 79.0, 80.0, 81.0, 82.0, 83.0,
84.0, 85.0, 86.0, 87.0, 88.0, 89.0, 90.0, 91.0, 92.0, 93.0,
94.0, 95.0, 96.0, 97.0, 98.0, 99.0, 100.0, 101.0, 102.0, 103.0,
104.0, 105.0, 106.0, 107.0, 108.0, 109.0, 110.0, 111.0, 112.0,
113.0, 114.0, 115.0, 116.0, 117.0, 118.0, 119.0, 120.0, 121.0,
122.0, 123.0, 124.0, 125.0, 126.0, 127.0, 128.0, 129.0, 130.0,
131.0, 132.0, 133.0, 134.0, 135.0, 136.0, 137.0, 138.0, 139.0,
140.0, 141.0, 142.0, 143.0, 144.0, 145.0, 146.0, 147.0, 148.0,
149.0, 150.0, 151.0, 152.0, 153.0, 154.0, 155.0, 156.0, 157.0,
158.0, 159.0, 160.0, 161.0, 162.0, 163.0, 164.0, 165.0, 166.0,
167.0, 168.0, 169.0, 170.0, 171.0, 172.0, 173.0, 174.0, 175.0,
176.0, 177.0, 178.0, 179.0, 180.0, 181.0, 182.0, 183.0, 184.0,
185.0, 186.0, 187.0, 188.0, 189.0, 190.0, 191.0, 192.0, 193.0,
194.0, 195.0, 196.0, 197.0, 198.0, 199.0, 200.0, 201.0, 202.0,
203.0, 204.0, 205.0, 206.0, 207.0, 208.0, 209.0, 210.0, 211.0,
212.0, 213.0, 214.0, 215.0, 216.0, 217.0, 218.0, 219.0, 220.0
]
w.writerow(head)
for peak in nin:
area = peak_sum_area(im, peak)
print('area=', area)
peak.set_area(area)
ms = peak.get_mass_spectrum()
#print("Peaks rt: ", peak.get_rt())
#print("Peaks ms_list: ", ms.mass_list)
print("Peaks ms_spec: ", list(ms.mass_spec))
p_rt = peak.get_rt()
its = []
items = list(ms.mass_spec)
for i in items:
x = float(i)
its.append(x)
ms_d = ([p_rt] + its)
print(ms_d)
# c = str(ms_d).split(',')
#f.write(str(ms_d))
w.writerow(ms_d)
f.close()
#
#
# #print(peak.get_rt(), items)
# # ms_d = ([peak.get_rt()] + its)
# # print(ms_d)
# # w = csv.writer(f)
# # w.writerow(x for x in list(ms_d))
#
# # w = csv.writer(f, delimiter=',')
# # w.writerows(list[p_rt + items])
# # ms_data.append((peak.get_rt(), list(ms.mass_spec)))
# # completeName = os.path.join(savePath, na+"2b.csv")
# # f = open(completeName, "w+")
# # for i in ms_data:
# # f.write("%s" % str(i))
# # f.close()
# # with open(completeName, 'w') as f:
# # f.write(str([peak.get_rt()] + items) + '\n')
# # f.write(str([peak.get_rt()] + items) + '\n')
# # f.write(str(peak.get_rt()) + str(items).replace('[', '').replace(']', '') + '\n')
# # x = str(peak.get_rt()) + str(items).replace('[', '').replace(']', '')
# # y = x.split(',')
# # print (str(y))
# # f.write(str(y) + '\n')
peakz.append(nin)
#print("...", counter)
counter += 1
#for pkz in peakz:
# print("Peaks detected: ", len(pkz))
#print("Peaks rt: ", pkz.get_rt())
#print("Peaks ms: ", pkz.get_mass_spectrum())
return peakz
# minimum number of ions, n
n = 3
# greater than or equal to threshold, t
t = 10000
# trim by relative intensity
pl = rel_threshold(peak_list, r)
# trim by threshold
real_peak_list = num_ions_threshold(pl, n, t)
print "Number of filtered peaks in real data: ", len(real_peak_list)
# Set the peak areas
for peak in real_peak_list:
area = peak_sum_area(real_im, peak)
peak.set_area(area)
# real_peak_list is PyMS' best guess at the true peak list
################## Run Simulator ######################
# Simulator takes a peak list, time_list and mass_list
# and returns an IntensityMatrix object.
# The mass_list and time_list are the same for the real
# data and the simulated data.
time_list = real_im.get_time_list()
mass_list = real_im.get_mass_list()
sim_im = gcms_sim(time_list, mass_list, real_peak_list)
def run(self):
print("Quantitative Processing in Progress...")
# TODO: Include data etc. in experiment file
self.update_pbar()
if self.filetype == ID_Format_jcamp:
# Load data using JCAMP_reader
from pyms.GCMS.IO.JCAMP import JCAMP_reader
data = JCAMP_reader(self.properties["Original Filename"])
elif self.filetype == ID_Format_mzML:
# Load data using JCAMP_reader
from pyms.GCMS.IO.MZML import MZML_reader
data = MZML_reader(self.properties["Original Filename"])
elif self.filetype == ID_Format_ANDI:
# Load data using JCAMP_reader
from pyms.GCMS.IO.ANDI import ANDI_reader
data = ANDI_reader(self.properties["Original Filename"])
else:
# Unknown Format
return
# TODO: Waters RAW, Thermo RAW, Agilent .d
self.update_pbar()
method = Method.Method(self.properties["Method"])
self.update_pbar()
# list of all retention times, in seconds
times = data.get_time_list()
# get Total Ion Chromatogram
tic = data.get_tic()
# RT Range, time step, no. scans, min, max, mean and median m/z
data.info()
# Build "intensity matrix" by binning data with integer bins and a
# window of -0.3 to +0.7, the same as NIST uses
im = build_intensity_matrix_i(data)
self.update_pbar()
# Show the m/z of the maximum and minimum bins
print(" Minimum m/z bin: {}".format(im.get_min_mass()))
print(" Maximum m/z bin: {}".format(im.get_max_mass()))
# Crop masses
min_mass, max_mass, *_ = method.mass_range
if min_mass < im.get_min_mass():
min_mass = im.get_min_mass()
if max_mass > im.get_max_mass():
max_mass = im.get_max_mass()
im.crop_mass(min_mass, max_mass)
self.update_pbar()
# Perform Data filtering
n_scan, n_mz = im.get_size()
# Iterate over each IC in the intensity matrix
for ii in range(n_mz):
# print("\rWorking on IC#", ii+1, ' ',end='')
ic = im.get_ic_at_index(ii)
if method.enable_sav_gol:
# Perform Savitzky-Golay smoothing.
# Note that Turbomass does not use smoothing for qualitative method.
ic = savitzky_golay(ic)
if method.enable_tophat:
# Perform Tophat baseline correction
# Top-hat baseline Correction seems to bring down noise,
# retaining shapes, but keeps points on actual peaks
ic = tophat(ic, struct=method.tophat_struct)
# Set the IC in the intensity matrix to the filtered one
im.set_ic_at_index(ii, ic)
self.update_pbar()
# Peak Detection based on Biller and Biemann (1974), with a window
# of <points>, and combining <scans> if they apex next to each other
peak_list = BillerBiemann(im,
points=method.bb_points,
scans=method.bb_scans)
self.update_pbar()
print(" Number of peaks identified before filtering: {}".format(
len(peak_list)))
if method.enable_noise_filter:
# Filtering peak lists with automatic noise filtering
noise_level = window_analyzer(tic)
# should we also do rel_threshold() here?
# https://pymassspec.readthedocs.io/en/master/pyms/BillerBiemann.html#pyms.BillerBiemann.rel_threshold
peak_list = num_ions_threshold(peak_list, method.noise_thresh,
noise_level)
self.update_pbar()
filtered_peak_list = []
for peak in peak_list:
# Get mass and intensity lists for the mass spectrum at the apex of the peak
apex_mass_list = peak.mass_spectrum.mass_list
apex_mass_spec = peak.mass_spectrum.mass_spec
# Determine the intensity of the base peak in the mass spectrum
base_peak_intensity = max(apex_mass_spec)
# Determine the index of the base peak in the mass spectrum
base_peak_index = [
index for index, intensity in enumerate(apex_mass_spec)
if intensity == base_peak_intensity
][0]
# Finally, determine the mass of the base peak
base_peak_mass = apex_mass_list[base_peak_index]
# skip the peak if the base peak is at e.g. m/z 73, i.e. septum bleed
if base_peak_mass in method.base_peak_filter:
continue
area = peak_sum_area(im, peak)
peak.set_area(area)
filtered_peak_list.append(peak)
self.update_pbar()
print(" Number of peaks identified: {}".format(
len(filtered_peak_list)))
# Create an experiment
self.expr = Experiment(self.sample_name, filtered_peak_list)
self.expr.sele_rt_range([
"{}m".format(method.target_range[0]),
"{}m".format(method.target_range[1])
])
self.update_pbar()
current_time = time_now()
# The date and time the experiment was created
self.properties["Date Created"] = current_time
# The date and time the experiment was last modified
self.properties["Date Modified"] = current_time
if self.pbar:
self.pbar.Update(self.pbar.Range)
self.tic = tic
self.filtered_peak_list = filtered_peak_list
def import_processing(jcamp_file, spectrum_csv_file, report_csv_file, combined_csv_file, bb_points = 9, bb_scans = 2, noise_thresh = 2, target_range = (0,120), tophat_struct="1.5m", nistpath = "../MSSEARCH", base_peak_filter = ['73'], ExprDir = "."):
global nist_path
nist_path = nistpath
# Parameters
base_peak_filter = [int(x) for x in base_peak_filter]
target_range = tuple(target_range)
sample_name = os.path.splitext(os.path.basename(jcamp_file))[0]
number_of_peaks = 80
data = JCAMP_reader(jcamp_file)
# list of all retention times, in seconds
times = data.get_time_list()
# get Total Ion Chromatogram
tic = data.get_tic()
# RT Range, time step, no. scans, min, max, mean and median m/z
data.info()
#data.write("output/data") # save output
# Mass Binning
im = build_intensity_matrix_i(data) # covnert to intensity matrix
#im.get_size() #number of scans, number of bins
masses = im.get_mass_list() # list of mass bins
print(" Minimum m/z bin: {}".format(im.get_min_mass()))
print(" Maximum m/z bin: {}".format(im.get_max_mass()))
# Write Binned Mass Spectra to OpenChrom-like CSV file
ms = im.get_ms_at_index(0) # first mass spectrum
spectrum_csv = open(spectrum_csv_file, 'w')
spectrum_csv.write('RT(milliseconds);RT(minutes) - NOT USED BY IMPORT;RI;')
spectrum_csv.write(';'.join(str(mz) for mz in ms.mass_list))
spectrum_csv.write("\n")
for scan in range(len(times)):
spectrum_csv.write("{};{};{};".format(int(times[scan]*1000),rounders((times[scan]/60),"0.0000000000"),0))
ms = im.get_ms_at_index(scan)
spectrum_csv.write(';'.join(str(intensity) for intensity in ms.mass_spec))
spectrum_csv.write('\n')
spectrum_csv.close()
## Data filtering
# Note that Turbomass does not use smoothing for qualitative method.
# Top-hat baseline Correction seems to bring down noise,
# retaning shapes, but keeps points on actual peaks
#dump_object(im, "output/im.dump") # un-processed output
n_scan, n_mz = im.get_size()
for ii in range(n_mz):
#print("\rWorking on IC#", ii+1, ' ',end='')
ic = im.get_ic_at_index(ii)
ic_smooth = savitzky_golay(ic)
ic_bc = tophat(ic_smooth, struct=tophat_struct)
im.set_ic_at_index(ii, ic_bc)
#dump_object(im, "output/im-proc.dump") # processed output
# Peak Detection based on Biller and Biemann, 1974, with a window
# of n points, and combining y scans if they apex next to each other
peak_list = BillerBiemann(im, points=bb_points, scans=bb_scans)
print(" Number of peaks identified before filtering: {}".format(len(peak_list)))
# Filtering peak lists with automatic noise filtering
noise_level = window_analyzer(tic)
peak_list = num_ions_threshold(peak_list, noise_thresh, noise_level)
# why use 2 for number of ions above threshold?
print(" Number of peaks identified: {}".format(len(peak_list)))
# Peak Areas
peak_area_list = []
filtered_peak_list = []
for peak in peak_list:
apex_mass_list = peak.get_mass_spectrum().mass_list
apex_mass_spec = peak.get_mass_spectrum().mass_spec
base_peak_intensity = max(apex_mass_spec)
base_peak_index = [index for index, intensity in enumerate(apex_mass_spec) if intensity == base_peak_intensity][0]
base_peak_mass = apex_mass_list[base_peak_index]
#print(base_peak_mass)
if base_peak_mass in base_peak_filter:
continue # skip the peak if the base peak is at e.g. m/z 73, i.e. septum bleed
area = peak_sum_area(im, peak)
peak.set_area(area)
peak_area_list.append(area)
filtered_peak_list.append(peak)
# Save the TIC and Peak List
tic.write(os.path.join(ExprDir,"{}_tic.dat".format(sample_name)),formatting=False)
store_peaks(filtered_peak_list,os.path.join(ExprDir,"{}_peaks.dat".format(sample_name)))
# from https://stackoverflow.com/questions/16878715/how-to-find-the-index-of-n-largest-elements-in-a-list-or-np-array-python?lq=1
top_peaks = sorted(range(len(peak_area_list)), key=lambda x: peak_area_list[x])
# Write to turbomass-like CSV file
report_csv = open(report_csv_file, "w")
# Write to GunShotMatch Combine-like CSV file
combine_csv = open(combined_csv_file, "w")
combine_csv.write(sample_name)
combine_csv.write("\n")
report_csv.write("#;RT;Scan;Height;Area\n")
combine_csv.write("Retention Time;Peak Area;;Lib;Match;R Match;Name;CAS Number;Scan\n")
report_buffer = []
for index in top_peaks:
# Peak Number (1-80)
peak_number = top_peaks.index(index)+1
# Retention time (minutes, 3dp)
RT = rounders(filtered_peak_list[index].get_rt()/60,"0.000")
if not target_range[0] < RT <= target_range[1]:
continue # skip the peak if it is outside the desired range
# scan number, not that we really nead it as the peak object has the spectrum
Scan = data.get_index_at_time(filtered_peak_list[index].get_rt())+1
# the binned mass spectrum
filtered_peak_list[index].get_mass_spectrum()
# TIC intensity, as proxy for Peak height, which should be from baseline
Height = '{:,}'.format(rounders(tic.get_intensity_at_index(data.get_index_at_time(filtered_peak_list[index].get_rt())),"0"))
# Peak area, originally in "intensity seconds", so dividing by 60 to
# get "intensity minutes" like turbomass uses
Area = '{:,}'.format(rounders(filtered_peak_list[index].get_area()/60,"0.0"))
#report_csv.write("{};{};{};{};{};{}\n".format(peak_number, RT, Scan, Height, Area,bounds))
report_buffer.append([peak_number, RT, Scan, Height, Area])
report_buffer = report_buffer[::-1] # Reverse list order
# List of peaks already added to report
existing_peaks = []
filtered_report_buffer = []
for row in report_buffer:
filtered_report_buffer.append(row)
filtered_report_buffer = filtered_report_buffer[:number_of_peaks]
filtered_report_buffer.sort(key=operator.itemgetter(2))
for row in filtered_report_buffer:
index = filtered_report_buffer.index(row)
report_csv.write(";".join([str(i) for i in row]))
ms = im.get_ms_at_index(row[2]-1)
create_msp("{}_{}".format(sample_name,row[1]),ms.mass_list, ms.mass_spec)
matches_dict = nist_ms_comparison("{}_{}".format(sample_name,row[1]),ms.mass_list, ms.mass_spec)
combine_csv.write("{};{};Page {} of 80;;;;;;{}\n".format(row[1],row[4],index+1,row[2]))
for hit in range(1,6):
report_csv.write(str(matches_dict["Hit{}".format(hit)]))
report_csv.write(";")
combine_csv.write(";;{};{};{};{};{};{};\n".format(hit,
matches_dict["Hit{}".format(hit)]["Lib"],
matches_dict["Hit{}".format(hit)]["MF"],
matches_dict["Hit{}".format(hit)]["RMF"],
matches_dict["Hit{}".format(hit)]["Name"],
matches_dict["Hit{}".format(hit)]["CAS"],
))
report_csv.write("\n")
time.sleep(2)
report_csv.close()
combine_csv.close()
# Create an experiment
expr = Experiment(sample_name, filtered_peak_list)
expr.sele_rt_range(["{}m".format(target_range[0]),"{}m".format(target_range[1])])
store_expr(os.path.join(ExprDir,"{}.expr".format(sample_name)), expr)
return 0
# minimum number of ions, n
n = 3
# greater than or equal to threshold, t
t = 10000
# trim by relative intensity
pl = rel_threshold(peak_list, r)
# trim by threshold
real_peak_list = num_ions_threshold(pl, n, t)
print("Number of filtered peaks in real data: ", len(real_peak_list))
# Set the peak areas
for peak in real_peak_list:
area = peak_sum_area(real_im, peak)
peak.area = area
# real_peak_list is PyMassSpec' best guess at the true peak list
################## Run Simulator ######################
# Simulator takes a peak list, time_list and mass_list
# and returns an IntensityMatrix object.
# The mass_list and time_list are the same for the real
# data and the simulated data.
time_list = real_im.time_list
mass_list = real_im.mass_list
sim_im = gcms_sim(time_list, mass_list, real_peak_list)
# sim_im is an IntensityMatrix object
def test_align_2_alignments(A1, pyms_datadir, tmp_pathplus):
expr_list = []
for jcamp_file in geco_codes:
im = build_intensity_matrix_i(
JCAMP_reader(pyms_datadir / f"{jcamp_file}.JDX"))
# Intensity matrix size (scans, masses)
n_scan, n_mz = im.size
# noise filter and baseline correct
for ii in range(n_mz):
ic = im.get_ic_at_index(ii)
ic_smooth = savitzky_golay(ic)
ic_bc = tophat(ic_smooth, struct="1.5m")
im.set_ic_at_index(ii, ic_bc)
peak_list = BillerBiemann(im, points=9, scans=2)
apl = rel_threshold(peak_list, 2)
new_peak_list = num_ions_threshold(apl, 3, 3000)
# ignore TMS ions and set mass range
for peak in new_peak_list:
peak.crop_mass(50, 400)
peak.null_mass(73)
peak.null_mass(147)
# find area
area = peak_sum_area(im, peak)
peak.area = area
area_dict = peak_top_ion_areas(im, peak)
peak.ion_areas = area_dict
expr = Experiment(jcamp_file, new_peak_list)
# set time range for all experiments
expr.sele_rt_range(["6.5m", "21m"])
expr_list.append(expr)
F2 = exprl2alignment(expr_list)
T2 = PairwiseAlignment(F2, Dw, Gw)
A2 = align_with_tree(T2, min_peaks=2)
# top_ion_list = A2.common_ion()
# A2.write_common_ion_csv(tmp_pathplus/'area1.csv', top_ion_list)
# between replicates alignment parameters
Db = 10.0 # rt modulation
Gb = 0.30 # gap penalty
print("Aligning input {1,2}")
T9 = PairwiseAlignment([A1, A2], Db, Gb)
A9 = align_with_tree(T9)
A9.write_csv(tmp_pathplus / "rt.csv", tmp_pathplus / "area.csv")
aligned_peaks = list(filter(None, A9.aligned_peaks()))
store_peaks(aligned_peaks, tmp_pathplus / "peaks.bin")
top_ion_list = A9.common_ion()
A9.write_common_ion_csv(tmp_pathplus / "area.csv", top_ion_list)
def test_max_bound_errors(self, im_i, peak, obj):
with pytest.raises(TypeError):
peak_sum_area(im_i, peak, max_bound=obj)
# do peak detection on pre-trimmed data
# get the list of Peak objects
pl = BillerBiemann(im, points, scans)
# trim by relative intensity
apl = rel_threshold(pl, r)
# trim by threshold
peak_list = num_ions_threshold(apl, n, t)
print "Number of Peaks found:", len(peak_list)
# ignore TMS ions and set mass range
for peak in peak_list:
peak.crop_mass(50,540)
peak.null_mass(73)
peak.null_mass(147)
# find area
area = peak_sum_area(im, peak)
peak.set_area(area)
# create an experiment
expr = Experiment("a0806_077", peak_list)
# set time range for all experiments
expr.sele_rt_range(["6.5m", "21m"])
store_expr("output/a0806_077.expr", expr)
def quantitative_processing(self, jcamp_file, log_stdout=True):
"""
Import JCAMP-DX Files
:param jcamp_file:
:type jcamp_file:
:param log_stdout:
:type log_stdout:
:return:
:rtype:
"""
# Determine the name of the sample from the filename
sample_name = os.path.splitext(os.path.basename(jcamp_file))[0]
# Log Stdout to File
if log_stdout:
sys.stdout = open(os.path.join(self.config.log_dir, sample_name + ".log"), "w")
# Load data using JCAMP_reader
data = JCAMP_reader(jcamp_file)
# list of all retention times, in seconds
times = data.get_time_list()
# get Total Ion Chromatogram
tic = data.get_tic()
# RT Range, time step, no. scans, min, max, mean and median m/z
data.info()
# Build "intensity matrix" by binning data with integer bins and a
# window of -0.3 to +0.7, the same as NIST uses
im = build_intensity_matrix_i(data)
# Show the m/z of the maximum and minimum bins
print(" Minimum m/z bin: {}".format(im.get_min_mass()))
print(" Maximum m/z bin: {}".format(im.get_max_mass()))
# Crop masses
min_mass, max_mass, *_ = self.config.mass_range
if min_mass < im.get_min_mass():
min_mass = im.get_min_mass()
if max_mass > im.get_max_mass():
max_mass = im.get_max_mass()
im.crop_mass(min_mass, max_mass)
# Perform Data filtering
n_scan, n_mz = im.get_size()
# Iterate over each IC in the intensity matrix
for ii in range(n_mz):
# print("\rWorking on IC#", ii+1, ' ',end='')
ic = im.get_ic_at_index(ii)
# Perform Savitzky-Golay smoothing.
# Note that Turbomass does not use smoothing for qualitative method.
ic_smooth = savitzky_golay(ic)
# Perform Tophat baseline correction
# Top-hat baseline Correction seems to bring down noise,
# retaining shapes, but keeps points on actual peaks
ic_bc = tophat(ic_smooth, struct=self.config.tophat_struct)
# Set the IC in the intensity matrix to the filtered one
im.set_ic_at_index(ii, ic_bc)
# Peak Detection based on Biller and Biemann (1974), with a window
# of <points>, and combining <scans> if they apex next to each other
peak_list = BillerBiemann(im, points=self.config.bb_points, scans=self.config.bb_scans)
print(" Number of peaks identified before filtering: {}".format(len(peak_list)))
# Filtering peak lists with automatic noise filtering
noise_level = window_analyzer(tic)
# should we also do rel_threshold() here?
# https://pymassspec.readthedocs.io/en/master/pyms/BillerBiemann.html#pyms.BillerBiemann.rel_threshold
peak_list = num_ions_threshold(peak_list, self.config.noise_thresh, noise_level)
filtered_peak_list = []
for peak in peak_list:
# Get mass and intensity lists for the mass spectrum at the apex of the peak
apex_mass_list = peak.mass_spectrum.mass_list
apex_mass_spec = peak.mass_spectrum.mass_spec
# Determine the intensity of the base peak in the mass spectrum
base_peak_intensity = max(apex_mass_spec)
# Determine the index of the base peak in the mass spectrum
base_peak_index = [
index for index, intensity in enumerate(apex_mass_spec)
if intensity == base_peak_intensity][0]
# Finally, determine the mass of the base peak
base_peak_mass = apex_mass_list[base_peak_index]
# skip the peak if the base peak is at e.g. m/z 73, i.e. septum bleed
if base_peak_mass in self.config.base_peak_filter:
continue
area = peak_sum_area(im, peak)
peak.set_area(area)
filtered_peak_list.append(peak)
print(" Number of peaks identified: {}".format(len(filtered_peak_list)))
# Save the TIC and Peak List
tic.write(os.path.join(self.config.expr_dir, "{}_tic.dat".format(sample_name)), formatting=False)
store_peaks(filtered_peak_list, os.path.join(self.config.expr_dir, "{}_peaks.dat".format(sample_name)))
# Create an experiment
expr = Experiment(sample_name, filtered_peak_list)
expr.sele_rt_range(["{}m".format(self.config.target_range[0]), "{}m".format(self.config.target_range[1])])
store_expr(os.path.join(self.config.expr_dir, "{}.expr".format(sample_name)), expr)
# greater than or equal to threshold, t
t = 4000
# trim by relative intensity
pl = rel_threshold(peak_list, r)
# trim by threshold
new_peak_list = num_ions_threshold(pl, n, t)
print("Number of filtered peaks: ", len(new_peak_list))
print("Peak areas")
print("UID, RT, height, area")
for peak in new_peak_list:
rt = peak.rt
# determine and set area
area = peak_sum_area(im, peak)
peak.area = area
# print some details
UID = peak.UID
# height as sum of the intensities of the apexing ions
height = sum(peak.get_mass_spectrum().mass_spec.tolist())
print(UID + f", {rt:.2f}, {height:.2f}, {peak.area:.2f}")
# TIC from raw data
tic = data.get_tic()
# baseline correction for TIC
tic_bc = tophat(tic, struct="1.5m")
# Get Ion Chromatograms for all m/z channels
n_mz = len(im.get_mass_list())
def Peak_detector(pp_im, noise, name, points, scans, percent, ni, name_tag, sdir):
"""
Intake cleansed intensity matrices and CMD args
Produces list of peaks and corresponding mass spectrum of each sample
@param pp_im: Cleansed intensity matrices from the Preprocess_IntensityMatrices method
@param noise: Noise level approximation produced by the matrix_from_cdf method
@param name: Sample name use from creating mass spectrum .csv files
@param points: Size of window use for peak detection in BillerBiemann method
@param scans: Number of adjacent windows to compare for peak detection in BillerBiemann method
@param percent: Percentile threshold a peak must exceed to be considered an informative peak
@param ni: Number of ions required per peak to be considered an informative peak
@param name_tag: String consisting of CMD args for identification, ie. 'p140s25%3n3'
@param sdir: Directory to save the mass spectrum .csv files
@return: List of peaks per sample
@return: csv files containing mass spectrum corresponding to each peak
"""
peakz = []
savePath = sdir
ms_data_files = []
print("len pp_im", len(list(pp_im)))
print("len noise", len(noise))
print("len name", len(name), name)
for im, n, na in itertools.izip(list(pp_im), noise, name):
poss_peaks = BillerBiemann(im, points=points, scans=scans)
pi = rel_threshold(poss_peaks, percent=percent)
nin = num_ions_threshold(pi, n=ni, cutoff=n)
completeName = os.path.join(savePath, na + name_tag + "ms_data.csv")
with open(completeName, 'w') as f:
w = csv.writer(f)
head = ['Area', 'RTs'] + [float(i) for i in range(35,221)]
w.writerow(head)
for peak in nin:
area = peak_sum_area(im, peak)
peak.set_area(area)
ms = peak.get_mass_spectrum()
p_rt = peak.get_rt()
its = []
ms_items = list(ms.mass_spec)
for spec in ms_items:
f_spec = float(spec)
its.append(f_spec)
ms_d = ([area] + [p_rt] + its)
w.writerow(ms_d)
f.close()
peakz.append(nin)
ms_data_files.append(completeName)
print('ms_data_files:', ms_data_files)
return [peakz, ms_data_files]
