def test_compress(): a = Trace(np.array([10, 20, 30, 40, 50]), np.array([1, 2, 3, 4, 5]), name='X') zdata = 'eJxjZWBgEAHiaKUIpVgGhgZ7INsBiIC4' \ 'AYgXADEIqEBpEyhtB6VdoLSnAwAZfgbw' assert base64.b64encode(a.compress()).decode('ascii') == zdata
def setUp(self): # box peak info = {} trace = Trace(np.ones(10), np.arange(10), name=1) baseline = Trace([0, 9], [0, 0], name=1) c = PeakComponent(info, trace, baseline) self.peak = Peak('box', components=c)
def test_add(): t = np.array([1, 2, 3, 4, 5]) a = Trace(np.array([10, 20, 30, 40, 50]), t, name='X') b = Trace(np.array([11, 21, 31, 41, 51]), t, name='X') c = a + b assert np.all(np.equal(c.values, np.array([21, 41, 61, 81, 101]))) assert np.all(np.equal(c.index, np.array([1, 2, 3, 4, 5])))
def setUp(self): # box peak info = {} v = [ 0.043, 0.067, 0.094, 0.117, 0.131, 0.131, 0.117, 0.094, 0.067, 0.043 ] trace = Trace(v, np.arange(10), name=1) baseline = Trace([0, 9], [0, 0], name=1) c = PeakComponent(info, trace, baseline) self.peak = Peak('gaussian', components=c)
def trace(self, name='', tol=0.5, twin=None): if isinstance(name, (int, float, np.float32, np.float64)): name = str(name) else: name = name.lower() # name of the 2d trace, if it exists if any(t.startswith('#') for t in self.traces): t2d = [t[1:] for t in self.traces if t.startswith('#')][0] # clip out the starting 'MS' if present if name.startswith(t2d): name = name[len(t2d):] else: t2d = '' # this is the only string we handle; all others handled in subclasses if name in ['tic', 'x', '']: return self.total_trace(twin) elif name in self.traces: return self._trace(name, twin) elif t2d != '': # this file contains 2d data; find the trace in that return self.data.trace(name, tol, twin) else: return Trace()
def mrm_trace(self, parent=None, daughter=None, tol=0.5, twin=None): # TODO: should override `trace` and then call parent's `trace` method # if name is not an mrm trace if twin is None: twin = (-np.inf, np.inf) tme, ic = [], [] for t, off, bc, pc, minx, maxx, d_mz, p_mz, z in self._scan_iter([ 'ScanTime', 'SpectrumOffset', 'ByteCount', 'PointCount', 'MinX', 'MaxX', 'BasePeakMZ', 'MzOfInterest', 'TIC' ]): if t < twin[0]: continue elif t > twin[1]: break if parent is not None: if np.abs(parent - p_mz) > tol: continue if daughter is not None: if np.abs(daughter - d_mz) > tol: continue tme.append(t) ic.append(z) return Trace(np.array(ic), np.array(tme), name=str(str(parent) + '→' + str(daughter)))
def series_from_str(val, times, name=''): # TODO: generate this without needing the times? just the time length # we can store time-series data as a list of timepoints # in certain info fields and query it here def is_num(x): # stupid function to determine if something is a number try: float(x) return True except ValueError: return False if ',' in val: # turn the time list into a dictionary tpts = dict([tpt.split(':') for tpt in val.split(',')]) # get the valid times out valid_x = [v for v in tpts if is_num(v)] # generate arrays from them x = np.array([float(v) for v in valid_x]) y = np.array([float(tpts[v]) for v in valid_x]) srt_ind = np.argsort(x) if 'S' in tpts: # there's a "S"tart value defined d = np.interp(times, x[srt_ind], y[srt_ind], float(tpts['S'])) else: d = np.interp(times, x[srt_ind], y[srt_ind]) elif is_num(val): d = np.ones(times.shape) * float(val) else: d = np.ones(times.shape) * np.nan return Trace(d, times, name=name)
def basemz(df): """ The mz of the most abundant ion. """ # returns the d = np.array(df.columns)[df.values.argmax(axis=1)] return Trace(d, df.index, name='basemz')
def total_trace(self, twin=None): # TODO: use twin? f = open(self.filename, 'rb') # get number of scans to read in f.seek(0x5) if f.read(4) == 'GC': f.seek(0x142) else: f.seek(0x118) nscans = struct.unpack('>H', f.read(2))[0] # find the starting location of the data f.seek(0x10A) f.seek(2 * struct.unpack('>H', f.read(2))[0] - 2) tme = np.zeros(nscans) tic = np.zeros(nscans) for i in range(nscans): npos = f.tell() + 2 * struct.unpack('>H', f.read(2))[0] tme[i] = struct.unpack('>I', f.read(4))[0] / 60000. f.seek(npos - 4) tic[i] = struct.unpack('>I', f.read(4))[0] f.seek(npos) f.close() return Trace(tic, tme, name='TIC')
def total_trace(self, twin=None): f = open(self.filename, 'rb') f.seek(0x11A) start_time = struct.unpack('>f', f.read(4))[0] / 60000. # end_time = struct.unpack('>f', f.read(4))[0] / 60000. # end_time 0x11E '>i' # FIXME: why is there this del_ab code here? # f.seek(0x284) # del_ab = struct.unpack('>d', f.read(8))[0] data = [] f.seek(0x400) delt = 0 while True: try: inp = struct.unpack('>h', f.read(2))[0] except struct.error: break if inp == 32767: inp = struct.unpack('>i', f.read(4))[0] inp2 = struct.unpack('>H', f.read(2))[0] delt = 0 data.append(inp * 65534 + inp2) else: delt += inp data.append(data[-1] + delt) f.close() # TODO: 0.4/60.0 should be obtained from the file??? times = np.array(start_time + np.arange(len(data)) * (0.2 / 60.0)) # times = np.linspace(start_time, end_time, data.shape[0]) return Trace(np.array([data]).T, times, name='TIC')
def fft(ts): """ Perform a fast-fourier transform on a Trace """ t_step = ts.index[1] - ts.index[0] oc = np.abs(np.fft.fftshift(np.fft.fft(ts.values))) / len(ts.values) t = np.fft.fftshift(np.fft.fftfreq(len(oc), d=t_step)) return Trace(oc, t)
def _trace(self, name, twin): # TODO: read info from new style REG files df = read_multireg_file(open(self.filename, 'rb'), title=self._tr_names[name]) if 'Trace' not in df: return Trace([], []) ts = df['Trace'] ts.name = name return ts.twin(twin)
def data(self): # convenience function for reading in data def rd(f, st): return struct.unpack(st, f.read(struct.calcsize(st))) # open the file f = open(self.filename, 'rb') nscans = rd(f, 'ii')[1] if nscans == 0: self.data = Trace(np.array([]), np.array([]), []) return times = np.array(rd(f, nscans * 'd')) / 60.0 f.seek(f.tell() + 4) # number of scans again # set up the array of column indices indptr = np.empty(nscans + 1, dtype=int) indptr[0] = 0 # figure out the total number of points dpos = f.tell() tot_pts = 0 for scn in range(nscans): npts = rd(f, 'i')[0] # rd(f, npts * 'f' + 'i' + n_pts * 'f') f.seek(f.tell() + 8 * npts + 4) tot_pts += npts indptr[scn + 1] = tot_pts f.seek(dpos) ions = [] i_lkup = {} idxs = np.empty(tot_pts, dtype=int) vals = np.empty(tot_pts, dtype=float) for scn in range(nscans): npts = rd(f, 'i')[0] rd_ions = rd(f, npts * 'f') f.seek(f.tell() + 4) # number of points again abun = rd(f, npts * 'f') nions = set([int(i) for i in rd_ions if int(i) not in i_lkup]) i_lkup.update( dict((ion, i + len(ions) - 1) for i, ion in enumerate(nions))) ions += nions idxs[indptr[scn]:indptr[scn + 1]] = \ [i_lkup[int(i)] for i in rd_ions] vals[indptr[scn]:indptr[scn + 1]] = \ abun idxs += 1 data = scipy.sparse.csr_matrix((vals, idxs, indptr), shape=(nscans, len(ions)), dtype=float) return Chromatogram(data, times, ions)
def mzminus(df, minus=0, noise=10000): """ The abundances of ions which are minus below the molecular ion. """ mol_ions = ((df.values > noise) * df.columns).max(axis=1) - minus mol_ions[np.abs(mol_ions) < 0] = 0 d = np.abs(np.ones(df.shape) * df.columns - (mol_ions[np.newaxis].T * np.ones(df.shape))) < 1 d = (df.values * d).sum(axis=1) return Trace(d, df.index, name='m-' + str(minus))
def total_trace(self, twin=None): # TODO: use twin # TODO: only get the scans with totIonCurrent; if none found # calculate from the data r = Et.parse(self.filename).getroot() s = r.findall('*//m:scan', namespaces=self.ns) d = np.array([float(i.get('totIonCurrent')) for i in s]) t = np.array([t_to_min(i.get('retentionTime')) for i in s]) return Trace(d, t, name='TIC')
def loads(ast_str): """ Create a Trace from a suitably compressed string. """ data = zlib.decompress(ast_str) li = struct.unpack('<L', data[0:4])[0] lt = struct.unpack('<L', data[4:8])[0] n = data[8:8 + li].decode('utf-8') t = np.fromstring(data[8 + li:8 + li + lt]) d = np.fromstring(data[8 + li + lt:]) return Trace(d, t, name=n)
def total_trace(self, twin=None): r = Et.parse(self.filename).getroot() # get it from the chromatogram list c = r.find('.//m:cvParam[@accession="MS:1000235"]/..', namespaces=self.ns) if c is not None: q = './/m:cvParam[@accession="MS:1000595"]/..' index = self.read_binary(c.find(q, namespaces=self.ns)) q = './/m:cvParam[@accession="MS:1000515"]/..' values = self.read_binary(c.find(q, namespaces=self.ns)) return Trace(values, index, name='tic')
def total_trace(self, twin=None): if twin is None: twin = (-np.inf, np.inf) tme = [] tic = [] for t, z in self._scan_iter(['ScanTime', 'TIC']): if t < twin[0]: continue elif t > twin[1]: break tme.append(t) tic.append(z) return Trace(np.array(tic), np.array(tme), name='TIC')
def total_trace(self, twin=None): if twin is None: twin = (-np.inf, np.inf) times, y = [], [] for s in self.scans(twin): t = float(s.name) if t < twin[0]: continue if t > twin[1]: break times.append(t) y.append(sum(s.abn)) return Trace(y, times, name='tic')
def generate_chromatogram(n=5, twin=None): """ Generates a trace with n gaussian peaks distributed through it. """ if twin is None: twin = (0, 60) t = np.linspace(twin[0], twin[1], 300) peak_locs = twin[1] * np.random.random(n) peak_ws = 0.2 + 0.8 * np.random.random(n) peak_hs = 0.2 + 0.8 * np.random.random(n) y = np.zeros(len(t)) for peak_loc, peak_w, peak_h in zip(peak_locs, peak_ws, peak_hs): y += gaussian(t, x=peak_loc, w=peak_w, h=peak_h) y += np.random.normal(scale=0.01, size=len(t)) return Trace(y, t, ['X'])
def total_trace(self, twin=None): f = open(self.filename, 'rb') f.seek(0x11A) start_time = struct.unpack('>f', f.read(4))[0] / 60000. end_time = struct.unpack('>f', f.read(4))[0] / 60000. # TODO: figure out if this exists and where? # FID signal seems like 10x higher than it should be? # f.seek(0x284) # del_ab = 0.1 # struct.unpack('>d', f.read(8))[0] # data = [] f.seek(0x1800) data = np.fromfile(f, '<f8') times = np.linspace(start_time, end_time, data.shape[0]) return Trace(data, times, name='TIC')
def trace(self, name='', tol=0.5, twin=None): if twin is None: twin = (-np.inf, np.inf) if name in {'tic', 'x', ''}: return self.total_trace(twin) times, y = [], [] for s in self.scans(twin): t = float(s.name) if t < twin[0]: continue if t > twin[1]: break times.append(t) # TODO: this can be vectorized with numpy? y.append(sum(j for i, j in zip(s.x, s.abn) if np.abs(i - name) < tol)) return Trace(y, times, name=name)
def read_mh_trace(filename, trace_name): f = open(filename, 'rb') fdat = open(filename[:-3] + '.cg', 'rb') ttab = { 'pres': 'Pressure', 'flow': 'Flow', 'slvb': '%B', 'temp': 'Temperature of Left Heat Exchanger' } # convenience function for reading in data def rd(st): return struct.unpack(st, f.read(struct.calcsize(st))) f.seek(0x4c) num_traces = rd('<I')[0] for _ in range(num_traces): cloc = f.tell() f.seek(cloc + 2) sl = rd('<B')[0] cur_trace_name = rd('<' + str(sl) + 's')[0] if ttab[trace_name] == cur_trace_name: f.seek(f.tell() + 4) foff = rd('<Q')[0] npts = rd('<I')[0] + 2 # +2 for the extra time info fdat.seek(foff) pts = struct.unpack('<' + npts * 'd', fdat.read(8 * npts)) # TODO: pts[0] is not the true offset? t = pts[0] + pts[1] * np.arange(npts - 2) d = np.array(pts[2:]) # get the units f.seek(f.tell() + 40) sl = rd('<B')[0] y_units = rd('<' + str(sl) + 's')[0] if y_units == 'bar': d *= 0.1 # convert to MPa for metricness elif y_units == '': pass # TODO: ul/min to ml/min return Trace(d, t, name=trace_name) f.seek(cloc + 87)
def test_compress(): a = Trace(np.array([10, 20, 30, 40, 50]), np.array([1, 2, 3, 4, 5]), name='X') zdata = 'eJxjZWBgEAHiaKUIpVgGhgZ7INsBiIC4' \ 'AYgXADEIqEBpEyhtB6VdoLSnAwAZfgbw' assert base64.b64encode(a.compress()).decode('ascii') == zdata
def generate_gaussian(): t = np.linspace(0, 60, 300) y = gaussian(t, x=30, w=2, h=1) return Trace(y, t, ['X'])
def read_reg_file(f, foff=0x2D): """ Given a file handle for an old-style Agilent *.REG file, this will parse that file into a dictonary of key/value pairs (including any tables that are in the *.REG file, which will be parsed into lists of lists). """ # convenience function for reading in data def rd(st): return struct.unpack(st, f.read(struct.calcsize(st))) f.seek(0x19) if f.read(1) != b'A': # raise TypeError("Version of REG file is too new.") return {} f.seek(foff) nrecs = rd('<I')[0] # TODO: should be '<H' rec_tab = [rd('<HHIII') for n in range(nrecs)] names = {} f.seek(foff + 20 * nrecs + 4) for r in rec_tab: d = f.read(r[2]) if r[1] == 1539: # '0306' # this is part of the linked list too, but contains a # reference to a table cd = struct.unpack('<HIII21sI', d) names[cd[5]] = cd[4].decode('iso8859').strip('\x00') # except: # pass elif r[1] == 32769 or r[1] == 32771: # b'0180' or b'0380' names[r[4]] = d[:-1].decode('iso8859') elif r[1] == 32774: # b'0680' # this is a string that is referenced elsewhere (in a table) names[r[4]] = d[2:-1].decode('iso8859') elif r[1] == 32770: # b'0280' # this is just a flattened numeric array names[r[4]] = np.frombuffer(d, dtype=np.uint32, offset=4) data = {} f.seek(foff + 20 * nrecs + 4) for r in rec_tab: d = f.read(r[2]) if r[1] == 1538: # '0206' # this is part of a linked list if len(d) == 43: cd = struct.unpack('<HIII21sd', d) data[cd[4].decode('iso8859').strip('\x00')] = cd[5] else: pass elif r[1] == 1537: # b'0106' # name of property n = d[14:30].split(b'\x00')[0].decode('iso8859') # with value from names data[n] = names.get(struct.unpack('<I', d[35:39])[0], '') elif r[1] == 1793: # b'0107' # this is a table of values nrow = struct.unpack('<H', d[4:6])[0] ncol = struct.unpack('<H', d[16:18])[0] if ncol != 0: cols = [ struct.unpack('<16sHHHHHI', d[20 + 30 * i:50 + 30 * i]) for i in range(ncol) ] colnames = [ c[0].split(b'\x00')[0].decode('iso8859') for c in cols ] # TODO: type 2 is not a constant size? 31, 17 rty2sty = { 1: 'H', 3: 'I', 4: 'f', 5: 'H', 7: 'H', 8: 'd', 11: 'H', 12: 'H', 13: 'I', 14: 'I', 16: 'H' } coltype = '<' + ''.join( [rty2sty.get(c[3], str(c[2]) + 's') for c in cols]) lencol = struct.calcsize(coltype) tab = [] for i in reversed(range(2, nrow + 2)): rawrow = struct.unpack(coltype, d[-i * lencol:(1 - i) * lencol]) row = [] for j, p in enumerate(rawrow): if cols[j][3] == 3: row.append(names.get(p, str(p))) else: row.append(p) tab.append(row) data[names[r[4]]] = [colnames, tab] elif r[1] == 1281 or r[1] == 1283: # b'0105' or b'0305' fm = '<HHBIIhIdII12shIddQQB8sII12shIddQQB8s' m = struct.unpack(fm, d) nrecs = m[4] # number of points in table # x_units = names.get(m[8], '') x_arr = m[14] * names.get(m[9], np.arange(nrecs - 1)) y_arr = m[25] * names.get(m[20]) y_units = names.get(m[19], '') if y_units == 'bar': y_arr *= 0.1 # convert to MPa # TODO: what to call this? data['Trace'] = Trace(y_arr, x_arr, name='') # elif r[1] == 1025: # b'0104' # # lots of zeros? maybe one or two numbers? # # only found in REG entries that have long 0280 records # fm = '<HQQQIHHHHIIHB' # m = struct.unpack(fm, d) # print(m) # #print(r[1], len(d), binascii.hexlify(d)) # pass # elif r[1] == 512: # b'0002' # # either points to two null pointers or two other pointers # # (indicates start of linked list?) # print(r[1], len(d), binascii.hexlify(d)) # elif r[1] == 769 or r[1] == 772: # b'0103' or b'0403' # # points to 2nd, 3rd & 4th records (two 0002 records and a 0180) # b = binascii.hexlify # print(b(d[10:14]), b(d[14:18]), b(d[18:22])) return data
def trace_resolver(istr, analyses, twin=None): avail_sources = [ i.lstrip('#*') for a in analyses for i in a.trace.split(',') ] istr, source = token_source(istr, avail_sources) if source is None: return Trace() for a in analyses: if source in [i.lstrip('#*') for i in a.trace.split(',')]: df = a.datafile break else: df = None if istr in {'coda', 'rnie', 'wmsm'}: # TODO: allow more complicated options to turn # Chromatograms into plotable Traces # coda # Windig W: The use of the Durbin-Watson criterion for # noise and background reduction of complex liquid # chromatography/mass spectrometry data and a new algorithm # to determine sample differences. Chemometrics and # Intelligent Laboratory Systems. 2005, 77:206-214. # rnie # Yunfei L, Qu H, and Cheng Y: A entropy-based method # for noise reduction of LC-MS data. Analytica Chimica # Acta 612.1 (2008) # wmsm # Fleming C et al. Windowed mass selection method: # a new data processing algorithm for LC-MS data. # Journal of Chromatography A 849.1 (1999) 71-85. pass elif istr.startswith('m_'): if istr == 'm_': m = 0.0 else: m = float(istr.split('_')[1]) return mzminus(df.data, m) elif istr == 'molmz': return molmz(df.data) elif istr == 'basemz': return basemz(df.data) elif istr in {'r45std', 'r46std'}: # TODO: calculate isotopic data -> needs integrated peak # references of associated peaks in order to make these calculations pass # calculate isotopic reference for chromatogram # if name == 'r45std': # topion = 45 # else: # topion = 46 # std_specs = [o for o in \ # self.children_of_type('peak') \ # if o.info['p-type'] == 'Isotope Standard'] # x = [float(o.info['p-s-time']) for o in std_specs] # y = [o.area(topion) / o.area(44) for o in std_specs \ # if o.area(44) != 0] # if len(x) == 0 or len(y) == 0: # return self._const(0.0, twin) # p0 = [y[0], 0] # errfunc = lambda p, x, y: p[0] + p[1] * x - y # try: # p, succ = leastsq(errfunc, p0, args=(np.array(x), \ # np.array(y))) # except: # p = p0 # sim_y = np.array(errfunc(p, t, np.zeros(len(t)))) # return TimeSeries(sim_y, t, [name]) else: # interpret tolerances if ':' in istr: st = float(istr.split(':')[0]) en = float(istr.split(':')[1]) tol = 0.5 * (en - st) istr = 0.5 * (en + st) elif u'±' in istr: tol = float(istr.split(u'±')[1]) istr = float(istr.split(u'±')[0]) else: tol = 0.5 return df.trace(istr, tol, twin=twin)
def molmz(df, noise=10000): """ The mz of the molecular ion. """ d = ((df.values > noise) * df.columns).max(axis=1) return Trace(d, df.index, name='molmz')
def parse_ion_string(istr, analyses, twin=None): """ Recursive string parser that handles "ion" strings. """ if istr.strip() == '': return Trace() # remove (unnessary?) pluses from the front # TODO: plus should be abs? istr = istr.lstrip('+') # invert it if preceded by a minus sign if istr[0] == '-': return -parse_ion_string(istr[1:], analyses, twin) # this is a function or paranthesized expression if is_parans_exp(istr): if ')' not in istr: # unbalanced parantheses pass fxn = istr.split('(')[0] args = istr[istr.find('(') + 1:istr.find(')')].split(',') if fxn == '': # strip out the parantheses and continue istr = args[0] else: ts = parse_ion_string(args[0], analyses, twin) # FIXME return ts # return fxn_resolver(ts, fxn, *args[1:]) # all the complicated math is gone, so simple lookup if set(istr).intersection(set('+-/*()')) == set(): if istr in SHORTCUTS: # allow some shortcuts to pull out common ions return parse_ion_string(SHORTCUTS[istr], analyses, twin) elif istr[0] == '!' and all(i in '0123456789.' for i in istr[1:]): # TODO: should this handle negative numbers? return float(istr[1:]) elif istr == '!pi': return np.pi elif istr == '!e': return np.e else: return trace_resolver(istr, analyses, twin) # go through and handle operators for token in '/*+-^': if len(tokenize(istr, token)) != 1: ts = tokenize(istr, token) s = parse_ion_string(ts[0], analyses, twin) for t in ts[1:]: if token == '/': s /= parse_ion_string(t, analyses, twin) elif token == '*': s *= parse_ion_string(t, analyses, twin) elif token == '+': s += parse_ion_string(t, analyses, twin) elif token == '-': s -= parse_ion_string(t, analyses, twin) elif token == '^': s **= parse_ion_string(t, analyses, twin) return s raise Exception('Parser hit a point it shouldn\'t have!')
def ratio_series(ts, pks, r2, r1): sim_y = ratio_f(pks, r2, r1) return Trace(sim_y(ts.index), ts.index, name='{:.1f}/{:.1f}'.format(r2, r1))