def read(filename): """ Parse a ptracker log file (plg) for the pitch trace. Returns a numpy recarray, with element number is indicated by a separate field. Note that this is different from the pitchtrace object that's used to store data on separate chains. """ from numpy import rec current_element = -1 field_names = None records = [] with open(filename, "rt") as fp: for line in fp: m = _el_rx.match(line) if m: current_element = int(m.group(1)) elif line.startswith("time"): field_names = line.split() elif line[0] not in ("*", "+", "-"): values = (current_element,) + tuple(float(x) for x in line.split()) records.append(values) elif line.find("error") > -1: raise Error, "Pitch file %s has an error: %s" % (filename, line) field_types = ("i4",) + ("f8",) * len(field_names) field_names.insert(0, "element") return rec.fromrecords(records, dtype={"names": field_names, "formats": field_types})
def read(filename): """ Parse a ptracker log file (plg) for the pitch trace. Returns a numpy recarray, with element number is indicated by a separate field. Note that this is different from the pitchtrace object that's used to store data on separate chains. """ from numpy import rec current_element = -1 field_names = None records = [] with open(filename, 'rt') as fp: for line in fp: m = _el_rx.match(line) if m: current_element = int(m.group(1)) elif line.startswith("time"): field_names = line.split() elif line[0] not in ('*', '+', '-'): values = (current_element,) + tuple(float(x) for x in line.split()) records.append(values) elif line.find("error") > -1: raise Error("file has an error: %s" % (line)) if len(records) == 0: raise Error("file contains no data") field_types = ('i4',) + ('f8',) * len(field_names) field_names.insert(0, 'element') return rec.fromrecords(records, dtype={'names': field_names, 'formats': field_types})
def add_field(A, field, array): if not len(array)==len(A): raise ValueError, 'Length of new field is not same as length of old array.' names = A.dtype.names + (field,) recs = [] for i in xrange(len(A)): recs.append(list(A[i]) + [array[i]]) return rec.fromrecords(recs, names=names)
def combine_by_field(A1, A2, fields): if not len(A1) == len(A2): print 'combine_by_field: Warning, arrays aren\'t same length.' # Find names that are unique and those that are shared. A1_names = set(A1.dtype.names) A2_names = set(A2.dtype.names) both_names = A1_names & A2_names A1_names -= both_names A2_names -= both_names names = list(A1_names | A2_names | both_names) A1_names = list(A1_names) A2_names = list(A2_names) both_names = list(both_names) # Iterate over records. out_recs = [] for i1 in xrange(len(A1)): # Find all the records in A2 that match all the records in A1 at fields. out_recs.append([]) val1 = [] i2_remaining = arange(len(A2)) for field in fields: i2_remaining = i2_remaining[where(A2[field][i2_remaining]==A1[i1][field])[0]] # Check for no matches or multiple matches and warn verbally if there are any. if len(i2_remaining)>1: print 'Warning, match underspecified ', i1,i2_remaining, len(i2_remaining) elif len(i2_remaining)==0: print 'Warning, no match ',i1 # Take one of the records in A2 that matches (if any) and fuse it to the current # record in A1. try: i2 = i2_remaining[0] for name in A1_names + both_names: out_recs[i1].append(A1[i1][name]) for name in A2_names: out_recs[i1].append(A2[i2][name]) except: pass A = rec.fromrecords(out_recs, names=A1_names + both_names + A2_names) return A
def Read(self): """ Read the data from the file. """ def addkeyval(lst, k, v): """ add new key to a list. if key already exists a number will be appended to the key name Parameters ---------- lst : list k : str key v : object value """ kcnt = 0 key = k while key in lst: key = k + "_%i" % (kcnt + 1) kcnt += 1 lst[key] = v col_names = [] col_units = [] col_types = [] rec_list = [] with open(self.filename, 'rb') as fid: for line in fid: line = line.decode('utf8', 'ignore') line = line.strip() # read the next line if the line starts with a "!" if re_end_section.match(line): continue # select the which section to read if re_comment_section.match(line): read_mode = 1 continue if re_parameter_section.match(line): read_mode = 2 continue if re_data_section.match(line): read_mode = 3 continue # here we decide how to proceed with the data if read_mode == 1: # read the file comments try: (key, value) = line.split("=") except ValueError: # avoid annoying output if config.VERBOSITY >= config.INFO_ALL: print("XU.io.SPECTRAFile.Read: cannot interpret " "the comment string: %s" % (line)) continue key = key.strip() # remove whitespaces to be conform with natural naming key = key.replace(' ', '') key = key.replace(':', '_') # remove possible number at first position if re_num.findall(key[0]) != []: key = "_" + key value = value.strip() if config.VERBOSITY >= config.DEBUG: print("XU.io.SPECTRAFile.Read: comment: k, v: %s, %s" % (key, value)) try: value = float(value) except ValueError: pass # need to handle the case, that a key may appear several # times in the list addkeyval(self.comments, key, value) elif read_mode == 2: # read scan parameters try: (key, value) = line.split("=") except ValueError: print("XU.io.SPECTRAFile.Read: cannot interpret the " "parameter string: %s" % (line)) key = key.strip() # remove whitespaces to be conform with natural naming key = key.replace(' ', '') key = key.replace(':', '_') # remove possible number at first position if re_num.findall(key[0]) != []: key = "_" + key value = value.strip() if config.VERBOSITY >= config.DEBUG: print( "XU.io.SPECTRAFile.Read: parameter: k, v: %s, %s" % (key, value)) try: value = float(value) except ValueError: # if the conversion of the parameter to float # fails it will be saved as a string pass # need to handle the case, that a key may appear several # times in the list addkeyval(self.params, key, value) elif read_mode == 3: if re_column.match(line): try: unit = re_unit.findall(line)[0] except IndexError: unit = "NONE" try: sline = re_obracket.split(line) if len(sline) == 1: raise IndexError lval = sline[0] rval = re_cbracket.split(line)[-1] dtype = rval.strip() lv = re_wspaces.split(lval) index = int(lv[1]) name = "".join(lv[2:]) name = name.replace(':', '_') except IndexError: lv = re_wspaces.split(line) index = int(lv[1]) dtype = lv[-1] name = "".join(lv[2:-1]) name = name.replace(':', '_') # store column definition self.data.append( SPECTRAFileDataColumn(index, name, unit, dtype)) if name in col_names: name += "%s_1" % name col_names.append("%s" % name) col_types.append("%s" % (dtype_map[dtype])) else: # read data dlist = re_wspaces.split(line) for i in range(len(dlist)): dlist[i] = float(dlist[i]) rec_list.append(tuple(dlist)) if config.VERBOSITY >= config.DEBUG: print("XU.io.SPECTRAFile.Read: data columns: name, type: %s, %s" % (col_names, col_types)) if rec_list: self.data.data = rec.fromrecords(rec_list, formats=col_types, names=col_names) else: self.data.data = None
def Read(self): """ Read the data from the file. """ def addkeyval(lst, k, v): """ add new key to a list. if key already exists a number will be appended to the key name Parameters ---------- lst : list k : str key v : object value """ kcnt = 0 key = k while key in lst: key = k + "_%i" % (kcnt + 1) kcnt += 1 lst[key] = v col_names = [] col_units = [] col_types = [] rec_list = [] with open(self.filename, 'rb') as fid: for line in fid: line = line.decode('utf8', 'ignore') line = line.strip() # read the next line if the line starts with a "!" if re_end_section.match(line): continue # select the which section to read if re_comment_section.match(line): read_mode = 1 continue if re_parameter_section.match(line): read_mode = 2 continue if re_data_section.match(line): read_mode = 3 continue # here we decide how to proceed with the data if read_mode == 1: # read the file comments try: (key, value) = line.split("=") except ValueError: # avoid annoying output if config.VERBOSITY >= config.INFO_ALL: print("XU.io.SPECTRAFile.Read: cannot interpret " "the comment string: %s" % (line)) continue key = key.strip() # remove whitespaces to be conform with natural naming key = key.replace(' ', '') key = key.replace(':', '_') # remove possible number at first position if re_num.findall(key[0]) != []: key = "_" + key value = value.strip() if config.VERBOSITY >= config.DEBUG: print("XU.io.SPECTRAFile.Read: comment: k, v: %s, %s" % (key, value)) try: value = float(value) except ValueError: pass # need to handle the case, that a key may appear several # times in the list addkeyval(self.comments, key, value) elif read_mode == 2: # read scan parameters try: (key, value) = line.split("=") except ValueError: print("XU.io.SPECTRAFile.Read: cannot interpret the " "parameter string: %s" % (line)) key = key.strip() # remove whitespaces to be conform with natural naming key = key.replace(' ', '') key = key.replace(':', '_') # remove possible number at first position if re_num.findall(key[0]) != []: key = "_" + key value = value.strip() if config.VERBOSITY >= config.DEBUG: print("XU.io.SPECTRAFile.Read: parameter: k, v: %s, %s" % (key, value)) try: value = float(value) except ValueError: # if the conversion of the parameter to float # fails it will be saved as a string pass # need to handle the case, that a key may appear several # times in the list addkeyval(self.params, key, value) elif read_mode == 3: if re_column.match(line): try: unit = re_unit.findall(line)[0] except IndexError: unit = "NONE" try: sline = re_obracket.split(line) if len(sline) == 1: raise IndexError lval = sline[0] rval = re_cbracket.split(line)[-1] dtype = rval.strip() lv = re_wspaces.split(lval) index = int(lv[1]) name = "".join(lv[2:]) name = name.replace(':', '_') except IndexError: lv = re_wspaces.split(line) index = int(lv[1]) dtype = lv[-1] name = "".join(lv[2:-1]) name = name.replace(':', '_') # store column definition self.data.append( SPECTRAFileDataColumn(index, name, unit, dtype)) if name in col_names: name += "%s_1" % name col_names.append("%s" % name) col_types.append("%s" % (dtype_map[dtype])) else: # read data dlist = re_wspaces.split(line) for i in range(len(dlist)): dlist[i] = float(dlist[i]) rec_list.append(tuple(dlist)) if config.VERBOSITY >= config.DEBUG: print("XU.io.SPECTRAFile.Read: data columns: name, type: %s, %s" % (col_names, col_types)) if rec_list: self.data.data = rec.fromrecords(rec_list, formats=col_types, names=col_names) else: self.data.data = None