def read(filename):
"""
Parse a ptracker log file (plg) for the pitch trace. Returns a
numpy recarray, with element number is indicated by a separate
field. Note that this is different from the pitchtrace object
that's used to store data on separate chains.
"""
from numpy import rec
current_element = -1
field_names = None
records = []
with open(filename, "rt") as fp:
for line in fp:
m = _el_rx.match(line)
if m:
current_element = int(m.group(1))
elif line.startswith("time"):
field_names = line.split()
elif line[0] not in ("*", "+", "-"):
values = (current_element,) + tuple(float(x) for x in line.split())
records.append(values)
elif line.find("error") > -1:
raise Error, "Pitch file %s has an error: %s" % (filename, line)
field_types = ("i4",) + ("f8",) * len(field_names)
field_names.insert(0, "element")
return rec.fromrecords(records, dtype={"names": field_names, "formats": field_types})
def read(filename):
"""
Parse a ptracker log file (plg) for the pitch trace. Returns a
numpy recarray, with element number is indicated by a separate
field. Note that this is different from the pitchtrace object
that's used to store data on separate chains.
"""
from numpy import rec
current_element = -1
field_names = None
records = []
with open(filename, 'rt') as fp:
for line in fp:
m = _el_rx.match(line)
if m:
current_element = int(m.group(1))
elif line.startswith("time"):
field_names = line.split()
elif line[0] not in ('*', '+', '-'):
values = (current_element,) + tuple(float(x) for x in line.split())
records.append(values)
elif line.find("error") > -1:
raise Error("file has an error: %s" % (line))
if len(records) == 0:
raise Error("file contains no data")
field_types = ('i4',) + ('f8',) * len(field_names)
field_names.insert(0, 'element')
return rec.fromrecords(records, dtype={'names': field_names, 'formats': field_types})
def add_field(A, field, array):
if not len(array)==len(A):
raise ValueError, 'Length of new field is not same as length of old array.'
names = A.dtype.names + (field,)
recs = []
for i in xrange(len(A)):
recs.append(list(A[i]) + [array[i]])
return rec.fromrecords(recs, names=names)
def combine_by_field(A1, A2, fields):
if not len(A1) == len(A2):
print 'combine_by_field: Warning, arrays aren\'t same length.'
# Find names that are unique and those that are shared.
A1_names = set(A1.dtype.names)
A2_names = set(A2.dtype.names)
both_names = A1_names & A2_names
A1_names -= both_names
A2_names -= both_names
names = list(A1_names | A2_names | both_names)
A1_names = list(A1_names)
A2_names = list(A2_names)
both_names = list(both_names)
# Iterate over records.
out_recs = []
for i1 in xrange(len(A1)):
# Find all the records in A2 that match all the records in A1 at fields.
out_recs.append([])
val1 = []
i2_remaining = arange(len(A2))
for field in fields:
i2_remaining = i2_remaining[where(A2[field][i2_remaining]==A1[i1][field])[0]]
# Check for no matches or multiple matches and warn verbally if there are any.
if len(i2_remaining)>1:
print 'Warning, match underspecified ', i1,i2_remaining, len(i2_remaining)
elif len(i2_remaining)==0:
print 'Warning, no match ',i1
# Take one of the records in A2 that matches (if any) and fuse it to the current
# record in A1.
try:
i2 = i2_remaining[0]
for name in A1_names + both_names:
out_recs[i1].append(A1[i1][name])
for name in A2_names:
out_recs[i1].append(A2[i2][name])
except:
pass
A = rec.fromrecords(out_recs, names=A1_names + both_names + A2_names)
return A
def Read(self):
"""
Read the data from the file.
"""
def addkeyval(lst, k, v):
"""
add new key to a list. if key already exists a number will be
appended to the key name
Parameters
----------
lst : list
k : str
key
v : object
value
"""
kcnt = 0
key = k
while key in lst:
key = k + "_%i" % (kcnt + 1)
kcnt += 1
lst[key] = v
col_names = []
col_units = []
col_types = []
rec_list = []
with open(self.filename, 'rb') as fid:
for line in fid:
line = line.decode('utf8', 'ignore')
line = line.strip()
# read the next line if the line starts with a "!"
if re_end_section.match(line):
continue
# select the which section to read
if re_comment_section.match(line):
read_mode = 1
continue
if re_parameter_section.match(line):
read_mode = 2
continue
if re_data_section.match(line):
read_mode = 3
continue
# here we decide how to proceed with the data
if read_mode == 1:
# read the file comments
try:
(key, value) = line.split("=")
except ValueError:
# avoid annoying output
if config.VERBOSITY >= config.INFO_ALL:
print("XU.io.SPECTRAFile.Read: cannot interpret "
"the comment string: %s" % (line))
continue
key = key.strip()
# remove whitespaces to be conform with natural naming
key = key.replace(' ', '')
key = key.replace(':', '_')
# remove possible number at first position
if re_num.findall(key[0]) != []:
key = "_" + key
value = value.strip()
if config.VERBOSITY >= config.DEBUG:
print("XU.io.SPECTRAFile.Read: comment: k, v: %s, %s" %
(key, value))
try:
value = float(value)
except ValueError:
pass
# need to handle the case, that a key may appear several
# times in the list
addkeyval(self.comments, key, value)
elif read_mode == 2:
# read scan parameters
try:
(key, value) = line.split("=")
except ValueError:
print("XU.io.SPECTRAFile.Read: cannot interpret the "
"parameter string: %s" % (line))
key = key.strip()
# remove whitespaces to be conform with natural naming
key = key.replace(' ', '')
key = key.replace(':', '_')
# remove possible number at first position
if re_num.findall(key[0]) != []:
key = "_" + key
value = value.strip()
if config.VERBOSITY >= config.DEBUG:
print(
"XU.io.SPECTRAFile.Read: parameter: k, v: %s, %s" %
(key, value))
try:
value = float(value)
except ValueError:
# if the conversion of the parameter to float
# fails it will be saved as a string
pass
# need to handle the case, that a key may appear several
# times in the list
addkeyval(self.params, key, value)
elif read_mode == 3:
if re_column.match(line):
try:
unit = re_unit.findall(line)[0]
except IndexError:
unit = "NONE"
try:
sline = re_obracket.split(line)
if len(sline) == 1:
raise IndexError
lval = sline[0]
rval = re_cbracket.split(line)[-1]
dtype = rval.strip()
lv = re_wspaces.split(lval)
index = int(lv[1])
name = "".join(lv[2:])
name = name.replace(':', '_')
except IndexError:
lv = re_wspaces.split(line)
index = int(lv[1])
dtype = lv[-1]
name = "".join(lv[2:-1])
name = name.replace(':', '_')
# store column definition
self.data.append(
SPECTRAFileDataColumn(index, name, unit, dtype))
if name in col_names:
name += "%s_1" % name
col_names.append("%s" % name)
col_types.append("%s" % (dtype_map[dtype]))
else:
# read data
dlist = re_wspaces.split(line)
for i in range(len(dlist)):
dlist[i] = float(dlist[i])
rec_list.append(tuple(dlist))
if config.VERBOSITY >= config.DEBUG:
print("XU.io.SPECTRAFile.Read: data columns: name, type: %s, %s" %
(col_names, col_types))
if rec_list:
self.data.data = rec.fromrecords(rec_list,
formats=col_types,
names=col_names)
else:
self.data.data = None
def Read(self):
"""
Read the data from the file.
"""
def addkeyval(lst, k, v):
"""
add new key to a list. if key already exists a number will be
appended to the key name
Parameters
----------
lst : list
k : str
key
v : object
value
"""
kcnt = 0
key = k
while key in lst:
key = k + "_%i" % (kcnt + 1)
kcnt += 1
lst[key] = v
col_names = []
col_units = []
col_types = []
rec_list = []
with open(self.filename, 'rb') as fid:
for line in fid:
line = line.decode('utf8', 'ignore')
line = line.strip()
# read the next line if the line starts with a "!"
if re_end_section.match(line):
continue
# select the which section to read
if re_comment_section.match(line):
read_mode = 1
continue
if re_parameter_section.match(line):
read_mode = 2
continue
if re_data_section.match(line):
read_mode = 3
continue
# here we decide how to proceed with the data
if read_mode == 1:
# read the file comments
try:
(key, value) = line.split("=")
except ValueError:
# avoid annoying output
if config.VERBOSITY >= config.INFO_ALL:
print("XU.io.SPECTRAFile.Read: cannot interpret "
"the comment string: %s" % (line))
continue
key = key.strip()
# remove whitespaces to be conform with natural naming
key = key.replace(' ', '')
key = key.replace(':', '_')
# remove possible number at first position
if re_num.findall(key[0]) != []:
key = "_" + key
value = value.strip()
if config.VERBOSITY >= config.DEBUG:
print("XU.io.SPECTRAFile.Read: comment: k, v: %s, %s"
% (key, value))
try:
value = float(value)
except ValueError:
pass
# need to handle the case, that a key may appear several
# times in the list
addkeyval(self.comments, key, value)
elif read_mode == 2:
# read scan parameters
try:
(key, value) = line.split("=")
except ValueError:
print("XU.io.SPECTRAFile.Read: cannot interpret the "
"parameter string: %s" % (line))
key = key.strip()
# remove whitespaces to be conform with natural naming
key = key.replace(' ', '')
key = key.replace(':', '_')
# remove possible number at first position
if re_num.findall(key[0]) != []:
key = "_" + key
value = value.strip()
if config.VERBOSITY >= config.DEBUG:
print("XU.io.SPECTRAFile.Read: parameter: k, v: %s, %s"
% (key, value))
try:
value = float(value)
except ValueError:
# if the conversion of the parameter to float
# fails it will be saved as a string
pass
# need to handle the case, that a key may appear several
# times in the list
addkeyval(self.params, key, value)
elif read_mode == 3:
if re_column.match(line):
try:
unit = re_unit.findall(line)[0]
except IndexError:
unit = "NONE"
try:
sline = re_obracket.split(line)
if len(sline) == 1:
raise IndexError
lval = sline[0]
rval = re_cbracket.split(line)[-1]
dtype = rval.strip()
lv = re_wspaces.split(lval)
index = int(lv[1])
name = "".join(lv[2:])
name = name.replace(':', '_')
except IndexError:
lv = re_wspaces.split(line)
index = int(lv[1])
dtype = lv[-1]
name = "".join(lv[2:-1])
name = name.replace(':', '_')
# store column definition
self.data.append(
SPECTRAFileDataColumn(index, name, unit, dtype))
if name in col_names:
name += "%s_1" % name
col_names.append("%s" % name)
col_types.append("%s" % (dtype_map[dtype]))
else:
# read data
dlist = re_wspaces.split(line)
for i in range(len(dlist)):
dlist[i] = float(dlist[i])
rec_list.append(tuple(dlist))
if config.VERBOSITY >= config.DEBUG:
print("XU.io.SPECTRAFile.Read: data columns: name, type: %s, %s"
% (col_names, col_types))
if rec_list:
self.data.data = rec.fromrecords(rec_list, formats=col_types,
names=col_names)
else:
self.data.data = None
