def make_log(raw, condition=64): # pragma: no cover
import warnings
warnings.warn("This code is not tested!")
codes = read_raw(maybe_open(raw), np.float64, True)[3]
log = []
for i in codes.nonzero()[0]:
log.append(
struct.pack("<HHHBB", codes[i], (i & 0xffff0000) >> 16, i & 0xffff,
condition, 0))
if codes[i] in (PAUSE_CODE, DELETE_CODE):
condition += 1
return "".join(log)
def make_log(raw, condition=64): # pragma: no cover
import warnings; warnings.warn("This code is not tested!")
codes = read_raw(maybe_open(raw), np.float64, True)[3]
log = []
for i in codes.nonzero()[0]:
log.append(struct.pack("<HHHBB",
codes[i], (i & 0xffff0000) >> 16, i & 0xffff,
condition,
0))
if codes[i] in (PAUSE_CODE, DELETE_CODE):
condition += 1
return "".join(log)
def read_log(file_like):
fo = maybe_open(file_like)
ticks = []
events = []
while True:
event = fo.read(8)
if not event:
break
(code, tick_hi, tick_lo, condition, flag) \
= struct.unpack("<HHHBB", event)
ticks.append(tick_hi << 16 | tick_lo)
events.append((code, condition, flag))
df = pandas.DataFrame(events, columns=["code", "condition", "flag"],
index=ticks)
df["flag_data_error"] = np.asarray(df["flag"] & 0o100, dtype=bool)
df["flag_rejected"] = np.asarray(df["flag"] & 0o40, dtype=bool)
df["flag_polinv"] = np.asarray(df["flag"] & 0o20, dtype=bool)
return df
def read_log(file_like):
fo = maybe_open(file_like)
ticks = []
events = []
while True:
event = fo.read(8)
if not event:
break
(code, tick_hi, tick_lo, condition, flag) \
= struct.unpack("<HHHBB", event)
ticks.append(tick_hi << 16 | tick_lo)
events.append((code, condition, flag))
df = pandas.DataFrame(events,
columns=["code", "condition", "flag"],
index=ticks)
df["flag_data_error"] = np.asarray(df["flag"] & 0o100, dtype=bool)
df["flag_rejected"] = np.asarray(df["flag"] & 0o40, dtype=bool)
df["flag_polinv"] = np.asarray(df["flag"] & 0o20, dtype=bool)
return df
def load_erpss(raw,
log,
calibration_events="condition == 0",
lazy=True,
calibrate=False,
calibrate_half_width_ticks=5,
calibrate_low_cursor_time=None,
calibrate_high_cursor_time=None,
calibrate_pulse_size=None,
calibrate_polarity=1):
dtype = np.float64
metadata = {}
if isinstance(raw, basestring):
metadata["raw_file"] = os.path.abspath(raw)
if isinstance(log, basestring):
metadata["log_file"] = os.path.abspath(log)
metadata["calibration_events"] = str(calibration_events)
raw = maybe_open(raw)
log = maybe_open(log)
(fetcher, hz, channel_names, raw_codes, data,
header_metadata) = read_raw(raw, dtype, lazy)
metadata.update(header_metadata)
if calibrate:
units = "uV"
else:
units = "RAW"
data_format = DataFormat(hz, units, channel_names)
total_ticks = raw_codes.shape[0]
raw_log_events = read_log(log)
expanded_log_codes = np.zeros(raw_codes.shape, dtype=int)
try:
expanded_log_codes[raw_log_events.index] = raw_log_events["code"]
except IndexError as e:
raise ValueError("log file claims event at position where there is "
"no data: %s" % (e, ))
# Sometimes people "delete" events by setting the high (sign) bit of the
# code in the log file (e.g. with 'logpoke'). So we ignore this bit when
# comparing log codes to raw codes -- mismatches here do not indicate an
# error -- and then are careful to use the log codes, rather than the
# raw codes, below.
if np.any((expanded_log_codes & ~0x8000) != (raw_codes & ~0x8000)):
raise ValueError("raw and log files have mismatched codes")
del raw_codes
del expanded_log_codes
pause_events = (raw_log_events["code"] == PAUSE_CODE)
delete_events = (raw_log_events["code"] == DELETE_CODE)
break_events = pause_events | delete_events
break_ticks = raw_log_events.index[break_events]
# The pause/delete code appears at the last sample of the old era, so if
# used directly, adjacent pause ticks give contiguous spans of recording
# as (pause1, pause2]. (Confirmed by checking by hand in a real recording
# that the data associated with the sample that has the pause code is
# contiguous with the sample before, but not the sample after.) Adding +1
# to each of them then converts this to Python style [pause1, pause2)
# intervals. There is a pause code at the last record of the file, but not
# one at the first, so we add that in explicitly.
break_ticks += 1
span_edges = np.concatenate(([0], break_ticks))
assert span_edges[0] == 0
assert span_edges[-1] == total_ticks
span_slices = [
slice(span_edges[i], span_edges[i + 1])
for i in xrange(len(span_edges) - 1)
]
dataset = Dataset(data_format)
for span_slice in span_slices:
if lazy:
lr = LazyRecspan(fetcher, dtype, len(channel_names),
span_slice.start, span_slice.stop)
dataset.add_lazy_recspan(lr, span_slice.stop - span_slice.start,
metadata)
else:
dataset.add_recspan(data[span_slice, :], metadata)
span_starts = [s.start for s in span_slices]
recspan_ids = []
start_ticks = []
for tick in raw_log_events.index:
recspan_id = bisect.bisect(span_starts, tick) - 1
span_slice = span_slices[recspan_id]
span_start = span_slice.start
span_stop = span_slice.stop
assert span_start <= tick < span_stop
recspan_ids.append(recspan_id)
start_ticks.append(tick - span_start)
stop_ticks = [tick + 1 for tick in start_ticks]
dataset.add_events(recspan_ids, start_ticks, stop_ticks, raw_log_events)
for delete_event in dataset.events_query({"code": DELETE_CODE}):
delete_event.recspan_info["deleted"] = True
for cal_event in dataset.events_query(calibration_events):
for key in list(cal_event):
del cal_event[key]
cal_event["calibration_pulse"] = True
if calibrate:
for kwarg in [
"calibrate_low_cursor_time", "calibrate_high_cursor_time",
"calibrate_pulse_size"
]:
if locals()[kwarg] is None:
raise ValueError("when calibrating, %s= argument must be "
"specified" % (kwarg, ))
half_width = dataset.data_format.ticks_to_ms(
calibrate_half_width_ticks)
cal_vals = {}
for which, cursor_time in [("low", calibrate_low_cursor_time),
("high", calibrate_high_cursor_time)]:
# Round cursor to nearest tick
cursor_tick = dataset.data_format.ms_to_ticks(cursor_time)
cursor_time = dataset.data_format.ticks_to_ms(cursor_tick)
erp = dataset.rerp("calibration_pulse",
cursor_time - half_width,
cursor_time + half_width,
"1",
all_or_nothing=True,
overlap_correction=False,
verbose=False)
cal_vals[which] = erp.betas["Intercept"].mean()
cal_diffs = cal_vals["high"] - cal_vals["low"]
calibrate_pulse_size *= calibrate_polarity
# For each channel, we want to multiply by a factor with units uV/raw
# We have calibrate_pulse_size uV = cal_diffs raw
cal_scaler = calibrate_pulse_size / cal_diffs
dataset.transform(np.diagflat(np.asarray(cal_scaler)))
return dataset
def load_erpss(raw, log, calibration_events="condition == 0",
lazy=True,
calibrate=False,
calibrate_half_width_ticks=5,
calibrate_low_cursor_time=None,
calibrate_high_cursor_time=None,
calibrate_pulse_size=None,
calibrate_polarity=1):
dtype = np.float64
metadata = {}
if isinstance(raw, basestring):
metadata["raw_file"] = os.path.abspath(raw)
if isinstance(log, basestring):
metadata["log_file"] = os.path.abspath(log)
metadata["calibration_events"] = str(calibration_events)
raw = maybe_open(raw)
log = maybe_open(log)
(fetcher, hz, channel_names, raw_codes, data, header_metadata
) = read_raw(raw, dtype, lazy)
metadata.update(header_metadata)
if calibrate:
units = "uV"
else:
units = "RAW"
data_format = DataFormat(hz, units, channel_names)
total_ticks = raw_codes.shape[0]
raw_log_events = read_log(log)
expanded_log_codes = np.zeros(raw_codes.shape, dtype=int)
try:
expanded_log_codes[raw_log_events.index] = raw_log_events["code"]
except IndexError as e:
raise ValueError("log file claims event at position where there is "
"no data: %s" % (e,))
# Sometimes people "delete" events by setting the high (sign) bit of the
# code in the log file (e.g. with 'logpoke'). So we ignore this bit when
# comparing log codes to raw codes -- mismatches here do not indicate an
# error -- and then are careful to use the log codes, rather than the
# raw codes, below.
if np.any((expanded_log_codes & ~0x8000) != (raw_codes & ~0x8000)):
raise ValueError("raw and log files have mismatched codes")
del raw_codes
del expanded_log_codes
pause_events = (raw_log_events["code"] == PAUSE_CODE)
delete_events = (raw_log_events["code"] == DELETE_CODE)
break_events = pause_events | delete_events
break_ticks = raw_log_events.index[break_events]
# The pause/delete code appears at the last sample of the old era, so if
# used directly, adjacent pause ticks give contiguous spans of recording
# as (pause1, pause2]. (Confirmed by checking by hand in a real recording
# that the data associated with the sample that has the pause code is
# contiguous with the sample before, but not the sample after.) Adding +1
# to each of them then converts this to Python style [pause1, pause2)
# intervals. There is a pause code at the last record of the file, but not
# one at the first, so we add that in explicitly.
break_ticks += 1
span_edges = np.concatenate(([0], break_ticks))
assert span_edges[0] == 0
assert span_edges[-1] == total_ticks
span_slices = [slice(span_edges[i], span_edges[i + 1])
for i in xrange(len(span_edges) - 1)]
dataset = Dataset(data_format)
for span_slice in span_slices:
if lazy:
lr = LazyRecspan(fetcher, dtype, len(channel_names),
span_slice.start, span_slice.stop)
dataset.add_lazy_recspan(lr, span_slice.stop - span_slice.start,
metadata)
else:
dataset.add_recspan(data[span_slice, :], metadata)
span_starts = [s.start for s in span_slices]
recspan_ids = []
start_ticks = []
for tick in raw_log_events.index:
recspan_id = bisect.bisect(span_starts, tick) - 1
span_slice = span_slices[recspan_id]
span_start = span_slice.start
span_stop = span_slice.stop
assert span_start <= tick < span_stop
recspan_ids.append(recspan_id)
start_ticks.append(tick - span_start)
stop_ticks = [tick + 1 for tick in start_ticks]
dataset.add_events(recspan_ids, start_ticks, stop_ticks,
raw_log_events)
for delete_event in dataset.events_query({"code": DELETE_CODE}):
delete_event.recspan_info["deleted"] = True
for cal_event in dataset.events_query(calibration_events):
for key in list(cal_event):
del cal_event[key]
cal_event["calibration_pulse"] = True
if calibrate:
for kwarg in ["calibrate_low_cursor_time",
"calibrate_high_cursor_time",
"calibrate_pulse_size"]:
if locals()[kwarg] is None:
raise ValueError("when calibrating, %s= argument must be "
"specified" % (kwarg,))
half_width = dataset.data_format.ticks_to_ms(calibrate_half_width_ticks)
cal_vals = {}
for which, cursor_time in [("low", calibrate_low_cursor_time),
("high", calibrate_high_cursor_time)]:
# Round cursor to nearest tick
cursor_tick = dataset.data_format.ms_to_ticks(cursor_time)
cursor_time = dataset.data_format.ticks_to_ms(cursor_tick)
erp = dataset.rerp("calibration_pulse",
cursor_time - half_width,
cursor_time + half_width,
"1",
all_or_nothing=True,
overlap_correction=False,
verbose=False)
cal_vals[which] = erp.betas["Intercept"].mean()
cal_diffs = cal_vals["high"] - cal_vals["low"]
calibrate_pulse_size *= calibrate_polarity
# For each channel, we want to multiply by a factor with units uV/raw
# We have calibrate_pulse_size uV = cal_diffs raw
cal_scaler = calibrate_pulse_size / cal_diffs
dataset.transform(np.diagflat(np.asarray(cal_scaler)))
return dataset