def datenrich(dat, channels, out, label, window):
# load and reshape dat file
data, params = load_dat(dat)
rate = params["sampling_rate"]
nchannels = params["n_channels"]
if not channels:
channels = np.arange(nchannels) # select all channels
rootname = os.path.splitext(dat)[0]
if not out:
out = rootname + "_enr.dat"
# cut out labelled segments
segs, newlabels = get_segments(label, window)
# convert to samples
segs = np.array(segs * rate, dtype=int)
n_samples = sum((b - a for a, b in segs))
outparams = params.copy()
outparams["n_samples"] = n_samples
outparams["n_channels"] = len(channels)
#write to new file
with open(out, "wb") as outfp:
for start, stop in segs:
outfp.write(data[start:stop, channels].tobytes())
write_metadata(out, **outparams)
newlabels.to_csv(os.path.splitext(out)[0] + ".label",
header=True,
index=False)
if os.path.isfile(label + ".meta"):
shutil.copyfile(label + ".meta",
os.path.splitext(out)[0] + ".label.meta")
def datfilt(dat, channels, out, order, highpass, lowpass, filttype):
params = read_metadata(dat)
dtype = params["dtype"]
nchannels = params["n_channels"]
rate = params["sampling_rate"]
if highpass:
params["highpass"] = highpass
if lowpass:
params["lowpass"] = lowpass
params["filter_order"] = order
if not channels:
channels = np.arange(nchannels) # select all channels
params["filter_channels"] = channels
if not out:
out = dat + "_filt.dat"
# load and reshape dat file
data = np.memmap(dat, dtype=dtype, mode="r").reshape(-1, nchannels)
if filttype == "butter":
fil = butter
elif filttype == "bessel":
fil = bessel
else:
raise Exception("filter must be 'butter' or 'bessel'")
if highpass and not lowpass:
coefs = [fil(order, highpass /
(rate / 2.), btype="highpass")] * nchannels
elif lowpass and not highpass:
coefs = [fil(order, lowpass /
(rate / 2.), btype="lowpass")] * nchannels
elif lowpass and highpass:
coefs = [
fil(order,
np.array((highpass, lowpass)) / (rate / 2.),
btype="bandpass")
] * nchannels
else:
raise Exception("must set either '--lowpass' or '--highpass'")
states = [lfiltic(c[0], c[1], [0]) for c in coefs]
copyfile(dat, out) # make a copy of the data to write over
outdat = np.memmap(out, dtype=dtype, mode="r+", shape=data.shape)
for i in range(0, len(data), BUFFER_SIZE):
for c in channels:
buffer = data[i:i + BUFFER_SIZE, c]
outdat[i:i + BUFFER_SIZE, c], states[c] = lfilter(coefs[c][0],
coefs[c][1],
buffer,
zi=states[c])
# run filter backwards (zero phase)
for i in list(range(0, len(data), BUFFER_SIZE))[::-1]:
for c in channels:
buffer = data[i:i + BUFFER_SIZE, c][::-1]
newbuffer, states[c] = lfilter(coefs[c][0],
coefs[c][1],
buffer,
zi=states[c])
outdat[i:i + BUFFER_SIZE, c] = newbuffer[::-1]
write_metadata(out, **params)
def datsplit(datfile, chan_select=(), outfile=None):
params = read_metadata(datfile)
nchannels = params.pop("n_channels")
dtype = params.pop("dtype")
if not chan_select:
chan_select = range(nchannels)
if outfile is None:
outfile = datfile + "_".join([str(x) for x in chan_select]) + ".dat"
nbuf = nchannels * BUFFER_SIZE
data = np.memmap(datfile, dtype=dtype, mode="r")
with open(outfile, "wb") as outfp:
for i in range(0, len(data), nbuf):
buffer = data[i:i + nbuf]
mask = np.zeros_like(buffer, dtype=np.bool).reshape(-1, nchannels)
mask[:, chan_select] = True
outfp.write(buffer[mask.ravel()].tobytes())
write_metadata(outfile, n_channels=len(chan_select), dtype=dtype, **params)
def datmerge(datfiles, outfile):
paramsets = [read_metadata(x) for x in datfiles]
for p in paramsets:
assert p["dtype"] == paramsets[0]["dtype"]
assert p["sampling_rate"] == paramsets[0]["sampling_rate"]
params = paramsets[0].copy()
params["n_channels"] = sum([p["n_channels"] for p in paramsets])
datasets = [
np.memmap(x, dtype=p["dtype"], mode="r").reshape(-1, p["n_channels"])
for x, p in zip(datfiles, paramsets)
]
n_samples = datasets[0].shape[0]
for d in datasets:
assert d.shape[0] == n_samples
out = np.memmap(outfile,
dtype=params["dtype"],
mode="w+",
shape=(n_samples, params["n_channels"]))
for i in range(0, len(out), BUFFER_SIZE):
out[i:i + BUFFER_SIZE, :] = np.column_stack(
(x[i:i + BUFFER_SIZE, :] for x in datasets))
write_metadata(outfile, **params)
def datmeta(datfiles):
sr = False
while not sr:
try:
print("sampling rate:")
isr = float(input())
assert isr > 0
sr = isr
except:
pass
n_channels = False
while not n_channels:
try:
print("number of channels:")
in_channels = int(input())
assert in_channels > 0
n_channels = in_channels
except:
pass
dtype = False
while not dtype:
try:
print("datatype, probably int16 or float64:")
idtype = input()
assert idtype in ('int8', 'int16', 'int32', 'int64', 'float32',
'float64', 'float128', 'uint8', 'uint16',
'uint32', 'uint64')
dtype = idtype
except:
pass
for datfile in datfiles:
write_metadata(datfile,
n_channels=n_channels,
sampling_rate=sr,
dtype=dtype)
def datref(datfile, outfile):
data, params = load_dat(datfile)
outparams = params.copy()
out = create_dat(outfile, outparams)
# determine reference coefficient
n_channels = params["n_channels"]
coefs = np.zeros((n_channels, len(range(0, len(out), BUF))))
power = np.zeros_like(coefs)
b, a = butter(3, 500 / (params["sampling_rate"] / 2), btype="lowpass")
for ith, i in enumerate(range(0, len(out), BUF)):
for c in range(n_channels):
refs = np.delete(data[i:i + BUF, :], c, axis=1) # remove col c
#refs = data[i:i+BUF,:]
ref = np.mean(refs, axis=1)
#ref = lfilter(b, a, ref)
x = data[i:i + BUF, c]
#x = lfilter(b, a, x)
coefs[c, ith] = np.dot(x, ref) / np.dot(ref, ref)
best_C = np.zeros(n_channels)
for c in range(n_channels):
c_coefs = coefs[c, :]
c_power = power[c, :]
mask = c_power >= np.percentile(c_power, 90)
best_C[c] = np.mean(c_coefs[mask])
print("best reference coefficients: {}".format(best_C))
outparams["reference_coefficients"] = best_C.tolist()
for i in range(0, len(out), BUF):
for c in range(n_channels):
refs = np.delete(data[i:i + BUF, :], c, axis=1) # remove col c
#refs = data[i:i+BUF,:]
ref = np.mean(refs, axis=1)
x = data[i:i + BUF, c]
out[i:i + BUF,
c] = data[i:i + BUF, c] - best_C[c] * np.median(refs, axis=1)
write_metadata(outfile, **outparams)