def make_meta_table(self, samprate, nchannels, fname, verbose=False):
"""Make the metadata table for this event store, specifying the sampling rate, number of channels, and full name of the raw data source."""
ret = True
self.sample_rate = samprate
self.nchannels = nchannels
self.recording = fname
self.schema.create_table(self, 'meta')
self.execute('SELECT * from meta;')
result = self.fetchone()
iprint("{}".format(result))
if result is None:
self.execute('INSERT INTO meta values ({}, {}, "{}");'.format(
samprate, nchannels, fname))
ret = True
else:
if result[0] != samprate or result[1] != nchannels or result[
2] != fname:
eprint(
"You supplied arguments that are inconsistent with this db file!"
)
eprint("db file says: {}".format(result))
eprint("args say: {} {} {}".format(samprate, nchannels, fname))
ret = False
else:
if verbose:
iprint("Valid db file for these arguments.")
ret = True
return ret
def check_orth(seq, tol=1.0e-04, verbose=True):
"""Check the orthogonality of the basis sequence *seq*."""
orth = True
if verbose: iprint('[I] Orthonormality check:')
i = 0
rowvec = np.zeros(len(seq))
for s1 in seq:
if verbose:
with ui.context(ui.colors.OKGREEN) as c:
print('[I] row {}:'.format(i), end="")
j = 0
for s2 in seq:
v = np.dot(s1, s2)
rowvec[j] = v
if i == j and abs(v - 1.0) > tol:
orth = False
if i != j and abs(v) > tol:
orth = False
j += 1
if verbose:
with ui.context(ui.colors.OKGREEN) as c:
for v in rowvec:
print("{0:5.2f} ".format(v), end="")
print(' ')
i += 1
if verbose:
if orth: iprint("Basis is orthonormal.")
else: eprint("Basis is NOT orthonormal")
return orth
def get_chunk(self, channel, start, end):
"""Returns the portion of the time series for the specified channel that is between start and end inclusive (time)."""
i0 = max(0, int(start - self.t0))
i1 = max(0, int(end - self.t0))
# We should probably call numpy.asarray to coerce this to a pure array:
# r = np.asarray(x[i0:i1])
if self.lazy:
x = self.raw[channel]
r = x[i0:i1].reshape(i1 - i0)
result = self.dsf.maybe_filter(r)
elif NEW_FILTERING:
x = self.get_filtered_data(channel)
else:
# If we miss with filtered_data, run the filter and add it
# to the filtered_data dictionary. This is where we could
# try filesystem caching:
try:
x = self.filtered_data[channel]
except KeyError:
x = self.raw[channel]
iprint("In get_chunk({},{},{},{}): Initializing filtered copy".
format(self, channel, start, end))
#self.get_filtered_data(x, channel)
self.filtered_data[channel] = self.dsf.maybe_filter(x)
x = self.filtered_data[channel]
try:
result = x[i0:i1].reshape(i1 - i0)
except:
# Most likely, we reached an edge, so just return a 0 vector:
result = np.zeros((i1 - i0))
# result = result.reshape(len(r))
return result
def __init__(self, filename):
self.schema = schema()
self.conn = s3.connect(filename)
self.cur = self.conn.cursor()
self.filename = filename
self.sample_rate = 0
self.nspikes = 0
self.nchannels = 0
self.recording = ''
self.bases = {} # Cache the basis functions as needed
self.basis_cache = {}
self.coefs_cache = {}
# Waveform min / max - these next three are examples of
# properties that should be cached within the database itself,
# somehow:
self.modified = False
self.min = None
self.max = None
try:
self.cur.execute('select * from meta;')
result = self.cur.fetchone()
self.sample_rate = result[0]
self.nchannels = result[1]
self.recording = result[2]
self.cur.execute('pragma foreign_keys=ON;')
except:
iprint("Uninitialized db.")
def filter_data(self, low_cutoff, high_cutoff, order=6):
# Only sets the filter parameters. Does not actually filter data yet.
self.lo = low_cutoff
self.hi = high_cutoff
self.order = order
# nyquist = 0.5 * self.samprate
iprint('Nyquist freq. is {}.'.format(self.samprate/2))
if self.lo is None:
if self.hi is None:
self.do_filter = False
iprint('No filtering.')
else:
f = 2 * self.hi / self.samprate
self.butter_info = butter(self.order, f, btype='lowpass')
iprint('Low-pass filtering (f<{} Hz).'.format(self.hi))
self.do_filter = True
else:
if self.hi is None:
f = 2 * self.lo / self.samprate
self.butter_info = butter(self.order, f, btype='highpass')
iprint('High-pass filtering (f>{} Hz).'.format(self.lo))
self.do_filter = True
else:
f0 = 2 * self.lo / self.samprate
f1 = 2 * self.hi / self.samprate
self.butter_info = butter(self.order, (f0,f1), btype='bandpass')
iprint('Bandpass filtering [{}, {}].'.format(self.lo, self.hi))
self.do_filter = True
def floats_to_nbf(dirname,
data,
sample_rate,
start,
end,
events=None,
relative=True,
source=None,
permute=None):
"""Converts an array of floats (data) into NBF form in the specified
dirname, with the sample rate, start and end times given by the
arguments. If supplied, 'events' is a list of time,event pairs."""
name = os.path.basename(dirname)
data_shape = data.shape
nchan = data_shape[0]
npts = data_shape[1]
md = nb_metadata(dirname,
sample_rate,
nchan,
start,
end,
relative,
date=None)
filename = md.filename
# If the directory doesn't exist, create it:
if not os.path.isdir(dirname):
os.makedirs(dirname)
if events:
eventfile = os.path.join(dirname, 'events.dat')
iprint("Events found:")
with open(eventfile, 'w') as f:
for (ts, label) in events:
iprint("{}: {}".format(label, ts))
if label[-1] != '\n':
f.write('{}:{}\n'.format(ts, label))
else:
f.write('{}:{}'.format(ts, label))
md.save(source)
# Initialize the raw data file:
rawfile = md.rawfile()
iprint('Mapping raw file {} using shape {}.'.format(rawfile, data_shape))
raw = np.memmap(rawfile, np.dtype('f4'), 'w+', shape=data_shape)
# print(raw)
iprint('Initializing raw file: {}'.format(rawfile))
if permute is None:
rawchan = [k for k in range(nchan)]
else:
rawchan = permute
for j in range(nchan):
k = rawchan[j]
raw[j][:] = data[k][:]
iprint('Wrote {} data elements'.format(len(data[k])))
# print(raw)
del raw
def read_nbm(filename):
"""Reads NBF metadata, which is usually saved in an 'nbm' file. This
file contains basic information about the recording."""
attrs = {}
iprint('Reading nbm file {}.'.format(filename))
with open(filename) as f:
for line in f:
x = line[0:-1].split('=') # No spaces allowed please!!
if len(x) == 2:
attrs[x[0]] = x[1]
iprint('Attribute: {} = {}'.format(x[0], x[1]))
else:
wprint("Don't know what to do with this line: {}".format(line))
try:
date = attrs['date']
except:
date = None
if attrs['dirname'][0] == '.':
new_dirname = os.path.dirname(os.path.abspath(filename))
attrs['dirname'] = new_dirname
metadata = nb_metadata(attrs['dirname'],
sample_rate=float(attrs['sample_rate']),
nchannels=int(attrs['nchannels']),
start=int(attrs['start']),
end=int(attrs['end']),
date=date,
attributes=attrs)
return metadata
def parse_uri(uri, filename=None):
true_uri, rel = resolve_relative_paths(uri, filename)
if rel and filename is None:
wprint('URI path is relative, but no base filename is specified.')
else:
uri = true_uri
iprint('URI: {}'.format(uri))
desc = urisplit(uri)
return desc, rel
def max_spike_id(self):
self.cur.execute('select max(spikeID) from spiketimes;')
x = self.cur.fetchone()
iprint('max_spike_id() = {}'.format(x))
n = x[0]
if n is None:
return -1
else:
return n
def create_table(self, es, name):
es.execute('pragma foreign_key=ON;')
try:
es.execute(self.defs[name])
if self.verbose:
iprint('Created table {}.'.format(name))
except Exception as e:
if self.verbose:
wprint('Could not create table {}. Error: {}'.format(name,e))
def maybe_fix_filenames(self, dirname):
old = os.path.dirname(self.filename)
iprint('maybe_fix_filenames: dirname={}'.format(dirname))
iprint('maybe_fix_filenames: old dirname={}'.format(old))
if dirname != old:
wprint('Directory was moved. Resetting...')
name = os.path.basename(dirname)
self.filename = os.path.join(dirname, '{}.nbm'.format(name))
self.rawfile = os.path.join(dirname, '{}.raw'.format(name))
self.meanfile = os.path.join(dirname, '{}-mean.pyr'.format(name))
self.minfile = os.path.join(dirname, '{}-min.pyr'.format(name))
self.maxfile = os.path.join(dirname, '{}-max.pyr'.format(name))
def spike_peaks(sig, threshold):
"""Given an LFP signal *sig* and a *threshold*, isolate all peaks that are above threshold, by finding the zero-crossings of the first derivative of the smoothed signal. Returns a vector of 1s and 0s such that the 1s identify spikes."""
upvec = sig > threshold
iprint('Peak vector computed.')
sys.stdout.flush()
smoothed = smooth_lfp(sig, 31, 6.0)
mean = smoothed.mean()
iprint('Smoothed signal computed')
sys.stdout.flush()
first = deriv(smoothed)
iprint('First derivative computed')
sys.stdout.flush()
zc = np.zeros(len(sig))
numzc = 0
numpts = len(zc)
tick = numpts / 100
for i in range(1, len(zc) - 1):
if (upvec[i]):
if (first[i] == 0 or first[i] * first[i - 1] < 0):
zc[i] = sig[i]
numzc += 1
if (i % tick == 0):
print('{}% ({})'.format(i / tick, numzc), end="")
sys.stdout.flush()
print(' ')
iprint('Zero-crossings of first derivative computed.')
sys.stdout.flush()
return zc
def map_filtered_data(self):
"""Maps filtered data files into memory for direct access to filtered data."""
dsf = self.dsf
dir = self.dirname
if len(self.filtered_data) == 0:
fdatafile = os.path.join(
dir, 'filtered_{}_{}.raw'.format(dsf.lo, dsf.hi))
iprint('Checking for filtered data file: {}'.format(fdatafile))
if not os.path.exists(fdatafile):
self.make_filtered_data()
iprint('Mapping filtered data file with shape {}...'.format(
self.data_shape))
self.filtered_data = np.memmap(fdatafile,
np.dtype('f4'),
'r+',
shape=self.data_shape)
def make_filtered_data(self, force=False):
"""Makes a cache of the filtered data, with filter parameters specified by the dsf object."""
dsf = self.dsf
dir = self.dirname
fdatafile = os.path.join(dir,
'filtered_{}_{}.raw'.format(dsf.lo, dsf.hi))
if force or not os.path.exists(fdatafile):
iprint('Creating raw file {} using shape {}.'.format(
fdatafile, self.data_shape))
fdata = np.memmap(fdatafile,
np.dtype('f4'),
'w+',
shape=self.data_shape)
iprint('Initializing filtered data file: {}'.format(fdatafile))
for k in range(self.data_shape[0]):
data = dsf.maybe_filter(self.raw[k])
fdata[k, :] = data[:]
# print(raw)
del fdata
def wfdb_to_nbf(filename,
to_dir,
channel_names=['V1', 'V2', 'V3', 'V4'],
date=None):
iprint("Creating nbf from file: {}".format(filename))
data, fields = wfdb.rdsamp(filename)
print("Available fields: {}".format(fields))
# end sample number:
start = 0
end = fields['sig_len']
# I think this is the sample rate:
sample_rate = fields['fs']
# This will barf if the channel_names don't exist in the file, but
# collect the indices of the channels that we are interested in
# (the Vn leads on the chest, for now):
indices = [fields['sig_name'].index(x) for x in channel_names]
nchan = len(indices)
md = nb_metadata(to_dir, sample_rate, nchan, start, end, True, date=date)
nbf_filename = md.filename
# If the directory doesn't exist, create it:
if not os.path.isdir(to_dir):
os.makedirs(to_dir)
# See floats_to_nbf for hints on Events. For now, we aren't
# looking for events, but this is where we would create an
# events.dat file.
# Save the metadata file:
md.save(source=filename)
# Create the raw data file:
rawfile = md.rawfile()
iprint('Mapping raw file {} using shape {}.'.format(rawfile, (nchan, end)))
raw = np.memmap(rawfile, np.dtype('f4'), 'w+', shape=(nchan, end))
# print(raw)
iprint('Initializing raw file: {}'.format(rawfile))
for j in range(nchan):
k = indices[j]
raw[j][:] = data[:, k]
iprint('Wrote {} data elements'.format(len(data[:, k])))
# print(raw)
del raw
def add_basis(self,
basis_id,
sigma,
width,
nfuncs=5,
offset=0,
name="gaussian_basis"):
"""Adds a Gaussian basis with the specified sigma and width in
SAMPLES to the database, with the desired ID."""
ret = True
self.cur.execute(
'SELECT * from basis where basisID == {};'.format(basis_id))
result = self.cur.fetchall()
iprint("{}".format(result))
if result is None or len(result) == 0:
self.cur.execute(
'INSERT INTO basis values ({}, {}, {}, {}, {}, "{}");'.format(
basis_id, offset, sigma, width, nfuncs, name))
self.conn.commit()
else:
result = result[0]
if math.fabs(result[1] - sigma) > 1.0e-06 or result[
2] != width or result[3] != nfuncs:
wprint(
"Basis ID {} is already in use with these parameters: sigma={}, window width={}, n={}"
.format(result[0], result[1], result[2], result[3]))
iprint("Here are the basis ID's in use in this file:")
self.cur.execute('SELECT * from basis;')
result = self.cur.fetchall()
iprint("BasisID offset sigma window nfuncs name")
for tuple in result:
iprint("{} {} {} {} {} {}".format(
tuple[0], tuple[1], tuple[2], tuple[3], tuple[4],
tuple[5]))
ret = False
else:
iprint(
"Your basis set is already available in {}.".format(self))
ret = True
return ret
def histogram(vector, nbins=100, min=None, max=None, clip=True):
"""Given a vector, returns a histogram vector along with metadata
about binning."""
if min == None:
min = vector.min()
if max == None:
max = vector.max()
dx = (max - min) / nbins
iprint("# min={}, max={}, dx={}".format(min, max, dx))
kmax = nbins - 1
hist = np.zeros(nbins)
for val in vector:
k = int((val - min) / dx)
if k < 0:
if not clip:
hist[0] += 1
elif k > kmax:
if not clip:
hist[kmax] += 1
else:
hist[k] += 1
return hist
def open(uri, filename=None):
"""Given a URI for a data source, open it and return the appropriate data source object."""
if True:
desc, rel = parse_uri(uri, filename)
else:
new_uri, rel = resolve_relative_paths(uri, filename)
if rel and filename is None:
wprint('Data store path is relative, but no event store is specified.')
else:
uri = new_uri
desc = urisplit(uri)
iprint('Opening data store at {}'.format(uriunsplit(desc)))
s = desc.scheme
if s is None:
return ndk.ds.neo_in.spike2(desc)
elif s == 'pre':
return ndk.ds.pre.pre_ds(desc)
elif s == 'smr':
return ndk.ds.neo_in.spike2(desc)
# elif s == 'file':
# return ndk.ds.neo.neo_in(desc)
elif s == 'cass':
return ndk.ds.cass.cdb(desc)
elif s == 'wav':
return ndk.ds.wav.wav_ds(desc.path)
elif s == 'ds':
return ndk.ds.mmap.mmap_ds(desc.path)
elif s == 'edf':
return ndk.ds.edf_ds(desc)
elif s == 'nbm':
iprint('NBF path: {}'.format(desc.path))
if rel and desc.path[0] == '/' and desc.path[1] == '.':
return ndk.ds.nbf(desc.path[1:]) # Hack!
else:
return ndk.ds.nbf(desc.path)
else:
print("Don't know what do with this URI scheme: {} (in {})".format(s, uri))
def map_raw_data(self, version):
"""Maps raw data files into memory for direct access to data."""
rawfile = self.md.rawfile()
vmax = get_version_count(rawfile)
iprint('Versions found: {}'.format(vmax))
if version is not None and version <= vmax:
self.version = version
iprint('Selecting version {}.'.format(self.version))
rawfile = rawfile + '.{}'.format(version)
iprint('Checking for raw file: {}'.format(rawfile))
if not os.path.exists(rawfile):
iprint('Creating raw file...')
arr = np.zeros(self.data_shape, np.dtype('f4'))
with open(rawfile, 'wb') as f:
f.write(arr)
iprint('Mapping raw file with shape {}...'.format(self.data_shape))
self.raw = np.memmap(rawfile,
np.dtype('f4'),
'r+',
shape=self.data_shape)
def get_basis(self, basis_id):
"""Returns the set of basis functions corresponding to 'basis_id'."""
try:
seq = self.bases[basis_id]
except:
self.cur.execute(
'SELECT * from basis where basisID == {};'.format(basis_id))
result = self.cur.fetchall()
# print(result)
if len(result) < 1:
return None
# This is the old default - older db files might not have
# this column:
nfuncs = 5
if len(result[0]) > 3:
nfuncs = int(result[0][4])
rate = self.sample_rate
if USE_MS:
# Convert to seconds, then to sample numbers:
offset = result[0][1]
sigma = result[0][2] * 0.001 * rate
iprint('rate = {}; result[0][2] = {}'.format(
rate, result[0][2]))
width = int(result[0][3] * 0.001 * rate)
iprint('# sigma={}, width={}'.format(sigma, width))
else:
offset = result[0][1]
sigma = result[0][2]
width = result[0][3]
# If no generator is supplied, then the functions table
# should contain explicit values for the function basis.
# Regardless, we derive the window size in samples and the
# number of basis functions from the basis table:
if result[0][5] == '':
seq = self.get_functions(basis_id, nfuncs, width)
else:
if width <= 0:
print(
"RECOVERABLE ERROR: window width for basis {} is 0 samples!"
.format(basis_id))
print(
" Setting width = 32 samples and sigma = 16 samples!"
)
width = 32
sigma = 16
iprint("# Window width = {}".format(width))
gen_name = 'gaussian_basis'
if len(result[0]) > 5:
gen_name = result[0][5]
gen_fn = self.schema.basis_generator(gen_name)
seq = gen_fn(sigma, width, nfuncs)
self.bases[basis_id] = seq
return seq
def __init__(self,
uri,
start=0,
end=0,
sample_rate=32000.0,
nchannels=1,
version=None):
iprint("Creating nbf from URI: {}".format(uri))
# This is sloppy. In this implementation, the URI scheme is
# optional. If the scheme is not present, then the URI is
# treated as a filename.
if True:
self.desc = ndk.ds.parse_uri(uri)
else:
if isinstance(uri, str) or isinstance(uri, unicode):
self.desc = urisplit(uri)
else:
self.desc = uri
self.filename = os.path.abspath(uri)
self.filtered_data = {}
iprint("Set up filtered_data list: {}".format(self.filtered_data))
self.lazy = False
dirname = os.path.dirname(self.filename)
self.dirname = dirname
iprint('Using directory {}'.format(dirname))
# These are defaults, can be overridden later:
name = os.path.basename(self.filename)
name = name.split('.')[0]
new = False
# If the directory doesn't exist, create it:
if not os.path.isdir(dirname):
os.makedirs(dirname)
# Compute the constituent filenames:
self.mdfile = os.path.join(dirname, '{}.nbm'.format(name))
efile = os.path.join(dirname, 'events.dat')
def getfirst(y):
return y[0]
# If there are any events, generate the list:
self.events = []
if os.path.exists(efile):
print("Events found:")
with open(efile, 'r') as f:
for line in f:
x = line.split(':')
elabel = x[1].split('\n')[0]
print("{}: {}".format(elabel, x[0]))
self.events.append((int(x[0]), elabel))
self.events.sort(key=getfirst)
# How? Each channel is a "strip" of consecutive samples at a rate of
# sample_rate * 2^(-i) samples per second.
# Thus, the first index is the channel, the second is sample.
# This will load metadata, but otherwise will create it with
# defaults:
iprint('Checking for metadata file {}'.format(self.mdfile))
if not os.path.exists(self.mdfile):
dirname = os.path.dirname(os.path.abspath(self.mdfile))
self.md = nb_metadata(dirname,
sample_rate=sample_rate,
start=start,
end=end,
nchannels=nchannels)
elif NEW_NBM:
self.md = read_nbm(self.mdfile)
else:
with open(self.mdfile, 'r') as f:
self.md = pickle.load(f)
self.md.maybe_fix_filenames(dirname)
self.dsf = dsfilter(self.md.samprate)
self.data_shape = (self.md.nchannels, self.md.npts())
iprint('Initializing nbf object with data shape {}.'.format(
self.data_shape))
self.t0 = self.md.start
self.t1 = self.md.end
self.version = None
self.map_raw_data(version)
def old(rawfile, data, nchan):
with open(rawfile, 'wb') as f:
for k in range(nchan):
f.write(data[k])
iprint('Wrote {} data elements'.format(len(data[k])))
