def make_meta_table(self, samprate, nchannels, fname, verbose=False): """Make the metadata table for this event store, specifying the sampling rate, number of channels, and full name of the raw data source.""" ret = True self.sample_rate = samprate self.nchannels = nchannels self.recording = fname self.schema.create_table(self, 'meta') self.execute('SELECT * from meta;') result = self.fetchone() iprint("{}".format(result)) if result is None: self.execute('INSERT INTO meta values ({}, {}, "{}");'.format( samprate, nchannels, fname)) ret = True else: if result[0] != samprate or result[1] != nchannels or result[ 2] != fname: eprint( "You supplied arguments that are inconsistent with this db file!" ) eprint("db file says: {}".format(result)) eprint("args say: {} {} {}".format(samprate, nchannels, fname)) ret = False else: if verbose: iprint("Valid db file for these arguments.") ret = True return ret
def check_orth(seq, tol=1.0e-04, verbose=True): """Check the orthogonality of the basis sequence *seq*.""" orth = True if verbose: iprint('[I] Orthonormality check:') i = 0 rowvec = np.zeros(len(seq)) for s1 in seq: if verbose: with ui.context(ui.colors.OKGREEN) as c: print('[I] row {}:'.format(i), end="") j = 0 for s2 in seq: v = np.dot(s1, s2) rowvec[j] = v if i == j and abs(v - 1.0) > tol: orth = False if i != j and abs(v) > tol: orth = False j += 1 if verbose: with ui.context(ui.colors.OKGREEN) as c: for v in rowvec: print("{0:5.2f} ".format(v), end="") print(' ') i += 1 if verbose: if orth: iprint("Basis is orthonormal.") else: eprint("Basis is NOT orthonormal") return orth
def get_chunk(self, channel, start, end): """Returns the portion of the time series for the specified channel that is between start and end inclusive (time).""" i0 = max(0, int(start - self.t0)) i1 = max(0, int(end - self.t0)) # We should probably call numpy.asarray to coerce this to a pure array: # r = np.asarray(x[i0:i1]) if self.lazy: x = self.raw[channel] r = x[i0:i1].reshape(i1 - i0) result = self.dsf.maybe_filter(r) elif NEW_FILTERING: x = self.get_filtered_data(channel) else: # If we miss with filtered_data, run the filter and add it # to the filtered_data dictionary. This is where we could # try filesystem caching: try: x = self.filtered_data[channel] except KeyError: x = self.raw[channel] iprint("In get_chunk({},{},{},{}): Initializing filtered copy". format(self, channel, start, end)) #self.get_filtered_data(x, channel) self.filtered_data[channel] = self.dsf.maybe_filter(x) x = self.filtered_data[channel] try: result = x[i0:i1].reshape(i1 - i0) except: # Most likely, we reached an edge, so just return a 0 vector: result = np.zeros((i1 - i0)) # result = result.reshape(len(r)) return result
def __init__(self, filename): self.schema = schema() self.conn = s3.connect(filename) self.cur = self.conn.cursor() self.filename = filename self.sample_rate = 0 self.nspikes = 0 self.nchannels = 0 self.recording = '' self.bases = {} # Cache the basis functions as needed self.basis_cache = {} self.coefs_cache = {} # Waveform min / max - these next three are examples of # properties that should be cached within the database itself, # somehow: self.modified = False self.min = None self.max = None try: self.cur.execute('select * from meta;') result = self.cur.fetchone() self.sample_rate = result[0] self.nchannels = result[1] self.recording = result[2] self.cur.execute('pragma foreign_keys=ON;') except: iprint("Uninitialized db.")
def filter_data(self, low_cutoff, high_cutoff, order=6): # Only sets the filter parameters. Does not actually filter data yet. self.lo = low_cutoff self.hi = high_cutoff self.order = order # nyquist = 0.5 * self.samprate iprint('Nyquist freq. is {}.'.format(self.samprate/2)) if self.lo is None: if self.hi is None: self.do_filter = False iprint('No filtering.') else: f = 2 * self.hi / self.samprate self.butter_info = butter(self.order, f, btype='lowpass') iprint('Low-pass filtering (f<{} Hz).'.format(self.hi)) self.do_filter = True else: if self.hi is None: f = 2 * self.lo / self.samprate self.butter_info = butter(self.order, f, btype='highpass') iprint('High-pass filtering (f>{} Hz).'.format(self.lo)) self.do_filter = True else: f0 = 2 * self.lo / self.samprate f1 = 2 * self.hi / self.samprate self.butter_info = butter(self.order, (f0,f1), btype='bandpass') iprint('Bandpass filtering [{}, {}].'.format(self.lo, self.hi)) self.do_filter = True
def floats_to_nbf(dirname, data, sample_rate, start, end, events=None, relative=True, source=None, permute=None): """Converts an array of floats (data) into NBF form in the specified dirname, with the sample rate, start and end times given by the arguments. If supplied, 'events' is a list of time,event pairs.""" name = os.path.basename(dirname) data_shape = data.shape nchan = data_shape[0] npts = data_shape[1] md = nb_metadata(dirname, sample_rate, nchan, start, end, relative, date=None) filename = md.filename # If the directory doesn't exist, create it: if not os.path.isdir(dirname): os.makedirs(dirname) if events: eventfile = os.path.join(dirname, 'events.dat') iprint("Events found:") with open(eventfile, 'w') as f: for (ts, label) in events: iprint("{}: {}".format(label, ts)) if label[-1] != '\n': f.write('{}:{}\n'.format(ts, label)) else: f.write('{}:{}'.format(ts, label)) md.save(source) # Initialize the raw data file: rawfile = md.rawfile() iprint('Mapping raw file {} using shape {}.'.format(rawfile, data_shape)) raw = np.memmap(rawfile, np.dtype('f4'), 'w+', shape=data_shape) # print(raw) iprint('Initializing raw file: {}'.format(rawfile)) if permute is None: rawchan = [k for k in range(nchan)] else: rawchan = permute for j in range(nchan): k = rawchan[j] raw[j][:] = data[k][:] iprint('Wrote {} data elements'.format(len(data[k]))) # print(raw) del raw
def read_nbm(filename): """Reads NBF metadata, which is usually saved in an 'nbm' file. This file contains basic information about the recording.""" attrs = {} iprint('Reading nbm file {}.'.format(filename)) with open(filename) as f: for line in f: x = line[0:-1].split('=') # No spaces allowed please!! if len(x) == 2: attrs[x[0]] = x[1] iprint('Attribute: {} = {}'.format(x[0], x[1])) else: wprint("Don't know what to do with this line: {}".format(line)) try: date = attrs['date'] except: date = None if attrs['dirname'][0] == '.': new_dirname = os.path.dirname(os.path.abspath(filename)) attrs['dirname'] = new_dirname metadata = nb_metadata(attrs['dirname'], sample_rate=float(attrs['sample_rate']), nchannels=int(attrs['nchannels']), start=int(attrs['start']), end=int(attrs['end']), date=date, attributes=attrs) return metadata
def parse_uri(uri, filename=None): true_uri, rel = resolve_relative_paths(uri, filename) if rel and filename is None: wprint('URI path is relative, but no base filename is specified.') else: uri = true_uri iprint('URI: {}'.format(uri)) desc = urisplit(uri) return desc, rel
def max_spike_id(self): self.cur.execute('select max(spikeID) from spiketimes;') x = self.cur.fetchone() iprint('max_spike_id() = {}'.format(x)) n = x[0] if n is None: return -1 else: return n
def create_table(self, es, name): es.execute('pragma foreign_key=ON;') try: es.execute(self.defs[name]) if self.verbose: iprint('Created table {}.'.format(name)) except Exception as e: if self.verbose: wprint('Could not create table {}. Error: {}'.format(name,e))
def maybe_fix_filenames(self, dirname): old = os.path.dirname(self.filename) iprint('maybe_fix_filenames: dirname={}'.format(dirname)) iprint('maybe_fix_filenames: old dirname={}'.format(old)) if dirname != old: wprint('Directory was moved. Resetting...') name = os.path.basename(dirname) self.filename = os.path.join(dirname, '{}.nbm'.format(name)) self.rawfile = os.path.join(dirname, '{}.raw'.format(name)) self.meanfile = os.path.join(dirname, '{}-mean.pyr'.format(name)) self.minfile = os.path.join(dirname, '{}-min.pyr'.format(name)) self.maxfile = os.path.join(dirname, '{}-max.pyr'.format(name))
def spike_peaks(sig, threshold): """Given an LFP signal *sig* and a *threshold*, isolate all peaks that are above threshold, by finding the zero-crossings of the first derivative of the smoothed signal. Returns a vector of 1s and 0s such that the 1s identify spikes.""" upvec = sig > threshold iprint('Peak vector computed.') sys.stdout.flush() smoothed = smooth_lfp(sig, 31, 6.0) mean = smoothed.mean() iprint('Smoothed signal computed') sys.stdout.flush() first = deriv(smoothed) iprint('First derivative computed') sys.stdout.flush() zc = np.zeros(len(sig)) numzc = 0 numpts = len(zc) tick = numpts / 100 for i in range(1, len(zc) - 1): if (upvec[i]): if (first[i] == 0 or first[i] * first[i - 1] < 0): zc[i] = sig[i] numzc += 1 if (i % tick == 0): print('{}% ({})'.format(i / tick, numzc), end="") sys.stdout.flush() print(' ') iprint('Zero-crossings of first derivative computed.') sys.stdout.flush() return zc
def map_filtered_data(self): """Maps filtered data files into memory for direct access to filtered data.""" dsf = self.dsf dir = self.dirname if len(self.filtered_data) == 0: fdatafile = os.path.join( dir, 'filtered_{}_{}.raw'.format(dsf.lo, dsf.hi)) iprint('Checking for filtered data file: {}'.format(fdatafile)) if not os.path.exists(fdatafile): self.make_filtered_data() iprint('Mapping filtered data file with shape {}...'.format( self.data_shape)) self.filtered_data = np.memmap(fdatafile, np.dtype('f4'), 'r+', shape=self.data_shape)
def make_filtered_data(self, force=False): """Makes a cache of the filtered data, with filter parameters specified by the dsf object.""" dsf = self.dsf dir = self.dirname fdatafile = os.path.join(dir, 'filtered_{}_{}.raw'.format(dsf.lo, dsf.hi)) if force or not os.path.exists(fdatafile): iprint('Creating raw file {} using shape {}.'.format( fdatafile, self.data_shape)) fdata = np.memmap(fdatafile, np.dtype('f4'), 'w+', shape=self.data_shape) iprint('Initializing filtered data file: {}'.format(fdatafile)) for k in range(self.data_shape[0]): data = dsf.maybe_filter(self.raw[k]) fdata[k, :] = data[:] # print(raw) del fdata
def wfdb_to_nbf(filename, to_dir, channel_names=['V1', 'V2', 'V3', 'V4'], date=None): iprint("Creating nbf from file: {}".format(filename)) data, fields = wfdb.rdsamp(filename) print("Available fields: {}".format(fields)) # end sample number: start = 0 end = fields['sig_len'] # I think this is the sample rate: sample_rate = fields['fs'] # This will barf if the channel_names don't exist in the file, but # collect the indices of the channels that we are interested in # (the Vn leads on the chest, for now): indices = [fields['sig_name'].index(x) for x in channel_names] nchan = len(indices) md = nb_metadata(to_dir, sample_rate, nchan, start, end, True, date=date) nbf_filename = md.filename # If the directory doesn't exist, create it: if not os.path.isdir(to_dir): os.makedirs(to_dir) # See floats_to_nbf for hints on Events. For now, we aren't # looking for events, but this is where we would create an # events.dat file. # Save the metadata file: md.save(source=filename) # Create the raw data file: rawfile = md.rawfile() iprint('Mapping raw file {} using shape {}.'.format(rawfile, (nchan, end))) raw = np.memmap(rawfile, np.dtype('f4'), 'w+', shape=(nchan, end)) # print(raw) iprint('Initializing raw file: {}'.format(rawfile)) for j in range(nchan): k = indices[j] raw[j][:] = data[:, k] iprint('Wrote {} data elements'.format(len(data[:, k]))) # print(raw) del raw
def add_basis(self, basis_id, sigma, width, nfuncs=5, offset=0, name="gaussian_basis"): """Adds a Gaussian basis with the specified sigma and width in SAMPLES to the database, with the desired ID.""" ret = True self.cur.execute( 'SELECT * from basis where basisID == {};'.format(basis_id)) result = self.cur.fetchall() iprint("{}".format(result)) if result is None or len(result) == 0: self.cur.execute( 'INSERT INTO basis values ({}, {}, {}, {}, {}, "{}");'.format( basis_id, offset, sigma, width, nfuncs, name)) self.conn.commit() else: result = result[0] if math.fabs(result[1] - sigma) > 1.0e-06 or result[ 2] != width or result[3] != nfuncs: wprint( "Basis ID {} is already in use with these parameters: sigma={}, window width={}, n={}" .format(result[0], result[1], result[2], result[3])) iprint("Here are the basis ID's in use in this file:") self.cur.execute('SELECT * from basis;') result = self.cur.fetchall() iprint("BasisID offset sigma window nfuncs name") for tuple in result: iprint("{} {} {} {} {} {}".format( tuple[0], tuple[1], tuple[2], tuple[3], tuple[4], tuple[5])) ret = False else: iprint( "Your basis set is already available in {}.".format(self)) ret = True return ret
def histogram(vector, nbins=100, min=None, max=None, clip=True): """Given a vector, returns a histogram vector along with metadata about binning.""" if min == None: min = vector.min() if max == None: max = vector.max() dx = (max - min) / nbins iprint("# min={}, max={}, dx={}".format(min, max, dx)) kmax = nbins - 1 hist = np.zeros(nbins) for val in vector: k = int((val - min) / dx) if k < 0: if not clip: hist[0] += 1 elif k > kmax: if not clip: hist[kmax] += 1 else: hist[k] += 1 return hist
def open(uri, filename=None): """Given a URI for a data source, open it and return the appropriate data source object.""" if True: desc, rel = parse_uri(uri, filename) else: new_uri, rel = resolve_relative_paths(uri, filename) if rel and filename is None: wprint('Data store path is relative, but no event store is specified.') else: uri = new_uri desc = urisplit(uri) iprint('Opening data store at {}'.format(uriunsplit(desc))) s = desc.scheme if s is None: return ndk.ds.neo_in.spike2(desc) elif s == 'pre': return ndk.ds.pre.pre_ds(desc) elif s == 'smr': return ndk.ds.neo_in.spike2(desc) # elif s == 'file': # return ndk.ds.neo.neo_in(desc) elif s == 'cass': return ndk.ds.cass.cdb(desc) elif s == 'wav': return ndk.ds.wav.wav_ds(desc.path) elif s == 'ds': return ndk.ds.mmap.mmap_ds(desc.path) elif s == 'edf': return ndk.ds.edf_ds(desc) elif s == 'nbm': iprint('NBF path: {}'.format(desc.path)) if rel and desc.path[0] == '/' and desc.path[1] == '.': return ndk.ds.nbf(desc.path[1:]) # Hack! else: return ndk.ds.nbf(desc.path) else: print("Don't know what do with this URI scheme: {} (in {})".format(s, uri))
def map_raw_data(self, version): """Maps raw data files into memory for direct access to data.""" rawfile = self.md.rawfile() vmax = get_version_count(rawfile) iprint('Versions found: {}'.format(vmax)) if version is not None and version <= vmax: self.version = version iprint('Selecting version {}.'.format(self.version)) rawfile = rawfile + '.{}'.format(version) iprint('Checking for raw file: {}'.format(rawfile)) if not os.path.exists(rawfile): iprint('Creating raw file...') arr = np.zeros(self.data_shape, np.dtype('f4')) with open(rawfile, 'wb') as f: f.write(arr) iprint('Mapping raw file with shape {}...'.format(self.data_shape)) self.raw = np.memmap(rawfile, np.dtype('f4'), 'r+', shape=self.data_shape)
def get_basis(self, basis_id): """Returns the set of basis functions corresponding to 'basis_id'.""" try: seq = self.bases[basis_id] except: self.cur.execute( 'SELECT * from basis where basisID == {};'.format(basis_id)) result = self.cur.fetchall() # print(result) if len(result) < 1: return None # This is the old default - older db files might not have # this column: nfuncs = 5 if len(result[0]) > 3: nfuncs = int(result[0][4]) rate = self.sample_rate if USE_MS: # Convert to seconds, then to sample numbers: offset = result[0][1] sigma = result[0][2] * 0.001 * rate iprint('rate = {}; result[0][2] = {}'.format( rate, result[0][2])) width = int(result[0][3] * 0.001 * rate) iprint('# sigma={}, width={}'.format(sigma, width)) else: offset = result[0][1] sigma = result[0][2] width = result[0][3] # If no generator is supplied, then the functions table # should contain explicit values for the function basis. # Regardless, we derive the window size in samples and the # number of basis functions from the basis table: if result[0][5] == '': seq = self.get_functions(basis_id, nfuncs, width) else: if width <= 0: print( "RECOVERABLE ERROR: window width for basis {} is 0 samples!" .format(basis_id)) print( " Setting width = 32 samples and sigma = 16 samples!" ) width = 32 sigma = 16 iprint("# Window width = {}".format(width)) gen_name = 'gaussian_basis' if len(result[0]) > 5: gen_name = result[0][5] gen_fn = self.schema.basis_generator(gen_name) seq = gen_fn(sigma, width, nfuncs) self.bases[basis_id] = seq return seq
def __init__(self, uri, start=0, end=0, sample_rate=32000.0, nchannels=1, version=None): iprint("Creating nbf from URI: {}".format(uri)) # This is sloppy. In this implementation, the URI scheme is # optional. If the scheme is not present, then the URI is # treated as a filename. if True: self.desc = ndk.ds.parse_uri(uri) else: if isinstance(uri, str) or isinstance(uri, unicode): self.desc = urisplit(uri) else: self.desc = uri self.filename = os.path.abspath(uri) self.filtered_data = {} iprint("Set up filtered_data list: {}".format(self.filtered_data)) self.lazy = False dirname = os.path.dirname(self.filename) self.dirname = dirname iprint('Using directory {}'.format(dirname)) # These are defaults, can be overridden later: name = os.path.basename(self.filename) name = name.split('.')[0] new = False # If the directory doesn't exist, create it: if not os.path.isdir(dirname): os.makedirs(dirname) # Compute the constituent filenames: self.mdfile = os.path.join(dirname, '{}.nbm'.format(name)) efile = os.path.join(dirname, 'events.dat') def getfirst(y): return y[0] # If there are any events, generate the list: self.events = [] if os.path.exists(efile): print("Events found:") with open(efile, 'r') as f: for line in f: x = line.split(':') elabel = x[1].split('\n')[0] print("{}: {}".format(elabel, x[0])) self.events.append((int(x[0]), elabel)) self.events.sort(key=getfirst) # How? Each channel is a "strip" of consecutive samples at a rate of # sample_rate * 2^(-i) samples per second. # Thus, the first index is the channel, the second is sample. # This will load metadata, but otherwise will create it with # defaults: iprint('Checking for metadata file {}'.format(self.mdfile)) if not os.path.exists(self.mdfile): dirname = os.path.dirname(os.path.abspath(self.mdfile)) self.md = nb_metadata(dirname, sample_rate=sample_rate, start=start, end=end, nchannels=nchannels) elif NEW_NBM: self.md = read_nbm(self.mdfile) else: with open(self.mdfile, 'r') as f: self.md = pickle.load(f) self.md.maybe_fix_filenames(dirname) self.dsf = dsfilter(self.md.samprate) self.data_shape = (self.md.nchannels, self.md.npts()) iprint('Initializing nbf object with data shape {}.'.format( self.data_shape)) self.t0 = self.md.start self.t1 = self.md.end self.version = None self.map_raw_data(version)
def old(rawfile, data, nchan): with open(rawfile, 'wb') as f: for k in range(nchan): f.write(data[k]) iprint('Wrote {} data elements'.format(len(data[k])))