Пример #1
0
def read_nse(fin):
    """Read single electrode spike record.
  Inputs:
    fin - file handle
    only_timestamps - if true, only load the timestamps, ignoring the waveform and feature data

  Output: Dictionary with fields
    'header'  - header info
    'packets' - pylab data structure with the spike data

  Notes:
    0. What is spike acquizition entity number? Ask neuralynx
    1. Removing LOAD_ATTR overhead by defining time_stamp_append = [time_stamp[n].append for n in xrange(100)] and using
       time_stamp_append[dwCellNumber](qwTimeStamp) in the loop does not seem to improve performance.
       It reduces readability so I did not use it.
    2. Disabling garbage collection did not help
    3. In general, dictionary lookups slow things down
    4. using numpy arrays, with preallocation is slower than the dumb, straightforward python list appending
  """

    hdr = read_header(fin)
    nse_packet = pylab.dtype([('timestamp', 'Q'), ('saen', 'I'),
                              ('cellno', 'I'), ('Features', '8I'),
                              ('waveform', '32h')])
    data = pylab.fromfile(fin, dtype=nse_packet, count=-1)
    return {'header': hdr, 'packets': data}
Пример #2
0
def readCsv(ifile):
    skeys = [
        'date', 'homeTeam', 'awayTeam', 'game_id', 'player', 'posteam',
        'oldstate', 'newstate'
    ]
    ikeys = ['seas', 'igame_id', 'yds']
    fkeys = []

    dt = []
    lines = [l.strip() for l in open(ifile).readlines()]
    hd = lines[0]
    ks = hd.split(',')
    for k in ks:
        if k in skeys:
            tmp = (k, 'S64')
        elif k in ikeys:
            tmp = (k, 'i4')
        elif k in fkeys:
            tmp = (k, 'f4')
        else:
            tmp = (k, 'f8')
        dt.append(tmp)

    dt = pylab.dtype(dt)
    data = pylab.genfromtxt(ifile, dtype=dt, skip_header=1, delimiter=',')
    return data
Пример #3
0
def read_nse(fin):
  """Read single electrode spike record.
  Inputs:
    fin - file handle
    only_timestamps - if true, only load the timestamps, ignoring the waveform and feature data

  Output: Dictionary with fields
    'header'  - header info
    'packets' - pylab data structure with the spike data

  Notes:
    0. What is spike acquizition entity number? Ask neuralynx
    1. Removing LOAD_ATTR overhead by defining time_stamp_append = [time_stamp[n].append for n in xrange(100)] and using
       time_stamp_append[dwCellNumber](qwTimeStamp) in the loop does not seem to improve performance.
       It reduces readability so I did not use it.
    2. Disabling garbage collection did not help
    3. In general, dictionary lookups slow things down
    4. using numpy arrays, with preallocation is slower than the dumb, straightforward python list appending
  """

  hdr = read_header(fin)
  nse_packet = pylab.dtype([
    ('timestamp', 'Q'),
    ('saen', 'I'),
    ('cellno', 'I'),
    ('Features', '8I'),
    ('waveform', '32h')
  ])
  data = pylab.fromfile(fin, dtype=nse_packet, count=-1)
  return {'header': hdr, 'packets': data}
Пример #4
0
 def normalize(self, img, order=1, dtype=dtype('f'), cval=0):
     dewarped = self.dewarp(img, cval=cval, dtype=dtype)
     h, w = dewarped.shape
     scaled = scale_to_h(dewarped,
                         self.target_height,
                         order=order,
                         dtype=dtype,
                         cval=cval)
     return scaled
Пример #5
0
 def dewarp(self, img, cval=0, dtype=dtype('f')):
     assert img.shape == self.shape
     h, w = img.shape
     hpadding = self.r
     padded = vstack(
         [cval * ones((hpadding, w)), img, cval * ones((hpadding, w))])
     center = self.center + hpadding
     dewarped = [
         padded[center[i] - self.r:center[i] + self.r, i] for i in range(w)
     ]
     dewarped = array(dewarped, dtype=dtype).T
     return dewarped
Пример #6
0
def scale_to_h(img, target_height, order=1, dtype=dtype('f'), cval=0):
    h, w = img.shape
    scale = target_height * 1.0 / h
    target_width = int(scale * w)
    output = interpolation.affine_transform(1.0 * img,
                                            eye(2) / scale,
                                            order=order,
                                            output_shape=(target_height,
                                                          target_width),
                                            mode='constant',
                                            cval=cval)
    output = array(output, dtype=dtype)
    return output
Пример #7
0
def read_nev(fin, parse_event_string=False):
  """Read an event file.
  Input:
    fin - file handle
    parse_event_string - If set to true then parse the eventstrings nicely. This takes extra time. Default is False
  Ouput:
    Dictionary with fields
      'header' - the file header
      'packets' - the events. This is a new pylab dtype with fields corresponding to the event packets.
        'nstx'
        'npkt_id'
        'npkt_data_size'
        'timestamp' - timestamp (us)
        'eventid'
        'nttl' - value of the TTL port
        'ncrc'
        'ndummy1'
        'ndummy2'
        'dnExtra'
        'eventstring' - The alphanumeric string NeuraLynx attaches to this event

      'eventstring' - Only is parse_event_string is set to True. This is a nicely formatted eventstring
  """
  hdr = read_header(fin)
  nev_packet = pylab.dtype([
    ('nstx', 'h'),
    ('npkt_id', 'h'),
    ('npkt_data_size', 'h'),
    ('timestamp', 'Q'),
    ('eventid', 'h'),
    ('nttl', 'H'),
    ('ncrc', 'h'),
    ('ndummy1', 'h'),
    ('ndummy2', 'h'),
    ('dnExtra', '8i'),
    ('eventstring', '128c')
  ])
  data = pylab.fromfile(fin, dtype=nev_packet, count=-1)
  logger.debug('{:d} events'.format(data['timestamp'].size))
  if parse_event_string:
    logging.info('Packaging the event strings. This makes things slower.')
    # Makes things slow. Often this field is not needed
    evstring = [None]*data['timestamp'].size
    for n in xrange(data['timestamp'].size):
      str = ''.join(data['eventstring'][n])
      evstring[n] = str.replace('\00','').strip()
    return {'header': hdr, 'packets': data, 'eventstring': evstring}
  else:
    return {'header': hdr, 'packets': data}
Пример #8
0
def read_nev(fin, parse_event_string=False):
    """Read an event file.
    Input:
    fin - file handle
    parse_event_string - If set to true then parse the eventstrings nicely. This takes extra time. Default is False
    Ouput:
    Dictionary with fields
    'header' - the file header
    'packets' - the events. This is a new pylab dtype with fields corresponding to the event packets.
    'nstx'
    'npkt_id'
    'npkt_data_size'
    'timestamp' - timestamp (us)
    'eventid'
    'nttl' - value of the TTL port
    'ncrc'
    'ndummy1'
    'ndummy2'
    'dnExtra'
    'eventstring' - The alphanumeric string NeuraLynx attaches to this event
    'eventstring' - Only is parse_event_string is set to True. This is a nicely formatted eventstring
    """
    hdr = read_header(fin)
    nev_packet = pylab.dtype([
        ('nstx', 'h'),
        ('npkt_id', 'h'),
        ('npkt_data_size', 'h'),
        ('timestamp', 'Q'),
        ('eventid', 'h'),
        ('nttl', 'H'),
        ('ncrc', 'h'),
        ('ndummy1', 'h'),
        ('ndummy2', 'h'),
        ('dnExtra', '8i'),
        ('eventstring', '128c')
    ])
    data = pylab.fromfile(fin, dtype=nev_packet, count=-1)
    logger.debug('{:d} events'.format(data['timestamp'].size))
    if parse_event_string:
        logging.info('Packaging the event strings. This makes things slower.')
        # Makes things slow. Often this field is not needed
        evstring = [None]*data['timestamp'].size
        for n in range(data['timestamp'].size):
            str = ''.join(data['eventstring'][n])
            evstring[n] = str.replace('\00','').strip()
        return {'header': hdr, 'events': data, 'eventstring': evstring}
    else:
        return {'header': hdr, 'events': data}
Пример #9
0
def loadPlayByPlay(csvfile, vbose=0):
    skeys = [
        'game_id', 'type', 'playerName', 'posTeam', 'awayTeam', 'homeTeam'
    ]
    ikeys = ['seas', 'igame_id', 'dwn', 'ytg', 'yfog', 'yds']
    fkeys = []

    lines = [l.strip() for l in open(csvfile).readlines()]
    hd = lines[0]
    ks = hd.split(',')
    dt = []
    for k in ks:
        # postgres copy to file makes headers lower-case; this is a kludge
        if k == 'playername':
            k = 'playerName'
        elif k == 'posteam':
            k = 'posTeam'
        elif k == 'away_team':
            k = 'awayTeam'
        elif k == 'awayteam':
            k = 'awayTeam'
        elif k == 'home_team':
            k = 'homeTeam'
        elif k == 'hometeam':
            k = 'homeTeam'

        if k in skeys:
            tmp = (k, 'S16')
        elif k in ikeys:
            tmp = (k, 'i4')
        else:
            tmp = (k, 'f8')

        if vbose >= 1:
            print k, tmp

        dt.append(tmp)
    dt = pylab.dtype(dt)
    data = pylab.genfromtxt(csvfile, dtype=dt, delimiter=',', skip_header=1)
    return data
Пример #10
0
def loadPlayByPlay(csvfile, vbose=0):
    skeys = ['game_id','type','playerName','posTeam','awayTeam','homeTeam']
    ikeys = ['seas','igame_id','dwn','ytg','yfog','yds']
    fkeys = []

    lines = [l.strip() for l in open(csvfile).readlines()]
    hd = lines[0]
    ks = hd.split(',')
    dt = []
    for k in ks:
        # postgres copy to file makes headers lower-case; this is a kludge
        if k=='playername':
            k = 'playerName'
        elif k=='posteam':
            k = 'posTeam'
        elif k=='away_team':
            k = 'awayTeam'
        elif k=='awayteam':
            k = 'awayTeam'
        elif k=='home_team':
            k = 'homeTeam'
        elif k=='hometeam':
            k = 'homeTeam'

        if k in skeys:
            tmp = (k, 'S16')
        elif k in ikeys:
            tmp = (k, 'i4')
        else:
            tmp = (k, 'f8')

        if vbose>=1:
            print k, tmp

        dt.append(tmp)
    dt = pylab.dtype(dt)
    data = pylab.genfromtxt(csvfile, dtype=dt, delimiter=',', skip_header=1)
    return data
Пример #11
0
def readCsv(ifile):
    skeys = ['date', 'homeTeam', 'awayTeam', 'game_id','player','posteam','oldstate','newstate']
    ikeys = ['seas','igame_id','yds']
    fkeys = []

    dt = []
    lines = [l.strip() for l in open(ifile).readlines()]
    hd = lines[0]
    ks = hd.split(',')
    for k in ks:
        if k in skeys:
            tmp = (k, 'S64')
        elif k in ikeys:
            tmp = (k, 'i4')
        elif k in fkeys:
            tmp = (k, 'f4')
        else:
            tmp = (k, 'f8')
        dt.append(tmp)

    dt = pylab.dtype(dt)
    data = pylab.genfromtxt(ifile, dtype=dt, skip_header=1, delimiter=',')
    return data
Пример #12
0
import pylab
import time
import os
from matplotlib import dates
import datetime


os.environ["DISPLAY"] = ":0.0"

plotdailylogfilename = '/home/pi/Desktop/powersystem-logfiles/' + time.strftime("%Y-%m-%d") + '.log'
#dt=pylab.dtype([('f0','datetime64[ms]'),('f1',str,(2)),('f2',float,(5))])
dt=pylab.dtype([('f0','datetime64[ms]'),('s1',str,(15)),('f2',float,(1)),('f3',float,(1)),('f4',float,(1)),('f5',float,(1)),('f6',float,(1)),('f7',float,(1)),('f8',float,(1)),('f9',float,(1)),('f10',float,(1)),('f11',float,(1)),('s12',str,(15)),('f13',float,(1)),('f14',float,(1)),('f15',float,(1)),('f16',float,(1)),('f17',float,(1)),('f19',float,(1)),('f20',float,(1)),('f21',float,(1)),('f22',float,(1)),('f23',float,(1)),('f24',float,(1)),('f25',float,(1)),('f26',float,(1)),('f27',float,(1)),('f28',float,(1)),('f29',float,(1)),('f30',float,(1)),('f31',float,(1)),('f32',float,(1)),('f33',float,(1)),('f34',float,(1))])
data = pylab.genfromtxt(plotdailylogfilename,dtype=dt,delimiter='\t')

pylab.plot( data['f0'], data['f2'], label="Output")
pylab.plot( data['f0'], data['f3'], label="Battery")
pylab.legend()
pylab.title("Charge Data for " + time.strftime("%Y-%m-%d"))
pylab.xlabel("Time")
pylab.ylabel("Voltage: \n")
pylab.savefig("/var/www/html/voltagegraph.png")
pylab.clf()

pylab.plot( data['f0'], data['f21'], label="Amp")
pylab.plot( data['f0'], data['f22'], label="Watts")
pylab.legend()
pylab.title("Charge Data for " + time.strftime("%Y-%m-%d"))
pylab.xlabel("Time")
pylab.ylabel("Charge")
pylab.legend(loc='lower left')
pylab.savefig("/var/www/html/whcgraph.png")
Пример #13
0
def read_csc(fin, assume_same_fs=True):
    """Read a continuous record file. We return the raw packets but, in addition, if we set assume_same_fs as true we
  return a trace with all the data concatenated together, assuming that a constant sampling frequency was maintained
  through out. Gaps in the record are padded with zeros.
  Input:
    fin - file handle
    assume_same_fs - if True, concatenate any segments together, fill time gaps with zeros and return average Fs
  Ouput:
    Dictionary with fields
      'header' - the file header
      'packets' - the actual packets as read. This is a new pylab dtype with fields:
        'timestamp' - timestamp (us)
        'chan' - channel
        'Fs' - the sampling frequency
        'Ns' - the number of valid samples in the packet
        'samp' - the samples in the packet.
          e.g. x['packets']['samp'] will return a 2D array, number of packets long and 512 wide (since each packet carries 512 wave points)
          similarly x['packets']['timestamp'] will return an array number of packets long
      'Fs': the average frequency computed from the timestamps (can differ from the nominal frequency the device reports)
      'trace': the concatenated data from all the packets
      't0': the timestamp of the first packet.
  NOTE: while 'packets' returns the exact packets read, 'Fs' and 'trace' assume that the record has no gaps and that the
  sampling frequency has not changed during the recording
  """
    hdr = read_header(fin)
    csc_packet = pylab.dtype([('timestamp', 'Q'), ('chan', 'I'), ('Fs', 'I'),
                              ('Ns', 'I'), ('samp', '512h')])

    data = pylab.fromfile(fin, dtype=csc_packet, count=-1)
    Fs = None
    trace = None
    if assume_same_fs:
        if data['Fs'].std() > 1e-6:  #
            logger.warning(
                'Fs is not fixed across trace, not packing packets together')
            assume_same_fs = False

    if not assume_same_fs: return {'header': hdr, 'packets': data}

    packet_duration_us = 512 * (1. / data['Fs'][0]) * 1e6
    #For the version we are dealing with, Neuralynx packets are always 512
    #This is actually a very poor estimate if the sampling freq is low, since it rounds to nearest Hz
    #So we'll not rely on this but come up with our own estimate

    samp = data['samp']
    ts_us = data['timestamp']
    dt_us = pylab.diff(ts_us).astype('f')
    idx = pylab.find(
        dt_us > packet_duration_us
    )  #This will find any instances where we paused the recording
    if idx.size == 0:  #No padding needed
        trace = samp.ravel()
        Fs = (data['Ns'][:-1] / (dt_us * 1e-6)).mean()
    else:  #We have some padding to do.
        logger.debug('Gaps in record, padding')
        #Our first task is to find all the contiguous sections of data
        idx += 1  #Shifting indexes to point at the packets that come after a gap
        idx = pylab.insert(
            idx, 0, 0
        )  #Now idx contains the indexes of every packet that starts a contiguous section
        idx = pylab.append(idx, ts_us.size)  #And the index of the last packet
        Ns = data['Ns']
        estimFs_sum = 0
        N_samps = 0
        sections = []
        for n in xrange(idx.size - 1):  #collect all the sections
            n0 = idx[n]
            n1 = idx[n + 1]
            sections.append(samp[n0:n1].ravel())
            if n1 - n0 > 1:  #We need more than one packet in a section to get an estimate
                estimFs_sum += (Ns[n0:n1 - 1] /
                                (dt_us[n0:n1 - 1] * 1e-6)).sum()
                N_samps += n1 - 1 - n0

        Fs = estimFs_sum / float(N_samps)
        #Now pad the data appropriately
        padded = [sections[0]]
        cum_N = sections[0].size
        for n in xrange(1, len(sections)):
            #Now figure out how many zeros we have to pad to get the right length
            Npad = int((ts_us[idx[n]] - ts_us[0]) * 1e-6 * Fs - cum_N)
            padded.append(pylab.zeros(Npad))
            padded.append(sections[n])
            cum_N += Npad + sections[n].size
        trace = pylab.concatenate(
            padded)  #From this packet to the packet before the gap

    return {
        'header': hdr,
        'packets': data,
        'Fs': Fs,
        'trace': trace,
        't0': ts_us[0]
    }
Пример #14
0
    def seek_packet(f):
        """Skip forward until we find the STX magic number."""
        #Read in 32bit increments until the magic number is found
        start = f.tell()
        pkt = f.read(4)
        while len(pkt) == 4:
            if pkt == '\x00\x08\x00\x00': #Magic number 2048 0x0800
                f.seek(-4,1) #Realign
                break
            pkt = f.read(4)
        stop = f.tell()
        return stop - start

        logger.info('Notice: you are using the slow version of the extractor. All error checks are done')

        #nrd packet format
        nrd_packet = pylab.dtype([
            ('stx', 'i'),
            ('pkt_id', 'i'),
            ('pkt_data_size', 'i'),
            ('timestamp high', 'I'), #Neuralynx timestamp is ... in its own 32 bit world
            ('timestamp low', 'I'),
            ('status', 'i'),
            ('ttl', 'I'),
            ('extra', '10i'),
            ('data', '{:d}i'.format(channels)),
            ('crc', 'i')
        ])
        packet_size = nrd_packet.itemsize

        pkt_cnt = 0
        garbage_bytes = 0
        stx_err_cnt = 0
        pkt_id_err_cnt = 0
        pkt_size_err_cnt = 0
        pkt_ts_err_cnt = 0
        pkt_crc_err_cnt = 0

        if max_pkts != -1: #An insidious bug was killed here!
            if buffer_size > max_pkts:
                buffer_size = max_pkts

        #The files we will write to.
        fts = open(ftsname,'wb')
        fttl = open(fttlname,'wb')
        fchan = [open(fcn,'wb') for fcn in fchanname]

        last_ts = 0
        with open(fname,'rb') as f:
            hdr = read_header(f)
            logger.info('File header: {:s}'.format(hdr))

            garbage_bytes += seek_packet(f)
            these_packets = pylab.fromfile(f, dtype=nrd_packet, count=buffer_size)
            while these_packets.size > 0:
                all_packets_good = True
                packets_read = these_packets.size

                idx = pylab.find(these_packets['stx'] != 2048)
                if idx.size > 0:
                    stx_err_cnt += 1
                    all_packets_good = False
                    max_good_packets = idx[0]
                    these_packets = these_packets[:max_good_packets]

                if these_packets.size > 0:
                    idx = pylab.find(these_packets['pkt_id'] != 1)
                    if idx.size > 0:
                      pkt_id_err_cnt += 1
                      all_packets_good = False
                      max_good_packets = idx[0]
                      these_packets = these_packets[:max_good_packets]

                if these_packets.size > 0:
                    idx = pylab.find(these_packets['pkt_data_size'] != 10 + channels)
                    if idx.size > 0:
                      pkt_size_err_cnt += 1
                      all_packets_good = False
                      max_good_packets = idx[0]
                      these_packets = these_packets[:max_good_packets]

                if these_packets.size > 0:
                    #crc computation
                    field32 = pylab.vstack([these_packets[k].T for k in nrd_packet.fields.keys()]).astype('I')
                    crc = pylab.zeros(these_packets.size,dtype='I')
                    for idx in range(field32.shape[0]):
                        crc ^= field32[idx,:]
                    idx = pylab.find(crc != 0)
                    if idx.size > 0:
                        pkt_crc_err_cnt += 1
                        all_packets_good = False
                        max_good_packets = idx[0]
                        these_packets = these_packets[:max_good_packets]

                if these_packets.size > 0:
                    ts = pylab.array((these_packets['timestamp high'].astype('uint64')<<32) | (these_packets['timestamp low']), dtype='uint64')
                    bad_idx = -1
                    if last_ts > ts[0]:#Time stamps out of order at buffer boundary
                        bad_idx = 0
                    else:
                        idx = pylab.find(ts[:-1] > ts[1:])
                        if idx.size > 0:
                            bad_idx = idx[0] + 1
                    if bad_idx > -1:
                        logger.info('Out of order timestamp {:d}'.format(int(ts[bad_idx])))
                        pkt_ts_err_cnt += 1
                        all_packets_good = False
                        max_good_packets = bad_idx
                        these_packets = these_packets[:max_good_packets]
                        ts = ts[:max_good_packets]

                if these_packets.size > 0:
                    last_ts = ts[-1] #Ready for the next read
                    ts.tofile(fts)
                    these_packets['ttl'].tofile(fttl)
                    for idx,ch in enumerate(channel_list):
                        these_packets['data'][:,ch].tofile(fchan[idx])

                pkt_cnt += these_packets.size
                if max_pkts != -1:
                    if pkt_cnt >= max_pkts: #NOTE: This may give us upto buffer_size -1 more packets than we want.
                        break

                if not all_packets_good:
                    f.seek((these_packets.size-packets_read)*packet_size+4,1) #Rewind all the way except 32 bits
                    garbage_bytes += seek_packet(f)

                if pkt_ts_err_cnt + pkt_crc_err_cnt + stx_err_cnt > error_bugout:
                    logger.warning('Too many errors, bugging out')
                    break

                these_packets = pylab.fromfile(f, dtype=nrd_packet, count=buffer_size)

        fts.close()
        fttl.close()
        [fch.close() for fch in fchan]

        logger.info('Extracted {:d} packets'.format(pkt_cnt))
        logger.info('{:d} garbage words'.format(garbage_bytes))
        logger.info('{:d} packets had bad stx'.format(stx_err_cnt))
        logger.info('{:d} packets had bad pkt id'.format(pkt_id_err_cnt))
        logger.info('{:d} packets had bad crc'.format(pkt_crc_err_cnt))
        logger.info('{:d} packets had out of order timestamps'.format(pkt_ts_err_cnt))
Пример #15
0
def extract_nrd_ec(fname,
                   ftsname,
                   fttlname,
                   fchanname,
                   channel_list,
                   channels=64,
                   max_pkts=-1,
                   buffer_size=10000,
                   error_bugout=1000000000):
    """Read and write out selected raw traces from the .nrd file with error checking.
  Inputs:
    fname - name of nrd file
    ftsname - name under which timestamp vector will be saved
    fttlname - name under which the events will be saved
    fchanname - a list of file names for the
    channel_list - Which AD channels to convert.
    channels - total channels in the system
    max_pkts - total packets to read. If set to -1 then read all packets
    buffer_size   - how many chunks to read at a time.
    error_bugout - If the sum of stx, crc and timestamp errors exceed this value quit reading the file
  Outputs:
    Data are written to file

  e.g.
  ----------------------------------------------------------------------------------------------------------------------
  from neurapy.neuralynx import lynxio
  import logging
  logging.basicConfig(level=logging.DEBUG)

  channels = 64
  fname = '/Users/kghose/Research/2013/Projects/Workingmemory/Data/NeuraLynx/2013-01-25_14-53-04/DigitalLynxRawDataFile.nrd'
  channel_list = [0,1,2]

  ftsname = 'timestamps.raw'
  fttlname = 'ttl.raw'
  fchanname = ['chan_{:000d}.raw'.format(ch) for ch in channel_list]
  lynxio.extract_nrd(fname, ftsname, fttlname, fchanname, channel_list, channels, max_pkts=1000)
  ----------------------------------------------------------------------------------------------------------------------

  Data are written as a pure stream of binary data and can be easily and efficiently read using the numpy read function.
  For convenience, a function that reads the timestamps, events and channels (read_extracted_data) is included in the library.

  """
    def seek_packet(f):
        """Skip forward until we find the STX magic number."""
        #Read in 32bit increments until the magic number is found
        start = f.tell()
        pkt = f.read(4)
        while len(pkt) == 4:
            if pkt == '\x00\x08\x00\x00':  #Magic number 2048 0x0800
                f.seek(-4, 1)  #Realign
                break
            pkt = f.read(4)
        stop = f.tell()
        return stop - start

    logger.info(
        'Notice: you are using the slow version of the extractor. All error checks are done'
    )

    #nrd packet format
    nrd_packet = pylab.dtype([
        ('stx', 'i'),
        ('pkt_id', 'i'),
        ('pkt_data_size', 'i'),
        ('timestamp high',
         'I'),  #Neuralynx timestamp is ... in its own 32 bit world
        ('timestamp low', 'I'),
        ('status', 'i'),
        ('ttl', 'I'),
        ('extra', '10i'),
        ('data', '{:d}i'.format(channels)),
        ('crc', 'i')
    ])
    packet_size = nrd_packet.itemsize

    pkt_cnt = 0
    garbage_bytes = 0
    stx_err_cnt = 0
    pkt_id_err_cnt = 0
    pkt_size_err_cnt = 0
    pkt_ts_err_cnt = 0
    pkt_crc_err_cnt = 0

    if max_pkts != -1:  #An insidious bug was killed here!
        if buffer_size > max_pkts:
            buffer_size = max_pkts

    #The files we will write to.
    fts = open(ftsname, 'wb')
    fttl = open(fttlname, 'wb')
    fchan = [open(fcn, 'wb') for fcn in fchanname]

    last_ts = 0L
    with open(fname, 'rb') as f:
        hdr = read_header(f)
        logger.info('File header: {:s}'.format(hdr))

        garbage_bytes += seek_packet(f)
        these_packets = pylab.fromfile(f, dtype=nrd_packet, count=buffer_size)
        while these_packets.size > 0:
            all_packets_good = True
            packets_read = these_packets.size

            idx = pylab.find(these_packets['stx'] != 2048)
            if idx.size > 0:
                stx_err_cnt += 1
                all_packets_good = False
                max_good_packets = idx[0]
                these_packets = these_packets[:max_good_packets]

            if these_packets.size > 0:
                idx = pylab.find(these_packets['pkt_id'] != 1)
                if idx.size > 0:
                    pkt_id_err_cnt += 1
                    all_packets_good = False
                    max_good_packets = idx[0]
                    these_packets = these_packets[:max_good_packets]

            if these_packets.size > 0:
                idx = pylab.find(
                    these_packets['pkt_data_size'] != 10 + channels)
                if idx.size > 0:
                    pkt_size_err_cnt += 1
                    all_packets_good = False
                    max_good_packets = idx[0]
                    these_packets = these_packets[:max_good_packets]

            if these_packets.size > 0:
                #crc computation
                field32 = pylab.vstack([
                    these_packets[k].T for k in nrd_packet.fields.keys()
                ]).astype('I')
                crc = pylab.zeros(these_packets.size, dtype='I')
                for idx in xrange(field32.shape[0]):
                    crc ^= field32[idx, :]
                idx = pylab.find(crc != 0)
                if idx.size > 0:
                    pkt_crc_err_cnt += 1
                    all_packets_good = False
                    max_good_packets = idx[0]
                    these_packets = these_packets[:max_good_packets]

            if these_packets.size > 0:
                ts = pylab.array(
                    (these_packets['timestamp high'].astype('uint64') << 32) |
                    (these_packets['timestamp low']),
                    dtype='uint64')
                bad_idx = -1
                if last_ts > ts[
                        0]:  #Time stamps out of order at buffer boundary
                    bad_idx = 0
                else:
                    idx = pylab.find(ts[:-1] > ts[1:])
                    if idx.size > 0:
                        bad_idx = idx[0] + 1
                if bad_idx > -1:
                    logger.info('Out of order timestamp {:d}'.format(
                        int(ts[bad_idx])))
                    pkt_ts_err_cnt += 1
                    all_packets_good = False
                    max_good_packets = bad_idx
                    these_packets = these_packets[:max_good_packets]
                    ts = ts[:max_good_packets]

            if these_packets.size > 0:
                last_ts = ts[-1]  #Ready for the next read
                ts.tofile(fts)
                these_packets['ttl'].tofile(fttl)
                for idx, ch in enumerate(channel_list):
                    these_packets['data'][:, ch].tofile(fchan[idx])

            pkt_cnt += these_packets.size
            if max_pkts != -1:
                if pkt_cnt >= max_pkts:  #NOTE: This may give us upto buffer_size -1 more packets than we want.
                    break

            if not all_packets_good:
                f.seek((these_packets.size - packets_read) * packet_size + 4,
                       1)  #Rewind all the way except 32 bits
                garbage_bytes += seek_packet(f)

            if pkt_ts_err_cnt + pkt_crc_err_cnt + stx_err_cnt > error_bugout:
                logger.warning('Too many errors, bugging out')
                break

            these_packets = pylab.fromfile(f,
                                           dtype=nrd_packet,
                                           count=buffer_size)

    fts.close()
    fttl.close()
    [fch.close() for fch in fchan]

    logger.info('Extracted {:d} packets'.format(pkt_cnt))
    logger.info('{:d} garbage words'.format(garbage_bytes))
    logger.info('{:d} packets had bad stx'.format(stx_err_cnt))
    logger.info('{:d} packets had bad pkt id'.format(pkt_id_err_cnt))
    logger.info('{:d} packets had bad crc'.format(pkt_crc_err_cnt))
    logger.info(
        '{:d} packets had out of order timestamps'.format(pkt_ts_err_cnt))
Пример #16
0
def extract_nrd_fast(fname,
                     ftsname,
                     fttlname,
                     fchanname,
                     channel_list,
                     channels=64,
                     max_pkts=-1,
                     buffer_size=10000):
    """Read and write out selected raw traces from the .nrd file.
  Inputs:
    fname - name of nrd file
    ftsname - name under which timestamp vector will be saved
    fttlname - name under which the events will be saved
    fchanname - a list of file names for the
    channel_list - Which AD channels to convert.
    channels - total channels in the system
    max_pkts - total packets to read. If set to -1 then read all packets
    buffer_size   - how many chunks to read at a time.
  Outputs:
    Data are written to file

  e.g.
  ----------------------------------------------------------------------------------------------------------------------
  from neurapy.neuralynx import lynxio
  import logging
  logging.basicConfig(level=logging.DEBUG)

  channels = 64
  fname = '/Users/kghose/Research/2013/Projects/Workingmemory/Data/NeuraLynx/2013-01-25_14-53-04/DigitalLynxRawDataFile.nrd'
  channel_list = [0,1,2]

  ftsname = 'timestamps.raw'
  fttlname = 'ttl.raw'
  fchanname = ['chan_{:000d}.raw'.format(ch) for ch in channel_list]
  lynxio.extract_nrd(fname, ftsname, fttlname, fchanname, channel_list, channels, max_pkts=1000)
  ----------------------------------------------------------------------------------------------------------------------

  Data are written as a pure stream of binary data and can be easily and efficiently read using the numpy read function.
  For convenience, a function that reads the timestamps, events and channels (read_extracted_data) is included in the library.

  In my experience STX, CRC, timestamp errors and garbage bytes between packets are extremely rare in a properly working system. This function eschews any kind of checks on the data read and just converts the packets. If you suspect that your data has dropped packets, crc or other issues you should try the regular version of this function. You can note if you have packet errors from your Cheetah software.

  Personally, I recommend using the _ec version of the code. It runs fast enough.
  """
    logger.info(
        'Notice: you are using the fast version of the extractor. No error checks are done'
    )

    #nrd packet format
    nrd_packet = pylab.dtype([
        ('stx', 'i'),
        ('pkt_id', 'i'),
        ('pkt_data_size', 'i'),
        ('timestamp high',
         'I'),  #Neuralynx timestamp is ... in its own 32 bit world
        ('timestamp low', 'I'),
        ('status', 'i'),
        ('ttl', 'I'),
        ('extra', '10i'),
        ('data', '{:d}i'.format(channels)),
        ('crc', 'i')
    ])
    #packet_size = nrd_packet.itemsize

    pkt_cnt = 0
    if max_pkts != -1:  #An insidious bug was killed here!
        if buffer_size > max_pkts:
            buffer_size = max_pkts

    #The files we will write to. fixme: test for properly opened?
    fts = open(ftsname, 'wb')
    fttl = open(fttlname, 'wb')
    fchan = [open(fcn, 'wb') for fcn in fchanname]

    with open(fname, 'rb') as f:
        hdr = read_header(f)
        logger.info('File header: {:s}'.format(hdr))

        #Read in 32bit increments until the magic number is found
        pkt = f.read(4)
        while len(pkt) == 4:
            if pkt == '\x00\x08\x00\x00':  #Magic number 2048 0x0800
                f.seek(-4, 1)  #Realign
                break
            pkt = f.read(4)

        these_packets = pylab.fromfile(f, dtype=nrd_packet, count=buffer_size)
        while these_packets.size > 0:
            ts = pylab.array((these_packets['timestamp high'] << 32) |
                             (these_packets['timestamp low']),
                             dtype='uint64')
            ts.tofile(fts)
            these_packets['ttl'].tofile(fttl)
            for idx, ch in enumerate(channel_list):
                these_packets['data'][:, ch].tofile(fchan[idx])

            pkt_cnt += these_packets.size
            if max_pkts != -1:
                if pkt_cnt >= max_pkts:  #NOTE: This may give us upto buffer_size -1 more packets than we want.
                    break
            these_packets = pylab.fromfile(f,
                                           dtype=nrd_packet,
                                           count=buffer_size)

    fts.close()
    fttl.close()
    [fch.close() for fch in fchan]

    logger.info('Extracted {:d} packets'.format(pkt_cnt))
Пример #17
0
def read_csc(fin, assume_same_fs=True):
  """Read a continuous record file. We return the raw packets but, in addition, if we set assume_same_fs as true we
  return a trace with all the data concatenated together, assuming that a constant sampling frequency was maintained
  through out. Gaps in the record are padded with zeros.
  Input:
    fin - file handle
    assume_same_fs - if True, concatenate any segments together, fill time gaps with zeros and return average Fs
  Ouput:
    Dictionary with fields
      'header' - the file header
      'packets' - the actual packets as read. This is a new pylab dtype with fields:
        'timestamp' - timestamp (us)
        'chan' - channel
        'Fs' - the sampling frequency
        'Ns' - the number of valid samples in the packet
        'samp' - the samples in the packet.
          e.g. x['packets']['samp'] will return a 2D array, number of packets long and 512 wide (since each packet carries 512 wave points)
          similarly x['packets']['timestamp'] will return an array number of packets long
      'Fs': the average frequency computed from the timestamps (can differ from the nominal frequency the device reports)
      'trace': the concatenated data from all the packets
      't0': the timestamp of the first packet.
  NOTE: while 'packets' returns the exact packets read, 'Fs' and 'trace' assume that the record has no gaps and that the
  sampling frequency has not changed during the recording
  """
  hdr = read_header(fin)
  csc_packet = pylab.dtype([
    ('timestamp', 'Q'),
    ('chan', 'I'),
    ('Fs', 'I'),
    ('Ns', 'I'),
    ('samp', '512h')
  ])

  data = pylab.fromfile(fin, dtype=csc_packet, count=-1)
  Fs = None
  trace = None
  if assume_same_fs:
    if data['Fs'].std() > 1e-6: #
      logger.warning('Fs is not fixed across trace, not packing packets together')
      assume_same_fs = False

  if not assume_same_fs: return {'header': hdr, 'packets': data}

  packet_duration_us = 512*(1./data['Fs'][0])*1e6
  #For the version we are dealing with, Neuralynx packets are always 512
  #This is actually a very poor estimate if the sampling freq is low, since it rounds to nearest Hz
  #So we'll not rely on this but come up with our own estimate

  samp = data['samp']
  ts_us = data['timestamp']
  dt_us = pylab.diff(ts_us).astype('f')
  idx = pylab.find(dt_us > packet_duration_us) #This will find any instances where we paused the recording
  if idx.size == 0:#No padding needed
    trace = samp.ravel()
    Fs = (data['Ns'][:-1]/(dt_us*1e-6)).mean()
  else: #We have some padding to do.
    logger.debug('Gaps in record, padding')
    #Our first task is to find all the contiguous sections of data
    idx += 1 #Shifting indexes to point at the packets that come after a gap
    idx = pylab.insert(idx, 0, 0) #Now idx contains the indexes of every packet that starts a contiguous section
    idx = pylab.append(idx,ts_us.size) #And the index of the last packet
    Ns = data['Ns']
    estimFs_sum = 0
    N_samps = 0
    sections = []
    for n in xrange(idx.size-1): #collect all the sections
      n0 = idx[n]; n1=idx[n+1]
      sections.append(samp[n0:n1].ravel())
      if n1-n0 > 1:#We need more than one packet in a section to get an estimate
        estimFs_sum += (Ns[n0:n1-1]/(dt_us[n0:n1-1]*1e-6)).sum()
        N_samps += n1-1-n0

    Fs = estimFs_sum / float(N_samps)
    #Now pad the data appropriately
    padded = [sections[0]]
    cum_N = sections[0].size
    for n in xrange(1,len(sections)):
      #Now figure out how many zeros we have to pad to get the right length
      Npad = int((ts_us[idx[n]] - ts_us[0])*1e-6*Fs - cum_N)
      padded.append(pylab.zeros(Npad))
      padded.append(sections[n])
      cum_N += Npad + sections[n].size
    trace = pylab.concatenate(padded) #From this packet to the packet before the gap

  return {'header': hdr, 'packets': data, 'Fs': Fs, 'trace': trace, 't0': ts_us[0]}
Пример #18
0
def extract_nrd_ec(fname, ftsname, fttlname, fchanname, channel_list, channels=64, max_pkts=-1, buffer_size=10000, error_bugout=1000000000):
  """Read and write out selected raw traces from the .nrd file with error checking.
  Inputs:
    fname - name of nrd file
    ftsname - name under which timestamp vector will be saved
    fttlname - name under which the events will be saved
    fchanname - a list of file names for the
    channel_list - Which AD channels to convert.
    channels - total channels in the system
    max_pkts - total packets to read. If set to -1 then read all packets
    buffer_size   - how many chunks to read at a time.
    error_bugout - If the sum of stx, crc and timestamp errors exceed this value quit reading the file
  Outputs:
    Data are written to file

  e.g.
  ----------------------------------------------------------------------------------------------------------------------
  from neurapy.neuralynx import lynxio
  import logging
  logging.basicConfig(level=logging.DEBUG)

  channels = 64
  fname = '/Users/kghose/Research/2013/Projects/Workingmemory/Data/NeuraLynx/2013-01-25_14-53-04/DigitalLynxRawDataFile.nrd'
  channel_list = [0,1,2]

  ftsname = 'timestamps.raw'
  fttlname = 'ttl.raw'
  fchanname = ['chan_{:000d}.raw'.format(ch) for ch in channel_list]
  lynxio.extract_nrd(fname, ftsname, fttlname, fchanname, channel_list, channels, max_pkts=1000)
  ----------------------------------------------------------------------------------------------------------------------

  Data are written as a pure stream of binary data and can be easily and efficiently read using the numpy read function.
  For convenience, a function that reads the timestamps, events and channels (read_extracted_data) is included in the library.

  """
  def seek_packet(f):
    """Skip forward until we find the STX magic number."""
    #Read in 32bit increments until the magic number is found
    start = f.tell()
    pkt = f.read(4)
    while len(pkt) == 4:
      if pkt == '\x00\x08\x00\x00': #Magic number 2048 0x0800
        f.seek(-4,1) #Realign
        break
      pkt = f.read(4)
    stop = f.tell()
    return stop - start

  logger.info('Notice: you are using the slow version of the extractor. All error checks are done')

  #nrd packet format
  nrd_packet = pylab.dtype([
    ('stx', 'i'),
    ('pkt_id', 'i'),
    ('pkt_data_size', 'i'),
    ('timestamp high', 'I'), #Neuralynx timestamp is ... in its own 32 bit world
    ('timestamp low', 'I'),
    ('status', 'i'),
    ('ttl', 'I'),
    ('extra', '10i'),
    ('data', '{:d}i'.format(channels)),
    ('crc', 'i')
  ])
  packet_size = nrd_packet.itemsize

  pkt_cnt = 0
  garbage_bytes = 0
  stx_err_cnt = 0
  pkt_id_err_cnt = 0
  pkt_size_err_cnt = 0
  pkt_ts_err_cnt = 0
  pkt_crc_err_cnt = 0

  if max_pkts != -1: #An insidious bug was killed here!
    if buffer_size > max_pkts:
      buffer_size = max_pkts

  #The files we will write to.
  fts = open(ftsname,'wb')
  fttl = open(fttlname,'wb')
  fchan = [open(fcn,'wb') for fcn in fchanname]

  last_ts = 0L
  with open(fname,'rb') as f:
    hdr = read_header(f)
    logger.info('File header: {:s}'.format(hdr))

    garbage_bytes += seek_packet(f)
    these_packets = pylab.fromfile(f, dtype=nrd_packet, count=buffer_size)
    while these_packets.size > 0:
      all_packets_good = True
      packets_read = these_packets.size

      idx = pylab.find(these_packets['stx'] != 2048)
      if idx.size > 0:
        stx_err_cnt += 1
        all_packets_good = False
        max_good_packets = idx[0]
        these_packets = these_packets[:max_good_packets]

      if these_packets.size > 0:
        idx = pylab.find(these_packets['pkt_id'] != 1)
        if idx.size > 0:
          pkt_id_err_cnt += 1
          all_packets_good = False
          max_good_packets = idx[0]
          these_packets = these_packets[:max_good_packets]

      if these_packets.size > 0:
        idx = pylab.find(these_packets['pkt_data_size'] != 10 + channels)
        if idx.size > 0:
          pkt_size_err_cnt += 1
          all_packets_good = False
          max_good_packets = idx[0]
          these_packets = these_packets[:max_good_packets]

      if these_packets.size > 0:
        #crc computation
        field32 = pylab.vstack([these_packets[k].T for k in nrd_packet.fields.keys()]).astype('I')
        crc = pylab.zeros(these_packets.size,dtype='I')
        for idx in xrange(field32.shape[0]):
          crc ^= field32[idx,:]
        idx = pylab.find(crc != 0)
        if idx.size > 0:
          pkt_crc_err_cnt += 1
          all_packets_good = False
          max_good_packets = idx[0]
          these_packets = these_packets[:max_good_packets]

      if these_packets.size > 0:
        ts = pylab.array((these_packets['timestamp high'].astype('uint64')<<32) | (these_packets['timestamp low']), dtype='uint64')
        bad_idx = -1
        if last_ts > ts[0]:#Time stamps out of order at buffer boundary
          bad_idx = 0
        else:
          idx = pylab.find(ts[:-1] > ts[1:])
          if idx.size > 0:
            bad_idx = idx[0] + 1
        if bad_idx > -1:
          logger.info('Out of order timestamp {:d}'.format(int(ts[bad_idx])))
          pkt_ts_err_cnt += 1
          all_packets_good = False
          max_good_packets = bad_idx
          these_packets = these_packets[:max_good_packets]
          ts = ts[:max_good_packets]

      if these_packets.size > 0:
        last_ts = ts[-1] #Ready for the next read
        ts.tofile(fts)
        these_packets['ttl'].tofile(fttl)
        for idx,ch in enumerate(channel_list):
          these_packets['data'][:,ch].tofile(fchan[idx])

      pkt_cnt += these_packets.size
      if max_pkts != -1:
        if pkt_cnt >= max_pkts: #NOTE: This may give us upto buffer_size -1 more packets than we want.
          break

      if not all_packets_good:
        f.seek((these_packets.size-packets_read)*packet_size+4,1) #Rewind all the way except 32 bits
        garbage_bytes += seek_packet(f)

      if pkt_ts_err_cnt + pkt_crc_err_cnt + stx_err_cnt > error_bugout:
        logger.warning('Too many errors, bugging out')
        break

      these_packets = pylab.fromfile(f, dtype=nrd_packet, count=buffer_size)

  fts.close()
  fttl.close()
  [fch.close() for fch in fchan]

  logger.info('Extracted {:d} packets'.format(pkt_cnt))
  logger.info('{:d} garbage words'.format(garbage_bytes))
  logger.info('{:d} packets had bad stx'.format(stx_err_cnt))
  logger.info('{:d} packets had bad pkt id'.format(pkt_id_err_cnt))
  logger.info('{:d} packets had bad crc'.format(pkt_crc_err_cnt))
  logger.info('{:d} packets had out of order timestamps'.format(pkt_ts_err_cnt))
Пример #19
0
def extract_nrd_fast(fname, ftsname, fttlname, fchanname, channel_list, channels=64, max_pkts=-1, buffer_size=10000):
  """Read and write out selected raw traces from the .nrd file.
  Inputs:
    fname - name of nrd file
    ftsname - name under which timestamp vector will be saved
    fttlname - name under which the events will be saved
    fchanname - a list of file names for the
    channel_list - Which AD channels to convert.
    channels - total channels in the system
    max_pkts - total packets to read. If set to -1 then read all packets
    buffer_size   - how many chunks to read at a time.
  Outputs:
    Data are written to file

  e.g.
  ----------------------------------------------------------------------------------------------------------------------
  from neurapy.neuralynx import lynxio
  import logging
  logging.basicConfig(level=logging.DEBUG)

  channels = 64
  fname = '/Users/kghose/Research/2013/Projects/Workingmemory/Data/NeuraLynx/2013-01-25_14-53-04/DigitalLynxRawDataFile.nrd'
  channel_list = [0,1,2]

  ftsname = 'timestamps.raw'
  fttlname = 'ttl.raw'
  fchanname = ['chan_{:000d}.raw'.format(ch) for ch in channel_list]
  lynxio.extract_nrd(fname, ftsname, fttlname, fchanname, channel_list, channels, max_pkts=1000)
  ----------------------------------------------------------------------------------------------------------------------

  Data are written as a pure stream of binary data and can be easily and efficiently read using the numpy read function.
  For convenience, a function that reads the timestamps, events and channels (read_extracted_data) is included in the library.

  In my experience STX, CRC, timestamp errors and garbage bytes between packets are extremely rare in a properly working system. This function eschews any kind of checks on the data read and just converts the packets. If you suspect that your data has dropped packets, crc or other issues you should try the regular version of this function. You can note if you have packet errors from your Cheetah software.

  Personally, I recommend using the _ec version of the code. It runs fast enough.
  """
  logger.info('Notice: you are using the fast version of the extractor. No error checks are done')

#nrd packet format
  nrd_packet = pylab.dtype([
    ('stx', 'i'),
    ('pkt_id', 'i'),
    ('pkt_data_size', 'i'),
    ('timestamp high', 'I'), #Neuralynx timestamp is ... in its own 32 bit world
    ('timestamp low', 'I'),
    ('status', 'i'),
    ('ttl', 'I'),
    ('extra', '10i'),
    ('data', '{:d}i'.format(channels)),
    ('crc', 'i')
  ])
  #packet_size = nrd_packet.itemsize

  pkt_cnt = 0
  if max_pkts != -1: #An insidious bug was killed here!
    if buffer_size > max_pkts:
      buffer_size = max_pkts

  #The files we will write to. fixme: test for properly opened?
  fts = open(ftsname,'wb')
  fttl = open(fttlname,'wb')
  fchan = [open(fcn,'wb') for fcn in fchanname]

  with open(fname,'rb') as f:
    hdr = read_header(f)
    logger.info('File header: {:s}'.format(hdr))

    #Read in 32bit increments until the magic number is found
    pkt = f.read(4)
    while len(pkt) == 4:
      if pkt == '\x00\x08\x00\x00': #Magic number 2048 0x0800
        f.seek(-4,1) #Realign
        break
      pkt = f.read(4)

    these_packets = pylab.fromfile(f, dtype=nrd_packet, count=buffer_size)
    while these_packets.size > 0:
      ts = pylab.array((these_packets['timestamp high']<<32) | (these_packets['timestamp low']), dtype='uint64')
      ts.tofile(fts)
      these_packets['ttl'].tofile(fttl)
      for idx,ch in enumerate(channel_list):
        these_packets['data'][:,ch].tofile(fchan[idx])

      pkt_cnt += these_packets.size
      if max_pkts != -1:
        if pkt_cnt >= max_pkts: #NOTE: This may give us upto buffer_size -1 more packets than we want.
         break
      these_packets = pylab.fromfile(f, dtype=nrd_packet, count=buffer_size)

  fts.close()
  fttl.close()
  [fch.close() for fch in fchan]

  logger.info('Extracted {:d} packets'.format(pkt_cnt))