def read_tdt_block(binPath, t1=None, t2=None, store=None, channel=None): """Wrapper arount tdt.read_block that avoids bug in the function. tdt.read_block returns "channel 1 not found in store" if first channel of a single-channel store. """ try: return tdt.read_block(binPath, t1=t1, t2=t2, store=store, channel=channel) except Exception as e: # tdt.read_block returns "channel 1 not found in store" if first channel # of a single-channel store. single_chan_store = tdt.read_block( binPath, t2=1.0, nodata=True ).streams[ store].data.ndim == 1 # data is 1-dim if there's one channel in store, 2 dim otherwise if single_chan_store and channel == 1: print(f"`{store}` have only 1 chan -> Contourn tdt.read_block bug") return tdt.read_block( binPath, t1=t1, t2=t2, store=store, channel=0) # Return "all" channels of single channel store raise e
def __init__(self, path, channels=None): self.path = path self.channels = channels if channels is None: self.tdt_obj = tdt.read_block(path) else: self.tdt_obj = tdt.read_block(path, channel=channels) self.streams = self.get_streams() self.block_name = self.tdt_obj['info']['blockname'] self.start_time = self.tdt_obj['info']['utc_start_time'] logger.debug('Streams: ' + ', '.join(self.streams))
def run_conversion( self, block_path, electrode_table_path, session_id, session_description="test read", ): # Converts TDT block to NWB data # read in TDT file print("Reading {} at {}".format(session_id, block_path)) tdt_data = tdt.read_block(block_path) # initialize nwb file, requires a session description, id, and start time self.nwb = NWBFile( session_description=session_description, # required identifier=session_id, # required session_start_time=tdt_data.info.start_date, ) electrode_group = self.create_electrode_devices() electrode_df = self.read_electrode_table(electrode_table_path) self.populate_electrode_tables(electrode_df, electrode_group) electrode_table_region = self.create_electrode_region(electrode_df) self.add_electrical_series(electrode_table_region, tdt_data) self.add_audio(tdt_data) self.write_nwb("{}.nwb".format(session_id))
def __extract_from_tdt_files(self, device_name, dev_conf): tdt_path = self.tdt_path header = tdt.read_block(tdt_path, headers=1) tdt_tmp = tdt.read_block(tdt_path, channel=1) num_channels = len(header.stores[device_name].chan) num_samples = len(tdt_tmp.streams[device_name].data) # Iteratively write one channel at a time data = DataChunkIterator(data=self.__iter_tdt_channels( tdt_path, device_name, header, num_channels), maxshape=(num_samples, num_channels), buffer_size=BUFFER_SIZE) tdt_params = {} tdt_params['start_time'] = header.start_time[ 0] #returning numeric since that's what nwb wants for an eseries tdt_params['stop_time'] = header.stop_time[0] #tdt_params['stop_time'] = dt.datetime.fromtimestamp(header.stop_time[0]) tdt_params['sample_rate'] = header.stores[device_name].fs return data, tdt_params
def get_tdt_object(self): ''' gets TDT meta data object Returns: - tdt_object: (struct) TDT meta data object ''' tdt_object = tdt.read_block(self.tdt_path, headers=1) return tdt_object
def make_ppp_video_dict(rat, diet, box, tank, cam, vidfile, posfile, scorer, cas_ttl, malt_ttl, casxy, maltxy): """ Function to make dictionary with position and cue information for rats in PPP experiment. Parameters ---------- tank : String Path to TDT tank. posfile : String Path to hd5 file generated by DeepLabCut. scorer : String Scorer string from DeepLabCut file. Can be generated automatically in future. cas_ttl : String TTL for casein trial times from TDT file. malt_ttl : String TTL for maltodextring trial times. casxy : Tuple Position of casein cue light. maltxy : Tuple Position of maltodextrin cue light. Returns ------- data : Dictionary Includes position data, cue data and other information needed for further analysis. """ epocs = (tdt.read_block(tank, evtype=['epocs'])).epocs cap = cv.VideoCapture(vidfile) fps = int(cap.get(cv.CAP_PROP_FPS)) success, image = cap.read() print(success, vidfile) cv.imwrite(Path(vidfile).stem + ".jpg", image) cap.release() df = pd.read_hdf(posfile) return { 'rat': rat, 'diet': diet, 'box': box, 'nose': df[scorer, 'nose'], 'L_ear': df[scorer, 'L_ear'], 'R_ear': df[scorer, 'R_ear'], 'fps': fps, 'cas_cue': getattr(epocs, cas_ttl).onset, 'malt_cue': getattr(epocs, malt_ttl).onset, 'cas_pos': casxy, 'malt_pos': maltxy }
def loaddata(tdtfile, SigBlue, SigUV): tdtfile=raw_datafolder+tdtfile try: tmp = tdt.read_block(tdtfile, evtype=['streams'], store=[SigBlue]) data = getattr(tmp.streams, SigBlue)['data'] fs = getattr(tmp.streams, SigBlue)['fs'] tmp = tdt.read_block(tdtfile, evtype=['streams'], store=[SigUV]) dataUV = getattr(tmp.streams, SigUV)['data'] ttls = tdt.read_block(tdtfile, evtype=['epocs']).epocs except: print('Unable to load data properly.') data=[] dataUV=[] ttls=[] fs=[] return data, dataUV, fs, ttls
def __iter_tdt_channels(self, tdt_path, stream, header, num_channels): #this is inefficient because it loads all streams. As far as I can tell, #there's no way to select streams to load for ch in range(num_channels): tdt_ch = ch + 1 tdt_struct = tdt.read_block(tdt_path, channel=tdt_ch, headers=header, evtype=['streams']) ch_data = tdt_struct.streams[stream].data ch_data = np.reshape( 1, -1 ) #weird quirk to make data samples x channels since it's tranposed downstream yield ch_data return
def load_stimulation_data(path): all_data = {} block_names = os.listdir(path) for block in block_names: full_path_tdt = os.path.join(path, block) if os.path.isdir(full_path_tdt) and "Block" in block: data = tdt.read_block(full_path_tdt, evtype=['epocs', 'snips', 'scalars'], nodata=1) block_number = int(block.split('-')[1]) all_data[block_number] = { "epocs": data.epocs, "snips": data.snips, "eNeu": data.snips.eNeu } return all_data
def get_data(blocks, channels): # extracts data from TDT tank and stores it in a MultiIndex pandas dataframe dframe = [] for i in range(len(blocks)): for c in channels: print(i) print(c) if i == 25 or i == 371 or i == 382 or i == 436 or i == 443 or i == 444 or i == 466 or i == 666: # skipping a weird block continue data = read_block(blocks[i], evtype=['streams', 'epocs'], channel=c) nblock = int(data.info.blockname.partition("-")[2]) # gets block # nepocs = pd.DataFrame(data.epocs).columns.values.tolist() if 'PROT' in nepocs: nepocs.remove('PROT') # df = pd.DataFrame([nepocs]).T # df.columns = [nblock] # dframe.merge(df) epoc_list = ['TLvl', 'FDur', 'vdur', 'Nois'] # use epoc_list to automate PARAM1 selection PARAM1 = next(iter(set(nepocs).intersection(epoc_list))) number_epocs = len(nepocs) filtered_data = epoc_filter(data, PARAM1, t=TRANGE) all_signals = pd.DataFrame( data=extract_lfp(filtered_data, ARTIFACT)) tankname = filtered_data.info.tankpath.split()[4].partition( "/")[2].partition("_") all_signals['catname'] = tankname[0] all_signals['tanknumber'] = tankname[2] all_signals['channel'] = c all_signals['blockname'] = str_blocknames(nblock, PARAM1) for e in range(len(nepocs)): all_signals[str(nepocs[e])] = data.epocs[nepocs[e]].data col_names = [ x for x in all_signals.columns.values.tolist() if not isinstance(x, int) ] # df['column_new_1'], df['column_new_2'], df['column_new_3'] = [np.nan, 'dogs', 3] id_signals = all_signals.set_index( col_names) * 1e6 #index df and data in microvolts dframe.append(id_signals) df = pd.concat(dframe, ignore_index=False) return df
def extract_data_object_photometry_raw_traces(Mouse_Date_FileName,path_to_import,path_to_save): l_mouse = list(Mouse_Date_FileName["mouse"]) l_photo_day = list(Mouse_Date_FileName["file name"]) raw_photo_data_per_trial_channels_data_set = [] counter = 1 numer_of_sessions_dataset = len(l_mouse) for (mouse, photo_day) in zip(l_mouse, l_photo_day): print ("session number"+" "+str(counter)+" "+"was imported (out of"+" "+str(numer_of_sessions_dataset)+")") photometry_data = tdt.read_block(path_to_import+"/"+mouse+"/"+photo_day) photometry_data = reading_in_photometryData_all_Channels(photometry_data) raw_photo_data_per_trial_channels_data_set.append(photometry_data) counter += 1 return raw_photo_data_per_trial_channels_data_set
def __extract_stream_data(self, device_name, dev_conf): ''' extracts TDT stream Args: - device_name: (str) stream name (eg. 'ECoG' or 'Poly') - dev_conf: (dict) metadata for the device. nwb_builder.metadata['device'][device_name] Returns: - data: (numpy array) [samples] by [channels] - parameters (dict) includes stream meta data ''' tdt_struct = tdt.read_block(self.tdt_path, store=device_name) stream = tdt_struct.streams[device_name] data = stream.data.T parameters = { 'start_time': stream.start_time, 'sample_rate': stream.fs } return data, parameters
print(read_sev.__doc__) doc.add_text(stdoutput) # handout: exclude doc.show() # handout: exclude """ Download demo data from the TDT website """ download_demo_data() """ This example uses our [example data sets](https://www.tdt.com/files/examples/TDTExampleData.zip). To import your own data, replace BLOCK_PATH with the full path to your own data block. In Synapse, you can find the block path in the database. Go to Menu > History. Find your block, then Right-Click > Copy path to clipboard. """ BLOCK_PATH = 'data/Algernon-180308-130351' data = read_block(BLOCK_PATH) print(data) doc.add_text(stdoutput) # handout: exclude doc.show() # handout: exclude """ **read_block** returns a structured object. It is a Python dictionary but also allows you to use the dot syntax like in Matlab, so you can access fields within the structure with either method. These two ways of looking at the block info field are equivalent: """ print(data.info) doc.add_text(stdoutput) # handout: exclude doc.show() # handout: exclude print(data['info']) doc.add_text(stdoutput) # handout: exclude doc.show() # handout: exclude """ These three methods to access the 'Wav1' store sampling rate are equivalent:
def AddTDTRawDataToNWB(tdt_block_dir, nwb_file_name='', verbose=False, elec_ids=None): """ Copies raw electrophysiological data from TDT format to Neurodata Without Borders (NWB) format. Typically, the NWB file will already be initialized by calling InitializeNWBFromTDT. Multiple electrodes can be added at once or with separate calls to this function. :param tdt_block_dir: {str} full path of TDT files to convert to NWB. If empty, will open dialog. :param nwb_file_name: [optional] {str} full path of NWB file to export to. Default is to change blackrock extension to 'nwb' :param verbose: [optional] {bool} whether to print updates while converting. Default is false. :param elec_ids: [optional] {list} List of electrode IDs to copy over. If empty, all are copied :return: {str} filename of NWB file (empty if error) """ # Check to see if user specified a TDT filename if not tdt_block_dir: # no file name passed # Ask user to specify a file if 'app' not in locals(): app = QApplication([]) tdt_block_dir = QFileDialog.getExistingDirectory( QFileDialog(), 'Select Directory', getcwd()) # Check to see if valid nwb_file_name is passed tdt_tsq_files = [ f for f in listdir(tdt_block_dir) if path.splitext(f)[1] == '.tsq' ] if not nwb_file_name: nwb_file_name = path.splitext(tdt_tsq_files[0])[0] + '.nwb' if verbose: print("Writing to NWB data file %s" % (nwb_file_name)) # Initialize the TDT file try: # Requires the raw data to be imported tdt_header = tdt.read_block(tdt_block_dir, headers=1) except: # catch *all* exceptions e = sys.exc_info()[0] raise FileNotFoundError( "Couldn't open TDT file. Error: {:s}".format(e)) # Initialize the NWB file nwb_io = [] try: if not path.isfile(nwb_file_name): # Initialize NWB file if verbose: print("NWB file doesn't exist. Creating new one: %s..." % (nwb_file_name)) InitializeNWBFromTDT(tdt_block_dir, nwb_file_name, verbose=verbose) # Append to existing file if verbose: print("Opening NWB file %s..." % (nwb_file_name), end='') nwb_file_append = True nwb_io = NWBHDF5IO(nwb_file_name, mode='a') nwb_file = nwb_io.read() if verbose: print("done.") except: # catch *all* exceptions e = sys.exc_info()[0] if nwb_io: nwb_io.close() raise FileExistsError("Couldn't open NWB file. Error: %s" % e) # Validate the elec_ids list if not elec_ids: # Grab electroe list from NWB file elec_ids = nwb_file.electrodes.id[:] # Load data for all of the electrodes if verbose: print("Loading raw data from TDT file...") chan_data = [] starting_time = 0 fs = 24414.0625 for cur_elec_id in elec_ids: # Convert electrode id into RAW and channel cur_raw = np.floor((cur_elec_id - 1) / 16) + 1 cur_raw = 'RAW{:1.0f}'.format(cur_raw) cur_chan = ((cur_elec_id - 1) % 16) + 1 print('{:s}_{:1.0f}'.format(cur_raw, cur_chan)) #Read this channel data in from file cur_chan_data = tdt.read_sev(tdt_block_dir, channel=cur_chan, event_name=cur_raw) fs = cur_chan_data[cur_raw].fs starting_time = cur_chan_data.time_ranges[0][0] cur_chan_data = cur_chan_data[cur_raw].data cur_chan_data = np.reshape(cur_chan_data, [len(cur_chan_data), 1]) if len(chan_data) == 0: chan_data = cur_chan_data else: chan_data = np.concatenate([chan_data, cur_chan_data], axis=1) if verbose: print("\tLoaded electrode %d..." % (cur_elec_id)) # Get resolution of signal in V if verbose: print("Processing electrode information.") V_resolution = 0.0 #TODO: See if we can get resolution information from TDT? # Create electrode table for new electrodes elec_table_ind = np.ones((len(elec_ids), 1)) * np.NaN for cur_elec_ind, cur_elec in enumerate(elec_ids): # In order to create electrode table, we have to have indexes for each channel into electrode list in NWB file cur_elec_table_ind = 0 while (cur_elec_table_ind < len(nwb_file.electrodes)) and ( nwb_file.electrodes[cur_elec_table_ind, 0] != cur_elec): cur_elec_table_ind = cur_elec_table_ind + 1 if cur_elec_table_ind >= len(nwb_file.electrodes): raise ValueError("Couldn't find electrode %d in NWB file list." % (cur_elec)) elec_table_ind[cur_elec_ind] = cur_elec_table_ind elec_table_ind = elec_table_ind.transpose().tolist()[ 0] # Convert to list for create_electrode_table_region electrode_table_region = nwb_file.create_electrode_table_region( elec_table_ind, "Electrodes %s" % (elec_ids)) # Write raw data for the electrode if verbose: print("\tAdding to NWB...", end='') ephys_ts = ElectricalSeries( 'Raw Data for Channels %s' % (elec_ids), chan_data, electrode_table_region, starting_time=starting_time, rate=fs, resolution=V_resolution, conversion=float( 1.0), #TODO: Check that TDT does read in V, as expected by NWB comments="", description="Raw data from TDT file.") nwb_file.add_acquisition(ephys_ts) if verbose: print("done") # Write the file if verbose: print("\tWriting NWB file and closing.") nwb_io.write(nwb_file) nwb_io.close() return nwb_file_name
def InitializeNWBFromTDT(tdt_block_dir, nwb_file_name='', experiment=None, subject=None, electrode_info=None, electrode_group=None, notes=None, overwrite=False, verbose=False): """ Initializes a NWB file to copy over data from Blackrock file. User specified information is added to NWB file as metadata about data. :param tdt_block_dir: {str} full path of TDT files to convert to NWB. If empty, will open dialog. :param nwb_file_name: [optional] {str} full path of NWB file to export to. Default is to change TDT extension to 'nwb' :param verbose: [optional] {bool} whether to print updates while converting. Default is false. :param experiment: [optional] {dict} Dictionary that contains information about experimenter, name, animal, lab, and institution :param subject: [optional] {dict} Dictionary that contains information about subject :param electrode_info: [optional] {list} List of dictionaries carrying information about electrodes :param electrode_group: [optional] {list} List of dictionaries carrying information about electrode groupings :param notes: [optional] {str} Notes relevant for the dataset :param overwrite: [optional] {bool} Whether to overwrite the NWB file if it already exists :return: {str} filename of NWB file (empty if error) """ # Check to see if user specified a TDT filename if not tdt_block_dir: # no file name passed # Ask user to specify a file if 'app' not in locals(): app = QApplication([]) tdt_block_dir = QFileDialog.getExistingDirectory( QFileDialog(), 'Select Directory', getcwd()) # Check to see if valid nwb_file_name is passed tdt_tsq_files = [ f for f in listdir(tdt_block_dir) if path.splitext(f)[1] == '.tsq' ] if not nwb_file_name: nwb_file_name = tdt_block_dir + path.splitext( tdt_tsq_files[0])[0] + '.nwb' if verbose: print("Writing to NWB data file %s" % (nwb_file_name)) # Initialize the TDT file, get header and basic information try: # Requires the raw data to be imported tdt_header = tdt.read_block(tdt_block_dir, headers=1) tdt_data = tdt.read_block(tdt_block_dir, evtype=['epocs']) except: # catch *all* exceptions e = sys.exc_info()[0] raise FileNotFoundError("Couldn't open TDT file. Error: %s" % e) # Process experiment information from inputs if (not experiment) or not (type(experiment) is dict): if verbose: print("Using default experimental details.") nwb_session_description = 'Experimental details not provided' experimenter = '' lab = '' institution = '' else: if 'name' not in experiment.keys(): experiment['name'] = '' if 'animal' not in experiment.keys(): experiment['animal'] = '' if 'experimenter' not in experiment.keys(): experiment['experimenter'] = '' if 'lab' not in experiment.keys(): experiment['lab'] = '' if 'institution' not in experiment.keys(): experiment['institution'] = '' experimenter = experiment['experimenter'] lab = experiment['lab'] institution = experiment['institution'] nwb_session_description = 'Experiment: ' + experiment['name'] + \ ', Animal: ' + experiment['animal'] + \ ', Date: ' + tdt_data.info.start_date.strftime('%m/%d/%Y') # description of the recording session if verbose: print("Experiment description: %s" % (nwb_session_description)) # Process subject information if (not subject) or not (type(subject) is dict): cur_subject = None else: try: # Create a subject object from the passed dictionary cur_subject = file.Subject(**subject) except: # catch *all* exceptions e = sys.exc_info()[0] raise ValueError("Couldn't create subject object. Error: %s" % (e)) # Define the NWB file identifier as the filename, as this should be unique nwb_identifier = path.split(tdt_tsq_files[0])[1] # unique file identifier # Initialize the NWB file nwb_io = [] try: if not overwrite and path.isfile(nwb_file_name): raise FileExistsError( "Can't update basic attributes of NWB file. Create new NWB file if you need to change attributes." ) else: if verbose: print("Creating new NWB file %s..." % (nwb_file_name), end='') nwb_file_append = False nwb_io = NWBHDF5IO(nwb_file_name, mode='w') tdt_notes = '' if 'Note' in tdt_data.epocs.keys(): for cur_note in tdt_data.epocs.Note: tdt_notes = tdt_notes + cur_note.notes + '; ' nwb_file = NWBFile(nwb_session_description, nwb_identifier, tdt_data.info.start_date, file_create_date=datetime.now(tzlocal()), experimenter=experimenter, subject=cur_subject, lab=lab, institution=institution, source_script='ConvertTDTToNWB.py', source_script_file_name='ConvertTDTToNWB.py', notes=notes + tdt_notes) if verbose: print("done.") except: # catch *all* exceptions e = sys.exc_info()[0] if nwb_io: nwb_io.close() tdt_data.clear() raise FileExistsError("Couldn't open NWB file. Error: %s" % e) if verbose: print("Created NWB file.") # Create device in the NWB file device = nwb_file.create_device(name='TDT') if verbose: print("Created device.") # Make sure electrode input is provided and valid if (not electrode_info) or not (type(electrode_info) is list): if verbose: print("Creating electrode information from directory.") # Grab information about SEV files tdt_sev_files = [ f for f in listdir(tdt_block_dir) if path.splitext(f)[1] == '.sev' ] electrode_list = [] for sev_ind, sev_file in enumerate(tdt_sev_files): sev_match = re.search(".+_RAW(\\d+)_Ch(\\d+).sev", sev_file) electrode_list.append((int(sev_match.groups()[0]) - 1) * 16 + int(sev_match.groups()[1])) electrode_list = np.sort(electrode_list) # Create electrode information for each electrode electrode_info = [] for cur_elec in electrode_list: electrode_info.append({ 'x': 0.0, 'y': 0.0, 'z': 0.0, 'impedance': -1.0, 'location': 'unknown', 'group': 'electrodes', 'id': cur_elec, 'filtering': 'Unknown' }) if verbose: print("\tCreated %d electrodes." % (len(electrode_info))) # Make sure electrode group input is provided and valid if verbose: print("Processing electrode groups.") default_electrode_group = { 'name': 'default electrode group', 'description': 'Generic electrode group for ungrouped electrodes.', 'location': 'Unknown' } if (not electrode_group) or not (type(electrode_group) is list): electrode_group = [default_electrode_group] else: electrode_group.insert(0, default_electrode_group) # Fill in any missing information about electrodes ovr_elec_group_list = [ ] # will be used to keep track of electrode group list if verbose: print("Processing electrode information.") for cur_elec_info_ind, cur_elec_info in enumerate(electrode_info): # If id is not provided, then throw an error if ('id' not in cur_elec_info.keys()) or (not cur_elec_info['id']): tdt_data.clear() nwb_io.close() raise ValueError( "Couldn't process electrode inputs. Must provide IDs for all electrodes or pass empty array." ) # If filtering is not provided, add it from the file (can't with TDT) if ('filtering' not in cur_elec_info.keys()) or ( not cur_elec_info['filtering']): electrode_info[cur_elec_info_ind]['filtering'] = 'Unknown' # If other variables are not provided, just initialize them to defaults if ('x' not in cur_elec_info.keys()) or (not cur_elec_info['x']): electrode_info[cur_elec_info_ind]['x'] = 0.0 if ('y' not in cur_elec_info.keys()) or (not cur_elec_info['y']): electrode_info[cur_elec_info_ind]['y'] = 0.0 if ('z' not in cur_elec_info.keys()) or (not cur_elec_info['z']): electrode_info[cur_elec_info_ind]['z'] = 0.0 if ('location' not in cur_elec_info.keys()) or ( not cur_elec_info['location']): electrode_info[cur_elec_info_ind]['location'] = 'unknown' if ('impedance' not in cur_elec_info.keys()) or ( not cur_elec_info['impedance']): electrode_info[cur_elec_info_ind]['impedance'] = float(-1.0) if ('group' not in cur_elec_info.keys()) or (not cur_elec_info['group']): electrode_info[cur_elec_info_ind][ 'group'] = 'default electrode group' # Find the index to the electrode group grp_elec_ind = 0 while (grp_elec_ind < len(electrode_group)) and ( cur_elec_info['group'] != electrode_group[grp_elec_ind]['name']): grp_elec_ind = grp_elec_ind + 1 # If we made it past the end, this electrode group doesn't exist, put it in the default if grp_elec_ind >= len(electrode_group): grp_elec_ind = -1 # Save the index into group dictionary for easy reference electrode_info[cur_elec_info_ind][ 'electrode_group_index'] = grp_elec_ind ovr_elec_group_list.append(grp_elec_ind) # Create electrode groups nwb_electrode_group = [] if verbose: print("Creating electrode groups.") for cur_elec_group_ind, cur_elec_group in enumerate(electrode_group): # Create and add to our list nwb_electrode_group.append( nwb_file.create_electrode_group( cur_elec_group['name'], description=cur_elec_group['description'], location=cur_elec_group['location'], device=device)) if verbose: print("\tCreated %d electrode groups." % (len(electrode_group))) # Create electrodes in NWB file if verbose: print("Adding electrodes to NWB.") nwb_file_elec_list = [] for cur_elec_ind, cur_elec in enumerate(electrode_info): # Add electrode to NWB file nwb_file.add_electrode( id=cur_elec['id'], x=cur_elec['x'], y=cur_elec['y'], z=cur_elec['z'], imp=cur_elec['impedance'], location=cur_elec['location'], filtering=cur_elec['filtering'], group=nwb_electrode_group[cur_elec['electrode_group_index']]) # Keep track of electrodes entered so we can index for the electrode table nwb_file_elec_list.append(cur_elec['id']) if verbose: print("\tAdded %d electrodes." % (len(electrode_info))) # Close NSx file tdt_data.clear() # Write to file and close if verbose: print("\tWriting NWB file and closing.") nwb_io.write(nwb_file) nwb_io.close() return nwb_file_name
def AddTDTAnalogDataToNWB(tdt_block_dir, nwb_file_name='', signal_info=None, module_name='behavior', verbose=False): """ Copies analog (continuous) data from the specified Blackrock file to Neurodata Without Borders (NWB) file. This is usually continuous signals about behavior (joystick position, screen refreshes, etc). User should provide information about the signals to help end users understand what is in each signal. Multiple calls can be used to load multiple data signals from multiple files. Typically, the NWB file will already be initialized by calling InitializeNWBFromBlackrock. :param tdt_block_dir: {str} full path of TDT files to convert to NWB. If empty, will open dialog. :param nwb_file_name: [optional] {str} full path of NWB file to export to. Default is to change blackrock extension to 'nwb' :param signal_info: [optional] {list} List of dictionaries with information about the signals to save. :param module_name: [optional] {str} Name of module to store data. Usually 'behavior' but could also be 'ecephys' or 'misc' :param verbose: [optional] {bool} whether to print updates while converting. Default is false. :return: {str} filename of NWB file (empty if error) """ # Check to see if user specified a TDT filename if not tdt_block_dir: # no file name passed # Ask user to specify a file if 'app' not in locals(): app = QApplication([]) tdt_block_dir = QFileDialog.getExistingDirectory( QFileDialog(), 'Select Directory', getcwd()) # Check to see if valid nwb_file_name is passed tdt_tsq_files = [ f for f in listdir(tdt_block_dir) if path.splitext(f)[1] == '.tsq' ] if not nwb_file_name: nwb_file_name = path.splitext(tdt_tsq_files[0])[0] + '.nwb' if verbose: print("Writing to NWB data file %s" % (nwb_file_name)) # Initialize the TDT file try: # Requires the raw data to be imported tdt_header = tdt.read_block(tdt_block_dir, headers=1) except: # catch *all* exceptions e = sys.exc_info()[0] raise FileNotFoundError( "Couldn't open TDT file. Error: {:s}".format(e)) # Initialize the NWB file nwb_io = [] try: if not path.isfile(nwb_file_name): # Initialize NWB file if verbose: print("NWB file doesn't exist. Creating new one: %s..." % (nwb_file_name)) InitializeNWBFromTDT(tdt_block_dir, nwb_file_name, verbose=verbose) # Append to existing file if verbose: print("Opening NWB file %s..." % (nwb_file_name), end='') nwb_file_append = True nwb_io = NWBHDF5IO(nwb_file_name, mode='a') nwb_file = nwb_io.read() if verbose: print("done.") except: # catch *all* exceptions e = sys.exc_info()[0] if nwb_io: nwb_io.close() raise FileExistsError("Couldn't open NWB file. Error: %s" % e) # Make sure module name is either behavior or misc module_name = module_name.lower() if (module_name != 'behavior') and (module_name != 'misc'): raise ValueError("Module type must either be 'behavior' or 'misc'.") # Parse the signal_info list if not signal_info: raise ValueError("Must specify signals to load.") elec_ids = [] for cur_signal_ind, cur_signal_info in enumerate(signal_info): if 'label' not in cur_signal_info.keys(): raise ValueError( "Signal information must have a label for each signal.") if 'name' not in cur_signal_info.keys(): raise ValueError( "Signal information must have a name for each signal. (Should be user-understandable)" ) if 'comments' not in cur_signal_info.keys(): signal_info[cur_signal_ind]['comments'] = '' # Find electrode IDs for this signal if ('elec_id' not in cur_signal_info.keys()) or ( not cur_signal_info['elec_id']): # Loop through and grab all signals of type 'streams' that aren't RAW data signal_info[cur_signal_ind]['elec_id'] = [] for cur_store in tdt_header.stores.keys(): # Grab all 'streams' but ignore RAWs if (tdt_header.stores[cur_store].type_str == 'streams') and (cur_store[0:3] != 'RAW'): signal_info[cur_signal_ind]['elec_id'].append(cur_store) # Create processing module for saving data if module_name not in nwb_file.processing.keys(): if verbose: print( "Specified processing module (%s) does not exist. Creating." % (module_name)) signal_module = ProcessingModule( name=module_name, description="Processing module for continuous signal data from %s." % (path.split(tdt_tsq_files[0])[1])) nwb_file.add_processing_module(signal_module) # Create data interface for the analog signals signal_info_str = signal_info[0]['name'] for i in range(1, len(signal_info)): signal_info_str = signal_info_str + ", " + signal_info[i]['name'] if verbose: print("Creating %s data interface for signals %s." % (module_name, signal_info_str)) if module_name == 'behavior': cur_data_interface = BehavioralTimeSeries(name="Analog signals (" + signal_info_str + ")") elif module_name == 'misc': cur_data_interface = AbstractFeatureSeries(name="Analog signals (" + signal_info_str + ")") else: raise ValueError("Module type must either be 'behavior' or 'misc'.") if verbose: print("Adding signals...") for cur_signal_ind, cur_signal_info in enumerate(signal_info): # Get data from file analog_data = [] analog_fs = [] analog_start_time = [] for cur_elec_id in cur_signal_info['elec_id']: cur_data = tdt.read_block(tdt_block_dir, store=cur_elec_id) cur_analog_data = cur_data.streams[cur_elec_id].data cur_analog_data = np.reshape(cur_analog_data, [len(cur_analog_data), 1]) if len(analog_data) == 0: analog_fs = cur_data.streams[cur_elec_id].fs analog_start_time = cur_data.streams[cur_elec_id].start_time analog_data = cur_analog_data else: analog_fs.append(cur_data.streams[cur_elec_id].fs) analog_start_time.append( cur_data.streams[cur_elec_id].start_time) analog_data = np.concatenate([analog_data, cur_analog_data], axis=1) # Make sure all of the fs and start_times are the same analog_start_time = np.unique(analog_start_time) analog_fs = np.unique(analog_fs) if len(analog_start_time) != 1 or len(analog_fs) != 1: raise ValueError( 'Start time and sampling frequency need to be the same for signals to be combined.' ) # Create time series cur_data_interface.create_timeseries( name=cur_signal_info['name'], data=analog_data, comments=cur_signal_info['comments'], unit="V", #TODO: Check that this is correct for TDT resolution=1.0, #TODO: Can we get this from TDT? conversion= 0.001, #TODO: Check what the correct conversion is for TDT starting_time=analog_start_time[0], rate=analog_fs[0], description="Signal %s from %s." % (cur_signal_info['label'], path.split(tdt_tsq_files[0])[1])) if verbose: print("\tAdded %s." % (cur_signal_info['label'])) # Add data interface to module in NWB file if verbose: print("Adding data interface to module.") nwb_file.processing[module_name].add(cur_data_interface) # Write the file if verbose: print("Writing NWB file and closing.") nwb_io.write(nwb_file) nwb_io.close() return nwb_file_name
fid.write(os.linesep) fid.close() print("Closing...") # Extract video timing information print("trying to get timing for video...") # OLD VERSION #tank = '\\'.join(re.split('\\\\', params['tank'])[0:-1]) #block = re.split('\\\\', params['tank'])[-1] #import win32com.client #h = win32com.client.Dispatch('matlab.application') #cmd = "data = TDTbin2mat(fullfile('%s' ,'%s'),'Type',{'epocs'}); onset=data.epocs.Cam1.onset; offset=data.epocs.Cam1.offset; save(fullfile('%s', '%s','video_timing.mat'), 'onset', 'offset')" % (tank,block, tank,block) #h.Execute(cmd) # END [OLD VERSION] #################### tank = params['tank'] ep_data = tdt.read_block(tank, evtype=['epocs']) onset = ep_data.epocs.Cam1.onset offset = ep_data.epocs.Cam1.offset so.savemat(os.path.join(tank, 'video_timing.mat'), { 'onset': onset, 'offset': offset }) # extract movie frames ffmpeg_path = 'C:\\Users\ChungWeberPC_04\\Documents\\Ground\\ffmpeg-20180227-fa0c9d6-win64-static\\bin\\ffmpeg' video_file = [ f for f in os.listdir(params['tank']) if re.match('^.*\.avi$', f) ][0] if not (os.path.isdir(os.path.join(params['tank'], 'Stack'))): os.mkdir(os.path.join(params['tank'], 'Stack'))
Set up the varibles for the data you want to extract. We will extract channel 1 from the eNe1 snippet data store, created by the PCA Sorting gizmo, and use our PulseGen epoc event `PC0/` as our stimulus onset. """ REF_EPOC = 'PC0/' SNIP_STORE = 'eNe1' SORTID = 'TankSort' CHANNEL = 3 SORTCODE = 0 # set to 0 to use all sorts TRANGE = [-0.3, 0.8] """ Now read the specified data from our block into a Python structure. The `nodata` flag means that we are only intereseted in the snippet timestamps, not the actual snippet waveforms in this example. """ data = read_block(BLOCK_PATH, evtype=['epocs', 'snips', 'scalars'], sortname=SORTID, channel=CHANNEL, nodata=1) """ ## Use epoc_filter to extract data around our epoc event Using the `t` parameter extracts data only from the time range around our epoc event. """ raster_data = epoc_filter(data, REF_EPOC, t=TRANGE) """ Adding the `tref` flag makes all of the timestamps relative to the epoc event, which is ideal for generating histograms. """ hist_data = epoc_filter(data, REF_EPOC, t=TRANGE, tref=1) """ And that's it! Your data is now in Python. The rest of the code is a simple plotting example. First, we'll find matching timestamps for our selected sort code (unit).
download_demo_data() BLOCK_PATH = 'data/Subject1-180426-120951' """ ### Set up the variables for the data you want to extract. We will extract channel 1 from the EEG1 stream data store. """ STORE = 'EEG1' CHANNEL = 1 ONSET = [-3] # relative onset, in seconds, from the note timestamp """ Now read the specified data from our block into a Python structure """ data = read_block(BLOCK_PATH, channel=CHANNEL) """ All user notes are stored in a special epoc event called 'Note' """ # find all the unique note values notes, counts = np.unique(data.epocs.Note.notes, return_counts=True) # find the highest number of occurrences (to inform our plot) maxOccur = np.max(counts) """ ### Loop through the notes for plotting """ # some useful variables num_notes = len(notes)
# import the primary functions from the tdt library only from tdt import read_block, download_demo_data """ ### Importing the Data This example uses our [example data sets](https://www.tdt.com/files/examples/TDTExampleData.zip). To import your own data, replace BLOCK_PATH with the full path to your own data block. In Synapse, you can find the block path in the database. Go to Menu > History. Find your block, then Right-Click > Copy path to clipboard. """ download_demo_data() BLOCK_PATH = 'data/Algernon-180308-130351' """ Now read all epocs and just channel 1 from all stream data into a Python structure called 'data' """ data = read_block(BLOCK_PATH, evtype=['streams', 'epocs'], channel=1) """ And that's it! Your data is now in Python. The rest of the code is a simple plotting example. ### Stream Store Plotting Let's create time vectors for each stream store for plotting in time. """ len_Wav1 = len(data.streams.Wav1.data) len_LFP1 = len(data.streams.LFP1.data) len_pNe1 = len(data.streams.pNe1.data) time_Wav1 = np.linspace(1, len_Wav1, len_Wav1) / data.streams.Wav1.fs time_LFP1 = np.linspace(1, len_LFP1, len_LFP1) / data.streams.LFP1.fs time_pNe1 = np.linspace(1, len_pNe1, len_pNe1) / data.streams.pNe1.fs """ ### Plot five seconds of data from each store
def read_TDT(binPath, downSample=None, tStart=None, tEnd=None, chanList=None, ds_method='interpolation'): """Load TDT data using the tdt python package. Args: binPath (str | pathlib.Path): Path to block Kwargs: downSample (int | float | None): Frequency in Hz at which the data is subsampled. No subsampling if None. (default None) tStart (float | None): Time in seconds from start of recording of first loaded sample. Default 0.0 tEnd (float | None): Time in seconds from start of recording of last loaded sample. Duration of recording by default chanList (list(string)): List of loaded channels. Should be non empty. The provided list should be formatted as follows:: [<score_name>-<channel_index>, ...] Where channels are 1-indexed, (IMPORTANT) not 0-indexed (for consistency with tdt methods) eg: [LFPs-1, LFPs-2, EEGs-1, EEGs-94, EMGs-1...] ds_method (str): Method for resampling. Passed to ``resample.signal_resample``. 'poly' is more accurate, 'interpolation' is faster (default 'interpolation') Returns: data (np.ndarray): The raw data of shape (n_channels, n_points) downsample (float):The down-sampling frequency used. chanList (list(str)): List of channels """ import tdt print(f"Load TDT block at {binPath}") if tStart is None: tStart = 0.0 if tEnd is None: tEnd = 0.0 print(f"tStart = {tStart}, tEnd={tEnd}") def validate_chan(s): return isinstance(s, str) and len(s.split('-')) == 2 def parse_chan(score_chan): store, chan = [s.strip(' ') for s in score_chan.split('-')] return store, int(chan) # Check length and formatting of `chanList parameter` chanList_error_msg = ( "`chanList` should be a non-empty list of strings and formatted as " "follows:\n [<score_name>-<channel_index>, ...], \n" "where channel indices are 1-indexed (not 0-indexed). eg: \n" " [LFPs-1, LFPs-2, EEGs-1, EEGs-94, EMGs-1...] \n" f"Currently chanList = {chanList}") if chanList is None: raise ValueError(chanList_error_msg) if not (len(chanList) > 0 and all([validate_chan(s) for s in chanList]) and all([parse_chan(s)[1] > 0 for s in chanList])): raise ValueError(chanList_error_msg) # Load and downsample data for all requested channels chan_dat_list = [] # List of channel data arrays to concatenate chan_ts_list = [] # List of timestamps for each channel # Iterate on channels: for store_chan in chanList: store, chan = parse_chan(store_chan) print(f"Load channel {chan} from store {store}", end=", ") blk = read_tdt_block(binPath, t1=tStart, t2=tEnd, store=store, channel=chan) # Check that the requested data is actually there if store not in blk.streams.keys(): stores = tdt.read_block(binPath, t2=1.0, nodata=True).streams.keys() raise Exception(f"Store `{store}` not found in data." f" Existing stores = {stores}") sRate = blk.streams[store].fs chandat = blk.streams[store].data # (nSamples x 0)-array # Downsample the data if downSample is None or downSample == sRate: downSample = sRate chan_dat_ds = chandat else: if downSample > sRate: print( f"Warning: The resampling rate ({downSample}) is greater " f"than the original sampling rate ({sRate})") print(f"-> Resampling from {sRate}Hz to {downSample}Hz using " f"'{ds_method}' method") if not chan_dat_list: # First channel: downsample to target chan_dat_ds = resample.signal_resample( chandat, sampling_rate=sRate, desired_sampling_rate=downSample, method=ds_method, ) else: # next channels: downsample to match first channel's length chan_dat_ds = resample.signal_resample( chandat, sampling_rate=sRate, desired_length=len(chan_dat_list[0]), method=ds_method, ) # Add data and timestamps chan_ts_ds = [ blk.streams[store].start_time + i / downSample for i in range(len(chan_dat_ds)) ] chan_dat_list.append(chan_dat_ds) chan_ts_list.append(chan_ts_ds) # Check same number of samples for all channels assert all([dat.shape == chan_dat_list[0].shape for dat in chan_dat_list]) # Check data is aligned for all channels (~same timestamps for each channel) MAX_DIFF = 0.001 # (s) ts_diff = [max(ts_list) - min(ts_list) for ts_list in zip(*chan_ts_list)] assert all([v <= MAX_DIFF for v in ts_diff]) return np.stack(chan_dat_list), downSample, chanList
""" ### Set up the variables for the data you want to extract. We will extract channel 3 from the LFP1 stream data store, created by the Neural Stream Processor gizmo, and use our PulseGen epoc event ('PC0/') as our stimulus onset. """ REF_EPOC = 'PC0/' STREAM_STORE = 'LFP1' ARTIFACT = np.inf # optionally set an artifact rejection level CHANNEL = 3 TRANGE = [-0.3, 0.8] # [start time relative to epoc onset, window duration] """ ### Now read the specified data from our block into a Python structure """ data = read_block(BLOCK_PATH, evtype=['epocs', 'scalars', 'streams'], channel=CHANNEL) """ ## Use epoc_filter to extract data around our epoc event Using the `t` parameter extracts data only from the time range around our epoc event. For stream events, the chunks of data are stored in a list. """ data = epoc_filter(data, 'PC0/', t=TRANGE) """ Optionally remove artifacts """ art1 = np.array( [np.any(x > ARTIFACT) for x in data.streams[STREAM_STORE].filtered], dtype=bool)