def load_events(file_path, channels=None): """ return time stamps in seconds of each digital channel :param file_path: :param channels: name of channels :return: dictionary, {channel: {'rise':[timestamps of rising events in seconds], 'fall':[timestamps of falling events in seconds}} """ print('\n') if file_path[-7:] != '.events': raise LookupError('The input file: ' + file_path + ' is not a .events file!') with open(file_path) as f: header = oe.readHeader(f) fs = float(header['sampleRate']) events = oe.loadEvents(file_path) detected_channel_number = int(max(events['channel']) + 1) real_channels = ['ch_' + ft.int2str(c, 3) for c in range(detected_channel_number)] if channels is not None: if detected_channel_number != len(channels): warning_msg = '\nThe number of digital channels detected: ' + str(detected_channel_number) + \ ' does not match input channel number: ' + str(len(channels)) warnings.warn(warning_msg) if len(channels) <= detected_channel_number: real_channels[0:len(channels)] = channels else: real_channels = channels[0:detected_channel_number] output = {} for i, ch in enumerate(real_channels): output.update({ch : {'rise' : [], 'fall' : []} }) rise = events['timestamps'][np.logical_and(events['channel'] == i, events['eventId'] == 1)] output[ch]['rise'] = np.array(rise.astype(np.float32) / fs).astype(np.float32) fall = events['timestamps'][np.logical_and(events['channel'] == i, events['eventId'] == 0)] output[ch]['fall'] = np.array(fall.astype(np.float32) / fs).astype(np.float32) print('events loaded.\n') return output
def load_sync_data_ephys(recording_to_process, prm): is_found = False sync_data = None print('loading sync channel...') file_path = recording_to_process + '/' + prm.get_sync_channel() if os.path.exists(file_path): sync_data = open_ephys_IO.get_data_continuous(prm, file_path) is_found = True else: print( 'Sync data was not found, I will check if Axona sync data is present and convert it if it is.' ) events_file = recording_to_process + '/all_channels.events' if os.path.exists(events_file): events = OpenEphys.load(events_file) time_stamps = events['timestamps'] channel = events['channel'] pulse_indices = time_stamps[np.where(channel == 0)] # for sample in pulse_indices: # make pulse wider # pulse_indices = np.append(pulse_indices, np.arange(sample, (sample + 5000))) # load any continuous data file to get length of recording for name in glob.glob(recording_to_process + '/*.continuous'): if os.path.exists(name): print(name) ch = open_ephys_IO.get_data_continuous(prm, name) length = len(ch) sync_data = np.zeros(length) sync_data[np.take( pulse_indices, np.where(pulse_indices < len(ch))).astype(int)] = 1 is_found = True return sync_data, is_found return sync_data, is_found
def loadTTLdata(rawDataDir): # from the .events file event_path = str(Path(rawDataDir).joinpath('all_channels.events')) print('loading: ' + event_path) eventsDict = OE.load(event_path) ### load TTL dict file print(eventsDict) print(eventsDict['eventType']) print(eventsDict['eventId']) print( '\nall the events that are saved have type TTL = 3 ; Network Event = 5' ) eventIDs = eventsDict['eventType'] print('event IDs: ' + str(eventIDs)) TTL_values = eventsDict['eventId'] print('shape of TTL values' + str(TTL_values.shape)) print('TTL values: ' + str(TTL_values)) TTL_timeStamps = eventsDict['timestamps'] print('shape of timestamps: ' + str(TTL_timeStamps.shape)) print('time stamps: ' + str(TTL_timeStamps)) ### DEBUG pdb.set_trace() return TTL_timeStamps, TTL_values
def get_data_spike(folder_path, file_path, name): data = OpenEphys.load( file_path) # returns a dict with data, timestamps, etc. timestamps = data['timestamps'] waveforms = data['spikes'] # print('{} waveforms were found in the spike file'.format(waveforms.shape[0])) waveforms, timestamps = delete_noise(folder_path, name, waveforms, timestamps) return waveforms, timestamps
def loadRawOEdir(rawDataDir): data = OE.loadFolderToArray(rawDataDir, channels=range(FIRST_CH, LAST_CH + 1), chprefix='CH', dtype=float, session='0', source='100') print(np.shape(data)) totalNumSamples = np.shape(data)[ 0] #no longer used but available for later print(data) return data
def load(path, nchan, cutmin): ndpoints = cutmin * 60 * 30000 dim = int(math.floor((ndpoints / 1024))) raw = np.empty((dim * 1024, nchan)) for i in range(nchan): i += 1 # dump=[] data = op.loadContinuous(str(path) + '\\100_CH%s.continuous' % i, stop_record=ndpoints / 1024) print('loaded %s' % i, 'start cutting') # cutting timestamps = data['data'] #[0:cutmin*60*30000] raw[:, i - 1] = (timestamps) return raw
def loadContinuous(folder, pluginId): ''' Loads continuous files and returns a MNE data array. Currently looses all information stored with the data besides electrode number and sampling rate. Anything else needs to be handled separately. Inputs: folder - The folder where the data is held (needs ending \) pluginId - from 100_CH1.continuous. Normally raw data is from 100. This can be found from config.xml file. ''' # Get file names and preform human sorting files = glob.glob(folder + str(pluginId) + '_CH*.continuous') files.sort(key=alphanum_key) # Load in Open Ephys data and save channel names datas = [] chanNames = [] for dataPath in files: chanNames.append(re.search('100_CH(.*).continuous', dataPath).group(1)) datas.append(OpenEphys.load(dataPath)) print('loading done! Now putting into MNE format') # Use eeg because that's the closest mne offers chanTypes = ['eeg'] * len(files) info = mne.create_info(ch_names=chanNames, sfreq=datas[0]['header']['sampleRate'], ch_types=chanTypes) # Make a list of the data from the channels rawData = [] for data in datas: rawData.append(data['data']) # Put into mne format matrix = np.array(rawData) raw = mne.io.RawArray(matrix, info) print('MNE conversion complete') return raw
def loadRawOEdir(rawDataDir): # print('loading: ' + pathToFile) # data_dict = OE.load(pathToFile) # returns a dict with data, timestamps, etc. # print(data_dict) # # open ephys's BROKEN (float given but expects int error) downsample code # # def downsample(trace,down): # # downsampled = scipy.signal.resample(trace,np.shape(trace)[0]/down) # # return downsampled data = OE.loadFolderToArray(rawDataDir, channels=range(FIRST_CH, LAST_CH + 1), chprefix='CH', dtype=float, session='0', source='100') print(np.shape(data)) totalNumSamples = np.shape(data)[ 0] #no longer used but available for later print(data) return data
def get_ttls(data_path): print 'data_path = ', data_path events_data = OpenEphys.load('all_channels.events') ########## LOAD THE TTLs!!! ############# #temp = np.intersect1d((np.where(events_data['channel'] == 0)[0]), (np.where(events_data['eventId'] == 1)[0])) ttl0_on = events_data['timestamps'][np.intersect1d((np.where(events_data['channel'] == 0)[0]), (np.where(events_data['eventId'] == 1)[0]))] #temp = np.intersect1d((np.where(events_data['channel'] == 0)[0]), (np.where(events_data['eventId'] == 0)[0])) ttl0_off = events_data['timestamps'][np.intersect1d((np.where(events_data['channel'] == 0)[0]), (np.where(events_data['eventId'] == 0)[0]))] #temp = np.intersect1d((np.where(events_data['channel'] == 1)[0]), (np.where(events_data['eventId'] == 1)[0])) ttl1_on = events_data['timestamps'][np.intersect1d((np.where(events_data['channel'] == 1)[0]), (np.where(events_data['eventId'] == 1)[0]))] #temp = np.intersect1d((np.where(events_data['channel'] == 1)[0]), (np.where(events_data['eventId'] == 0)[0])) ttl1_off = events_data['timestamps'][np.intersect1d((np.where(events_data['channel'] == 1)[0]), (np.where(events_data['eventId'] == 0)[0]))] return ttl0_on, ttl0_off, ttl1_on, ttl1_off
def loadTTLdata(self, *RAW_DATA_DIR): # from the .events file if RAW_DATA_DIR: print('raw data dir specified by user: '******'all_channels.events')) eventsDict = OE.load(event_path) ### load TTL dict file eventIDs = eventsDict['eventType'] print('\nloading: ' + event_path) print(eventsDict['eventType']) print(eventsDict['eventId']) print( '\nall the events that are saved have type TTL = 3 ; Network Event = 5' ) print('event IDs: ' + str(eventIDs)) self.sampleRate = float(eventsDict['header']['sampleRate']) TTL_timesInSamples = eventsDict['timestamps'] self.TTL_values = eventsDict['eventId'] self.TTL_timesInSecs = TTL_timesInSamples / self.sampleRate print('shape of TTL values' + str(self.TTL_values.shape)) print('TTL values: ' + str(self.TTL_values)) print('shape of timestamps: ' + str(TTL_timesInSamples.shape)) print('time stamps: ' + str(TTL_timesInSamples)) print('sample rate (from all_channels.events): ' + str(self.sampleRate)) print('converting times from sample number to secs') ### skip weird bad values at the beginning... TO DO: find out why these are so rapid yet still have event id 3. check the other data set # TTL_timeStamps = TTL_times[4::] # self.TTL_values = self.TTL_values[4::] return self.TTL_timesInSecs, self.TTL_values
# RAW_DATA_PATH = '/home/orthogonull/a_MHR/a_Research/a_WIP/2018-04-29_23-20-48' # pathToFile = '/home/orthogonull/a_MHR/a_Research/a_WIP/2018-04-29_23-20-48/100_CH1.continuous' # figSaveDir = '/home/orthogonull/a_MHR/a_Research/a_WIP' ### ephys computer paths pathToFile = 'C:\\Users\\yatang\\Documents\\Ephys\\Open Ephys\\2018-05-04_13-55-49_MR1_r2_pre\\100_CH1.continuous' RAW_DATA_PATH = 'C:\\Users\\yatang\\Documents\\Ephys\\Open Ephys\\2018-05-04_13-55-49_MR1_r2_pre\\' figSaveDir = 'C:\\Users\\yatang\\Documents\\Ephys\\Open Ephys\\' else: sys.exit( 'ERROR: user parameters in this script failed to find a valid directory of recordings!' ) print('loading: ' + RAW_DATA_PATH) data_dict = OE.load(pathToFile) # returns a dict with data, timestamps, etc. print(data_dict) # open ephys's BROKEN (float given but expects int error) downsample code # def downsample(trace,down): # downsampled = scipy.signal.resample(trace,np.shape(trace)[0]/down) # return downsampled data = OE.loadFolderToArray(RAW_DATA_PATH, channels=range(FIRST_CH, LAST_CH + 1), chprefix='CH', dtype=float, session='0', source='100') print(np.shape(data)) totalNumSamples = np.shape(data)[0] #no longer used but available for later print(data)
This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see <http://www.gnu.org/licenses/>. Just sketches for now. """ #%% import OpenEphys import matplotlib.pyplot as plt OpenEphysData = OpenEphys.load('100_CH6.continuous') OpenEphysEvents = OpenEphys.load('all_channels.events') RawData = OpenEphysData["data"].tolist() Events = OpenEphysEvents["timestamps"].tolist() EventChannels = OpenEphysEvents["channel"].tolist() EventTypes = OpenEphysEvents["eventType"].tolist() TTL0 = [0] * len(RawData) TTL1 = [0] * len(RawData)
#del stim_array[0] #st= np.array(stim_array)/25000. ##file_name = '/home/cat/neuron/in_vivo/dan/M149873/2015-02-06_13-02-32_flash_CSD/data/stim_times.txt' #file_name = file_dir+'stim_times.txt' #np.savetxt(file_name,st, fmt='%8.3f') #quit() #******************************* LOAD RAW DATA AND PREP FOR .TSF FILE *************************** data=[] for i in range(128): file_name = file_dir + '100_CH'+ str(i+1) +'.continuous' data.append(oe.loadContinuous(file_name)['data']) dt = np.array(data, dtype=np.int16) print dt.shape ####Plot first 10000 steps from channel 0 #x = np.arange(0,len(dt1),1.)/SampleFrequency #plt.plot(x,dt1, 'r-', color='black',linewidth=1) #plt.plot(x,dt2+1000, 'r-', color='black',linewidth=1) #plt.plot(x,dt[probe128[0]][0:100000], 'r-', color='black',linewidth=1) #plt.plot(x,dt[probe128[2]][0:100000]-100, 'r-', color='black',linewidth=1) #plt.show()
def wirtecontinous(filepath, timestamps, header, dtype=float, start_record=None, stop_record=None, ignore_last_record=True): # This is what the record marker should look like spec_record_marker = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 255]) recordingNumbers = [] samples = [] samples_read = 0 records_read = 0 # Open the file with file(filepath, 'wb') as f: # Read header info, file length, and number of records header = header record_length_bytes = 2 * header['blockLength'] + 22 fileLength = os.fstat(f.fileno()).st_size n_records = op.get_number_of_records(filepath) # Use this to set start and stop records if not specified if start_record is None: start_record = 0 if stop_record is None: stop_record = n_records # We'll stop reading after this many records are read n_records_to_read = stop_record - start_record # Seek to the start location, relative to the current position # right after the header. f.seek(record_length_bytes * start_record, 1) # Keep reading till the file is finished while f.tell() < fileLength and records_read < n_records_to_read: # Skip the last record if requested, which usually contains # incomplete data if ignore_last_record and f.tell() == (fileLength - record_length_bytes): break # Read the timestamp for this record # litte-endian 64-bit signed integer timestamps.append(np.fromfile(f, np.dtype('<i8'), 1)) # Read the number of samples in this record # little-endian 16-bit unsigned integer N = np.fromfile(f, np.dtype('<u2'), 1).item() # Read and store the recording numbers # big-endian 16-bit unsigned integer recordingNumbers.append(np.fromfile(f, np.dtype('>u2'), 1)) # Read the data # big-endian 16-bit signed integer data = np.fromfile(f, np.dtype('>i2'), N) if len(data) != N: raise IOError("could not load the right number of samples") # Optionally convert dtype if dtype == float: data = data * header['bitVolts'] # Store the data samples.append(data) # Extract and test the record marker record_marker = np.fromfile(f, np.dtype('<u1'), 10) if np.any(record_marker != spec_record_marker): raise IOError("corrupted record marker at record %d" % records_read) # Update the count samples_read += len(samples) records_read += 1
import os import encode from mlpy import writemda16i import sys cwd=os.getcwd() sys.path.append(cwd) sys.path.append('~/repos/mountainlab/old/packages/mountainsort/packages/pyms') dump = [] for i in range(64): i+=1 data=[] data=op.load('2044_2018-06-19_13-42-08/100_CH%s.continuous' %i) print('loaded %s'%i,'start cutting') # cutting timestamps = data['timestamps'][0:30*60*30000] #writing # encode, write to another openephys # encode.writecontinous(filepath='cut_CH%s.continuous'%i,timestamps=timestamps, header=data['header']) # accumulate to N*M, to mda dump.append(timestamps) #append to row, need to check if need transpose #convert to mda, writemda16i(dump, 'raw30min.mda')
def get_events(prm, file_path): events = OpenEphys.load(file_path) return events
import sys sys.path.append( '/Volumes/Mac HD/Dropbox (coxlab)/Scripts/Repositories/continuous-ephys/open-ephys analysis/' ) import OpenEphys import numpy as np import matplotlib.pyplot as plt #eventsfile = "/Volumes/labuser/Desktop/EPHYS/test data/10HzDigitalInput/2015-05-11_14-06-08_diginputtest/all_channels.events" #eventsfile = "/Volumes/GG Data Raid/Ephys/2015-04-29_17-35-04_digitalinputtest/all_channels_4.events" #eventsfile = "/Volumes/labuser/Desktop/EPHYS/test data/MWorksPixelInput/2015-05-11_15-53-23_digintest/all_channels.events" #eventsfile = "/Volumes/labuser/Desktop/EPHYS/test data/MWorksPixelInput/2015-05-12_12-00-22_digintest/all_channels.events" #eventsfile = "/Volumes/labuser/Desktop/EPHYS/test data/MWorksPixelInput/2015-05-12_17-51-29_captest/all_channels_2.events" eventsfile = "/Volumes/labuser/Desktop/EPHYS/Data/grat03/2015-05-30_12-46-18/all_channels.events" events_data = OpenEphys.load(eventsfile) pixel_ch1 = [] # np.zeros((len(events_data['timestamps']),1)) pixel_ch2 = [] #np.zeros((len(events_data['timestamps']),1)) pixel_ch1_time = [] #np.zeros((len(events_data['timestamps']),1)) pixel_ch2_time = [] counter = 0 while counter < len(events_data['timestamps']): # go thru all timestamps if events_data['channel'][counter] == 3: if events_data['eventId'][counter] == 1: pixel_ch1.append(1) pixel_ch1_time.append(events_data['timestamps'][counter]) elif events_data['eventId'][counter] == 0: pixel_ch1.append(0) pixel_ch1_time.append(events_data['timestamps'][counter])
if rank == size-1: last_val = num_files else: last_val = np.floor((rank+1)*num_files/size) rank_vals = np.arange(st_val, last_val) rank_vals = rank_vals.astype(int) data = {} for i in rank_vals: print('i', i) st = time.time() data[continuous_files[i].replace('.continuous', '')] = OpenEphys.load(folderpath + '/'+continuous_files[i]) en = time.time() print(en-st) print('done') #data = OpenEphys.loadFolder('D:/Open_ephys_testing_ground_data') #df = pandas.DataFrame(data) #df.to_hdf('./trialhdf', 'trial')
#==============================================================================# ntot = len(dfold_list) ind = 0 for dfold in dfold_list: ind = ind + 1 nrem = ntot - ind print('Converting ' + dfold + ': ' + str(nrem) + ' remaining') folderpath = ddir + dfold # folderpath where data is stored #=============== SUBSAMPLE LOWPASS (LFP) SIGNAL AND CONVERT TO DAT ===========# lfpname = '102_CH2.continuous' filepath = ddir + dfold + '/' + lfpname LFP = OpenEphys.loadContinuous(filepath, dtype=float) LFP = LFP['data'] ssLFP = LFP[::10].copy() #subsampled LFP # save as dat file datnameLFP = dfold + '_LFP.dat' ssLFP.tofile(os.path.join(folderpath, datnameLFP)) #==============================================================================# #======================== CONVERT HIGHPASS SIGNALS TO DAT ======================# # Load highpass signals of each sahnk in geometric order (bottom of shank to top) # then save to flat binary file # create shank1 and shank2 folders os.makedirs(folderpath + '/2017/' + shank1_fname) os.makedirs(folderpath + '/2017/' + shank2_fname)
# -*- coding: utf-8 -*- """ Created on Mon Jun 11 13:26:20 2018 @author: behrenslab """ import data_import as di import OpenEphys as op import Esync as es import numpy as np session = di.Session( '/home/behrenslab/ephys/2018-06-05-reversal_learning_3_tasks_recording/m483-2018-06-07-161545.txt' ) rsync_events = [ event.time for event in session.events if event.name in ['Rsync'] ] rsync_ephys = op.loadEvents( '/home/behrenslab/ephys/2018-06-07_16-15-43/all_channels.events') rsync_timestamps = rsync_ephys['timestamps'] rsync_timestamps = np.array(rsync_timestamps)[::2] rsync_events = np.array(rsync_events) rsync_timestamps = rsync_timestamps / 30 aligner = es.Rsync_aligner(rsync_events, rsync_timestamps, plot=True) times_B = aligner.A_to_B(rsync_events) #A pycontrol times_A = aligner.B_to_A(rsync_timestamps) pycontrol_to_ephys = aligner.A_to_B
for file_ephys in files_ephys: if file_ephys in m486: match_ephys = re.search(r'\d{4}-\d{2}-\d{2}', file_ephys) date_ephys = datetime.strptime(match_ephys.group(), '%Y-%m-%d').date() date_ephys = match_ephys.group() for file_behaviour in files_behaviour: match_behaviour = re.search(r'\d{4}-\d{2}-\d{2}', file_behaviour) date_behaviour = datetime.strptime(match_behaviour.group(), '%Y-%m-%d').date() date_behaviour = match_behaviour.group() if date_ephys == date_behaviour: behaviour_path = behaviour_filename+'/'+file_behaviour behaviour_session = di.Session(behaviour_path) ephys_path = ephys_data_folder+'/'+file_ephys + '/' + subject print(behaviour_path) print(ephys_path) ephys_events = op.loadEvents(os.path.join(ephys_path,'all_channels.events')) data_folder = file_ephys check_if_kilosort_exists = os.listdir(ephys_path) check_if_npy_exists = os.listdir(spikes_df_csv_out_folder) file_ephys_npy = file_ephys +'.npy' if file_ephys_npy not in check_if_npy_exists: # For multiple files in one session add the requirement of cluster_groups.csv file for file in check_if_kilosort_exists: if fnmatch.fnmatch(file, '*.csv'): # Get offset in samples of Kilosort spike data relative to TTL events. with open(os.path.join(ephys_path,'messages.events')) as f: message = f.read() recording_start_sample, sampling_rate = [ int(x) for x in message.split('start time: ')[1].split('Hz')[0].split('@')]
def get_data_continuous(prm, file_path): data = OpenEphys.load(file_path) signal = data['data'] signal = np.asanyarray(signal) return signal
def plot_responses(data_path,save_path): ttl0_on, ttl0_off, ttl1_on, ttl1_off = get_ttls(data_path) fs = 3e4 time_axis = np.arange(-.250,.750,1/fs) channels = range(64) # [0,1] # allchans_binoc = dict() ######### LOAD EPHYS for ch in channels: ephys_data = OpenEphys.load('100_CH%i.continuous' % (ch+1)) filt_data = filter(ephys_data['data'],[1,200]) ###### take ttl0_on times (in samples), get corresponding chunks from the ephys (250ms before that time to 750ms after) offset_time = ephys_data['timestamps'][0] right_led_ephys = np.zeros([ttl1_on.shape[0],int(1.0*fs)]) for idx,on_time in enumerate(ttl1_on): offset_ontime = on_time - offset_time #print idx,on_time,offset_ontime if offset_ontime-int(.250*fs)>0: if filt_data.shape[0] - offset_ontime > 0: if offset_ontime+int(.750*fs) <filt_data.shape[0] : right_led_ephys[idx,:] = filt_data[int(offset_ontime-int(.250*fs)):int(offset_ontime+int(.750*fs))] left_led_ephys = np.zeros([ttl0_on.shape[0],int(1.0*fs)]) for idx,on_time in enumerate(ttl0_on): offset_ontime = on_time - offset_time #print idx,on_time,offset_ontime if offset_ontime-int(.250*fs)>0: if filt_data.shape[0] - offset_ontime > 0: if offset_ontime+int(.750*fs) <filt_data.shape[0] : left_led_ephys[idx,:] = filt_data[int(offset_ontime-int(.250*fs)):int(offset_ontime+int(.750*fs))] mean_left_led = np.mean(left_led_ephys,axis=0) #std_left_led = np.std(left_led_ephys,axis=0) sem_left_led = stats.sem(left_led_ephys,axis=0) mean_right_led = np.mean(right_led_ephys,axis=0) #std_right_led = np.std(right_led_ephys,axis=0) sem_right_led = stats.sem(right_led_ephys,axis=0) ##### binocularity index for each electrode: left_max = np.max(abs(mean_left_led)) right_max = np.max(abs(mean_right_led)) binocularity = (left_max - right_max) / (left_max + right_max) print 'binocularity for ch %i = %6f' % ((ch+1),binocularity) allchans_binoc[str(ch+1)] = [binocularity] plot = 1 if plot == 1: f, axarr = plt.subplots(2, sharex=True,sharey=True) #plt.figure(dpi=600) f.dpi = 600 f.suptitle('binocularity for ch %i = %6f' % (ch,binocularity)) #ax = sns.tsplot(data=mean_left_led_ch22,time=time_axis, linewidth=0.1) axarr[0].plot(time_axis,mean_left_led,linewidth=0.1) axarr[0].fill_between(time_axis, mean_left_led-sem_left_led, mean_left_led+sem_left_led,alpha=0.1) axarr[0].set_ylabel('Left LED Response (uV)') axarr[1].plot(time_axis,mean_right_led,linewidth=0.1) axarr[1].fill_between(time_axis, mean_right_led-sem_right_led, mean_right_led+sem_right_led,alpha=0.1) axarr[1].set_ylabel('Right LED Response (uV)') axarr[1].set_xlabel('Time (sec)') axarr[1].set_xticks start, end = axarr[1].get_xlim() stepsize = 0.1 axarr[1].xaxis.set_ticks(np.arange(start, end, stepsize)) # add a square wave showing when the stimulus happened: plt.plot([-0.25,0,0,0.25,0.25,0.75],[-75,-75,-50,-50,-75,-75],color='black') f.savefig(save_path + 'ch%i.pdf'%(ch+1),dpi=600)
#file_dir = 'M150566/2015-04-01_16-41-14_flashes_vis/' #file_dir = 'M150566/2015-04-01_20-55-32_flashes_vis/' #file_dir = 'M181420/181420_2015-04-07_13-25-07_flashes_vis/' #file_dir = 'M181420/181420_2015-04-07_18-10-46_flash_vis/' ##********************** LOAD STIMULUS TIMES AND SAVE THEM ***************************** if False: file_name = '103_PAI7.StimTimeshere.continuous' data=[] data = oe.loadContinuous(file_dir+mouse_dir+'/'+exp+dir+'/'+file_name) dt = np.array(data['data']) stim_array=[] state=0 for i in range(len(dt)): if state==0: if dt[i]>1: print "ON" temp=i state=1 if state==1: if dt[i]<1: print "OFF" stim_array.append([temp,i]) state=0
import OpenEphys as OE if __name__ == '__main__': data = OE.load('/Users/fpbatta/dataLisa/rat27_plusmaze_base_II_2016-03-24_14-10-08/100_CH9.continuous') print(data)
pathname = f"./plx/{dirname}" #Ephys形式で記録されたチャネルマップ配置 channel_map = [9, 8, 10, 7, 13, 4, 12, 5, 15, 2, 16, 1, 14, 3, 11, 6] #刺激方法の設定をSetting.jsonのファイルから読み込む #jsonファイルは日程-時間で記録されているため、result.jsonに記載された日程を参照する with open(f"./json/{jsonfile[:14]}Settings.json", encoding="UTF-8") as json_fp: stim_setting = json.load(json_fp) if not (os.path.exists(f"./waveform/{dirname}")): os.mkdir(f"./waveform/{dirname}") num = 1 for i in channel_map: #OpenEphys.pyを使用してcontinuousファイルの読み込み tmp = OpenEphys.loadContinuous( f"./plx/{dirname}/101_CH{str(i)}.continuous") #lowpassフィルタがかけられた波形を抽出 waveform = glia.nucleus.Waveform(lowpass(tmp["data"], 30000, 450, 600, 10, 30), rate=glia.Frequency(30000)) #tmp(['header', 'timestamps', 'data', 'recordingNumber'])が記載されている #timestampsはサンプリングレートで記録された記録点 timestamps = tmp["timestamps"][0] if num < 10: glia.save(f"./waveform/{dirname}/0{num}.lfp", waveform) else: glia.save(f"./waveform/{dirname}/{num}.lfp", waveform) del tmp, waveform gc.collect() num += 1
# continue # elif ephys_folder.find('m480') == 104: # if 'm480.dat' not in os.listdir(ephys_folder): # op.pack_2(ephys_folder,filename =ephys_folder +'/'+'m480'+'.dat', channels =channels_m486_m482_m480_m478, chprefix = 'CH', dref = 'ave') # elif ephys_folder.find('m478') == 104: # if 'm478.dat' not in os.listdir(ephys_folder): # op.pack_2(ephys_folder, filename =ephys_folder +'/'+'m478'+'.dat', channels =channels_m486_m482_m480_m478, chprefix = 'CH', dref = 'ave') # elif ephys_folder.find('m486') == 104: # if 'm486.dat' not in os.listdir(ephys_folder): # op.pack_2(ephys_folder, filename =ephys_folder +'/'+'m486'+'.dat', channels =channels_m486_m482_m480_m478, chprefix ='CH', dref = 'ave') # elif ephys_folder.find('m482') == 104: # if 'm482.dat' not in os.listdir(ephys_folder): # op.pack_2(ephys_folder, filename =ephys_folder +'/'+'m482'+'.dat', channels =channels_m486_m482_m480_m478, chprefix ='CH', dref = 'ave') # elif ephys_folder.find('m484') == 109: # if 'm484.dat' not in os.listdir(ephys_folder): # op.pack_2(ephys_folder, filename =ephys_folder +'/'+'m484'+'.dat', channels =channels_m484_m479, chprefix ='CH', dref = 'ave') # elif ephys_folder.find('m479') == 109: # if 'm479.dat' not in os.listdir(ephys_folder): # op.pack_2(ephys_folder, filename =ephys_folder +'/'+'m479'+'.dat', channels =channels_m484_m479, chprefix ='CH', dref = 'ave') # for file in files: ephys_folder = os.path.join(files_folder, file) if ephys_folder.find('m480') == 104: if 'm480.dat' not in os.listdir(ephys_folder): op.pack_2(ephys_folder, filename=ephys_folder + '/' + 'm480' + '.dat', channels=channels_m486_m482_m480_m478_second_box, chprefix='CH', dref='ave')
def load_continuous(file_path, dtype=np.float32, start_ind=0): """ Jun's wrapper to load .continuous data from OpenEphys data files, this will try to load the whole file into memory, using np.fromfile. It can also start from any position in the file (defined by the start_ind) :param file_path: :param dtype: np.float32 or np.int16 :param start_ind: non-negative int, default 0. start index to extract data. :return: header: dictionary, standard open ephys header for continuous file samples: 1D np.array """ assert dtype in (np.float32, np.int16), \ 'Invalid data type specified for loadContinous, valid types are np.float32 and np.int16' print("\nLoading continuous data from " + file_path) bytes_per_block = CONTINUOUS_TIMESTAMP_DTYPE.itemsize + CONTINUOUS_SAMPLE_PER_RECORD_DTYPE.itemsize + \ CONTINUOUS_RECORDING_NUMBER_DTYPE.itemsize + CONTINUOUS_MARKER_BYTES + \ CONTINUOUS_SAMPLE_DTYPE.itemsize * oe.SAMPLES_PER_RECORD input_array = np.fromfile(file_path, dtype='<u1') file_length = input_array.shape[0] print('total length of the file: ', file_length, 'bytes.') valid_block_start = find_next_valid_block(input_array, bytes_per_block=bytes_per_block, start_index=start_ind) print('the beginning index of the first valid block after index: ' + str(start_ind) + ' is ' + \ str(valid_block_start)) print('bytes per record block: ', bytes_per_block) # read in the data f = open(file_path, 'rb') header = oe.readHeader(f) block_num = (file_length - valid_block_start) // bytes_per_block print('number of potential valid blocks after index', start_ind, ':', block_num) samples = np.empty(oe.SAMPLES_PER_RECORD * block_num, dtype=dtype) _ = np.fromfile(f, np.dtype('<u1'), valid_block_start - oe.NUM_HEADER_BYTES) for i in range(block_num): if i == 0: # to get the timestamp of the very first record (block) # for alignment of the digital event start_ind = np.fromfile(f, CONTINUOUS_TIMESTAMP_DTYPE, 1) start_time = float(start_ind) / float(header['sampleRate']) else: _ = np.fromfile(f, CONTINUOUS_TIMESTAMP_DTYPE, 1) N = np.fromfile(f, CONTINUOUS_SAMPLE_PER_RECORD_DTYPE, 1)[0] if N != oe.SAMPLES_PER_RECORD: print(('samples per record specified in block ' + str(i) + ' (' + str(N) + ') does not equal to expected value (' + str(oe.SAMPLES_PER_RECORD) + ')!')) samples = samples[0 : i * oe.SAMPLES_PER_RECORD] print('samples per record specified in block ' + str(i) + ' (' + str(N) + \ ') does not equal to expected value (' + str(oe.SAMPLES_PER_RECORD) + ')!') break _ = (np.fromfile(f, CONTINUOUS_RECORDING_NUMBER_DTYPE, 1)) if dtype == np.float32: curr_samples = (np.fromfile(f, CONTINUOUS_SAMPLE_DTYPE, N) * float(header['bitVolts'])).astype(np.float32) elif dtype == np.int16: curr_samples = np.fromfile(f, CONTINUOUS_SAMPLE_DTYPE, N).astype(np.int16) else: raise ValueError('Error in reading data of block:' + str(i)) samples[i*oe.SAMPLES_PER_RECORD : (i+1)*oe.SAMPLES_PER_RECORD] = curr_samples record_mark = np.fromfile(f, dtype=np.dtype('<u1'), count=10) if not np.array_equal(record_mark, oe.RECORD_MARKER): print(('record marker specified in block ' + str(i) + ' (' + str(record_mark) + ') does not equal to expected array (oe.RECORD_MARKER)!')) break header.update({'start_time': start_time}) print('continuous channel start time (for aligning digital events): ', start_time) return header, samples
def loadEventsOE(path, tsStart): raw = OpenEphys.load(path) return Event(raw, tsStart)
def load_continuous_old(file_path, dtype=np.float32): """ Jun's wrapper to load .continuous data from OpenEphys data files :param file_path: :param dtype: np.float32 or np.int16 :return: header: dictionary, standard open ephys header for continuous file samples: 1D np.array """ assert dtype in (np.float32, np.int16), \ 'Invalid data type specified for loadContinous, valid types are np.float32 and np.int16' print("\nLoading continuous data from " + file_path) bytes_per_block = CONTINUOUS_TIMESTAMP_DTYPE.itemsize + CONTINUOUS_SAMPLE_PER_RECORD_DTYPE.itemsize + \ CONTINUOUS_RECORDING_NUMBER_DTYPE.itemsize + CONTINUOUS_MARKER_BYTES + \ CONTINUOUS_SAMPLE_DTYPE.itemsize * oe.SAMPLES_PER_RECORD # read in the data f = open(file_path, 'rb') file_length = os.fstat(f.fileno()).st_size print('total length of the file: ', file_length, 'bytes.') print('bytes per record block: ', bytes_per_block) block_num = (file_length - oe.NUM_HEADER_BYTES) // bytes_per_block print('total number of valid blocks: ', block_num) header = oe.readHeader(f) samples = np.empty(oe.SAMPLES_PER_RECORD * block_num, dtype) is_break = False for i in range(block_num): if i == 0: # to get the timestamp of the very first record (block) # for alignment of the digital event start_ind = np.fromfile(f, CONTINUOUS_TIMESTAMP_DTYPE, 1) start_time = float(start_ind) / float(header['sampleRate']) else: _ = np.fromfile(f, CONTINUOUS_TIMESTAMP_DTYPE, 1) N = np.fromfile(f, CONTINUOUS_SAMPLE_PER_RECORD_DTYPE, 1)[0] if N != oe.SAMPLES_PER_RECORD: print(('samples per record specified in block ' + str(i) + ' (' + str(N) + ') does not equal to expected value (' + str(oe.SAMPLES_PER_RECORD) + ')!')) samples = samples[0 : i * oe.SAMPLES_PER_RECORD] is_break = True break # raise Exception('samples per record specified in block ' + str(i) + ' (' + str(N) + \ # ') does not equal to expected value (' + str(oe.SAMPLES_PER_RECORD) + ')!') _ = (np.fromfile(f, CONTINUOUS_RECORDING_NUMBER_DTYPE, 1)) if dtype == np.float32: curr_samples = (np.fromfile(f, CONTINUOUS_SAMPLE_DTYPE, N) * float(header['bitVolts'])).astype(np.float32) elif dtype == np.int16: curr_samples = np.fromfile(f, CONTINUOUS_SAMPLE_DTYPE, N).astype(np.int16) else: raise ValueError('Error in reading data of block:' + str(i)) samples[i*oe.SAMPLES_PER_RECORD : (i+1)*oe.SAMPLES_PER_RECORD] = curr_samples record_mark = np.fromfile(f, dtype=np.dtype('<u1'), count=10) if not np.array_equal(record_mark, oe.RECORD_MARKER): print(('record marker specified in block ' + str(i) + ' (' + str(record_mark) + ') does not equal to expected array (' + str(oe.RECORD_MARKER) + ') !')) is_break = True break if is_break: samples = samples[0: oe.SAMPLES_PER_RECORD * (i -1)] header.update({'start_time': start_time}) return header, samples
DataFiles = glob.glob(''.join([RecFolder,'/*.continuous'])) DataFiles.sort() #%% Generate TTLs os.makedirs('DataArrays', exist_ok=True) DirList = glob.glob('OpenEphysFiles/*'); DirList.sort() for RecFolder in DirList: DataFiles = glob.glob(''.join([RecFolder,'/*.continuous'])) EventsFile = ''.join([RecFolder,'/all_channels.events']) DataFiles.sort() TempData = OpenEphys.loadContinuous(DataFiles[0]) # Any ch would work ;) Events = OpenEphys.loadEvents(EventsFile) # Draw TTL print('Drawing TTL channel...') if len(Events['timestamps']) > 100: # arbitrary value, I never use < 100 stimulation pulses TTLSound = [0] * len(TempData['data']) for EvTS in range(len(Events['timestamps'])): if Events['eventType'][EvTS] == 3: # if event is a TTL if Events['eventId'][EvTS] == 1: # if TTL is on TTLSound(find(TempTimestamps==EventsTimestamps(EvTS)): ... find(TempTimestamps==EventsTimestamps(EvTS+1))) = 1; end end end else
import numpy as np import scipy.signal as signal #%% Specifiy the input and output #make provisions such that the script can be run independently or through snakemake if 'snakemake' not in locals(): input = [ f'test/100_CH{x}.continuous' for x in range(1, setting.numChan + 1) ] input.append(f'test/100_{setting.positionChannel}.continuous') output = ['test/processing/eeg.npy', 'test/processing/position.npy'] else: input = snakemake.input output = snakemake.output #%% Load and downsample to create data data = [] for fn in input[:-1]: d = OpenEphys.loadContinuousFast(fn)['data'] data.append(signal.decimate(d, setting.Fs // setting.eeg_Fs, ftype='fir')) #%% Convert to numpy array and save rawSignal = np.vstack(data).T np.save(output[0], rawSignal) #%% Load and save position data d = OpenEphys.loadContinuousFast(input[-1])['data'] np.save(output[1], d)
duration = glia.milliseconds(400) filename = i jsonfile = rec_stim_matching[filename] filename = "plx/" + filename with open("json/" + jsonfile[:14] + "Settings.json", encoding="utf-8") as json_fp: stim_setting = json.load(json_fp) if not reload and os.path.isfile("./waveform/" + filename[4:-4] + ".lfp") and os.path.isfile( "./waveform/" + filename[4:-4] + ".trig"): print("Load lfp, trig") else: wave = [] for j in channel_map: tmp = OpenEphys.loadContinuous(filename + "/101_CH" + str(j) + ".continuous") #gliaの形式にそってローパスフィルタをかけた波形の出力 wave.append( glia.nucleus.Waveform(lowpass(tmp["data"], 30000, 450, 600, 10, 30), rate=glia.Frequency(30000))) timestamp = tmp["timestamps"][0] events = OpenEphys.loadEvents(filename + "/all_channels.events") e_tmp = (events["timestamps"] - timestamp)[0::2] trigger = [] for i in (e_tmp / 30): trigger.append(glia.millisecond(i)) trigger = np.array(trigger) glia.save("./waveform/" + filename[4:] + ".lfp", wave) glia.save("./waveform/" + filename[4:] + ".trig", trigger) if not reload and os.path.isfile("./waveform/" + filename[4:-4] +
def loadConOE(path): raw = OpenEphys.load(path) return Con(raw)
#==============================================================================# ntot = len(dfold_list) ind = 0 for dfold in dfold_list: ind = ind + 1 nrem = ntot - ind print('Converting ' + dfold + ': ' + str(nrem) + ' remaining') folderpath = ddir + dfold # folderpath where data is stored #=============== SUBSAMPLE LOWPASS (LFP) SIGNAL AND CONVERT TO DAT ===========# lfpname = '102_CH2.continuous' filepath = ddir + dfold + '/' + lfpname LFP = OpenEphys.loadContinuous(filepath, dtype=float) LFP = LFP['data'] ssLFP = LFP[::10].copy() #subsampled LFP # save as dat file datnameLFP = dfold + '_LFP.dat' ssLFP.tofile(os.path.join(folderpath, datnameLFP)) #==============================================================================# #======================== CONVERT HIGHPASS SIGNALS TO DAT ======================# # Load highpass signals of each sahnk in geometric order (bottom of shank to top) # then save to flat binary file # create shank1 and shank2 folders os.makedirs(folderpath + '/shank1') os.makedirs(folderpath + '/shank2') # use channels ordered by intan headstage!!!