def load_sync_data_ephys(recording_to_process, prm):
is_found = False
sync_data = None
print('loading sync channel...')
file_path = recording_to_process + '/' + prm.get_sync_channel()
if os.path.exists(file_path):
sync_data = open_ephys_IO.get_data_continuous(prm, file_path)
is_found = True
else:
print(
'Sync data was not found, I will check if Axona sync data is present and convert it if it is.'
)
events_file = recording_to_process + '/all_channels.events'
if os.path.exists(events_file):
events = OpenEphys.load(events_file)
time_stamps = events['timestamps']
channel = events['channel']
pulse_indices = time_stamps[np.where(channel == 0)]
# for sample in pulse_indices: # make pulse wider
# pulse_indices = np.append(pulse_indices, np.arange(sample, (sample + 5000)))
# load any continuous data file to get length of recording
for name in glob.glob(recording_to_process + '/*.continuous'):
if os.path.exists(name):
print(name)
ch = open_ephys_IO.get_data_continuous(prm, name)
length = len(ch)
sync_data = np.zeros(length)
sync_data[np.take(
pulse_indices,
np.where(pulse_indices < len(ch))).astype(int)] = 1
is_found = True
return sync_data, is_found
return sync_data, is_found
def loadTTLdata(rawDataDir): # from the .events file
event_path = str(Path(rawDataDir).joinpath('all_channels.events'))
print('loading: ' + event_path)
eventsDict = OE.load(event_path) ### load TTL dict file
print(eventsDict)
print(eventsDict['eventType'])
print(eventsDict['eventId'])
print(
'\nall the events that are saved have type TTL = 3 ; Network Event = 5'
)
eventIDs = eventsDict['eventType']
print('event IDs: ' + str(eventIDs))
TTL_values = eventsDict['eventId']
print('shape of TTL values' + str(TTL_values.shape))
print('TTL values: ' + str(TTL_values))
TTL_timeStamps = eventsDict['timestamps']
print('shape of timestamps: ' + str(TTL_timeStamps.shape))
print('time stamps: ' + str(TTL_timeStamps))
### DEBUG
pdb.set_trace()
return TTL_timeStamps, TTL_values
def get_data_spike(folder_path, file_path, name):
data = OpenEphys.load(
file_path) # returns a dict with data, timestamps, etc.
timestamps = data['timestamps']
waveforms = data['spikes']
# print('{} waveforms were found in the spike file'.format(waveforms.shape[0]))
waveforms, timestamps = delete_noise(folder_path, name, waveforms,
timestamps)
return waveforms, timestamps
def loadContinuous(folder, pluginId):
'''
Loads continuous files and returns a MNE data array.
Currently looses all information stored with the data besides electrode number and sampling rate.
Anything else needs to be handled separately.
Inputs:
folder - The folder where the data is held (needs ending \)
pluginId - from 100_CH1.continuous. Normally raw data is from 100. This can be found from config.xml file.
'''
# Get file names and preform human sorting
files = glob.glob(folder + str(pluginId) + '_CH*.continuous')
files.sort(key=alphanum_key)
# Load in Open Ephys data and save channel names
datas = []
chanNames = []
for dataPath in files:
chanNames.append(re.search('100_CH(.*).continuous', dataPath).group(1))
datas.append(OpenEphys.load(dataPath))
print('loading done! Now putting into MNE format')
# Use eeg because that's the closest mne offers
chanTypes = ['eeg'] * len(files)
info = mne.create_info(ch_names=chanNames,
sfreq=datas[0]['header']['sampleRate'],
ch_types=chanTypes)
# Make a list of the data from the channels
rawData = []
for data in datas:
rawData.append(data['data'])
# Put into mne format
matrix = np.array(rawData)
raw = mne.io.RawArray(matrix, info)
print('MNE conversion complete')
return raw
def get_ttls(data_path):
print 'data_path = ', data_path
events_data = OpenEphys.load('all_channels.events')
########## LOAD THE TTLs!!! #############
#temp = np.intersect1d((np.where(events_data['channel'] == 0)[0]), (np.where(events_data['eventId'] == 1)[0]))
ttl0_on = events_data['timestamps'][np.intersect1d((np.where(events_data['channel'] == 0)[0]), (np.where(events_data['eventId'] == 1)[0]))]
#temp = np.intersect1d((np.where(events_data['channel'] == 0)[0]), (np.where(events_data['eventId'] == 0)[0]))
ttl0_off = events_data['timestamps'][np.intersect1d((np.where(events_data['channel'] == 0)[0]), (np.where(events_data['eventId'] == 0)[0]))]
#temp = np.intersect1d((np.where(events_data['channel'] == 1)[0]), (np.where(events_data['eventId'] == 1)[0]))
ttl1_on = events_data['timestamps'][np.intersect1d((np.where(events_data['channel'] == 1)[0]), (np.where(events_data['eventId'] == 1)[0]))]
#temp = np.intersect1d((np.where(events_data['channel'] == 1)[0]), (np.where(events_data['eventId'] == 0)[0]))
ttl1_off = events_data['timestamps'][np.intersect1d((np.where(events_data['channel'] == 1)[0]), (np.where(events_data['eventId'] == 0)[0]))]
return ttl0_on, ttl0_off, ttl1_on, ttl1_off
def loadTTLdata(self, *RAW_DATA_DIR): # from the .events file
if RAW_DATA_DIR:
print('raw data dir specified by user: '******'all_channels.events'))
eventsDict = OE.load(event_path) ### load TTL dict file
eventIDs = eventsDict['eventType']
print('\nloading: ' + event_path)
print(eventsDict['eventType'])
print(eventsDict['eventId'])
print(
'\nall the events that are saved have type TTL = 3 ; Network Event = 5'
)
print('event IDs: ' + str(eventIDs))
self.sampleRate = float(eventsDict['header']['sampleRate'])
TTL_timesInSamples = eventsDict['timestamps']
self.TTL_values = eventsDict['eventId']
self.TTL_timesInSecs = TTL_timesInSamples / self.sampleRate
print('shape of TTL values' + str(self.TTL_values.shape))
print('TTL values: ' + str(self.TTL_values))
print('shape of timestamps: ' + str(TTL_timesInSamples.shape))
print('time stamps: ' + str(TTL_timesInSamples))
print('sample rate (from all_channels.events): ' +
str(self.sampleRate))
print('converting times from sample number to secs')
### skip weird bad values at the beginning... TO DO: find out why these are so rapid yet still have event id 3. check the other data set
# TTL_timeStamps = TTL_times[4::]
# self.TTL_values = self.TTL_values[4::]
return self.TTL_timesInSecs, self.TTL_values
import os
import encode
from mlpy import writemda16i
import sys
cwd=os.getcwd()
sys.path.append(cwd)
sys.path.append('~/repos/mountainlab/old/packages/mountainsort/packages/pyms')
dump = []
for i in range(64):
i+=1
data=[]
data=op.load('2044_2018-06-19_13-42-08/100_CH%s.continuous' %i)
print('loaded %s'%i,'start cutting')
# cutting
timestamps = data['timestamps'][0:30*60*30000]
#writing
# encode, write to another openephys
# encode.writecontinous(filepath='cut_CH%s.continuous'%i,timestamps=timestamps, header=data['header'])
# accumulate to N*M, to mda
dump.append(timestamps) #append to row, need to check if need transpose
#convert to mda,
writemda16i(dump, 'raw30min.mda')
def get_events(prm, file_path):
events = OpenEphys.load(file_path)
return events
def get_data_continuous(prm, file_path):
data = OpenEphys.load(file_path)
signal = data['data']
signal = np.asanyarray(signal)
return signal
# RAW_DATA_PATH = '/home/orthogonull/a_MHR/a_Research/a_WIP/2018-04-29_23-20-48'
# pathToFile = '/home/orthogonull/a_MHR/a_Research/a_WIP/2018-04-29_23-20-48/100_CH1.continuous'
# figSaveDir = '/home/orthogonull/a_MHR/a_Research/a_WIP'
### ephys computer paths
pathToFile = 'C:\\Users\\yatang\\Documents\\Ephys\\Open Ephys\\2018-05-04_13-55-49_MR1_r2_pre\\100_CH1.continuous'
RAW_DATA_PATH = 'C:\\Users\\yatang\\Documents\\Ephys\\Open Ephys\\2018-05-04_13-55-49_MR1_r2_pre\\'
figSaveDir = 'C:\\Users\\yatang\\Documents\\Ephys\\Open Ephys\\'
else:
sys.exit(
'ERROR: user parameters in this script failed to find a valid directory of recordings!'
)
print('loading: ' + RAW_DATA_PATH)
data_dict = OE.load(pathToFile) # returns a dict with data, timestamps, etc.
print(data_dict)
# open ephys's BROKEN (float given but expects int error) downsample code
# def downsample(trace,down):
# downsampled = scipy.signal.resample(trace,np.shape(trace)[0]/down)
# return downsampled
data = OE.loadFolderToArray(RAW_DATA_PATH,
channels=range(FIRST_CH, LAST_CH + 1),
chprefix='CH',
dtype=float,
session='0',
source='100')
print(np.shape(data))
totalNumSamples = np.shape(data)[0] #no longer used but available for later
print(data)
def loadEventsOE(path, tsStart):
raw = OpenEphys.load(path)
return Event(raw, tsStart)
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
Just sketches for now.
"""
#%%
import OpenEphys
import matplotlib.pyplot as plt
OpenEphysData = OpenEphys.load('100_CH6.continuous')
OpenEphysEvents = OpenEphys.load('all_channels.events')
RawData = OpenEphysData["data"].tolist()
Events = OpenEphysEvents["timestamps"].tolist()
EventChannels = OpenEphysEvents["channel"].tolist()
EventTypes = OpenEphysEvents["eventType"].tolist()
TTL0 = [0] * len(RawData)
TTL1 = [0] * len(RawData)
def plot_responses(data_path,save_path):
ttl0_on, ttl0_off, ttl1_on, ttl1_off = get_ttls(data_path)
fs = 3e4
time_axis = np.arange(-.250,.750,1/fs)
channels = range(64) # [0,1] #
allchans_binoc = dict()
######### LOAD EPHYS
for ch in channels:
ephys_data = OpenEphys.load('100_CH%i.continuous' % (ch+1))
filt_data = filter(ephys_data['data'],[1,200])
###### take ttl0_on times (in samples), get corresponding chunks from the ephys (250ms before that time to 750ms after)
offset_time = ephys_data['timestamps'][0]
right_led_ephys = np.zeros([ttl1_on.shape[0],int(1.0*fs)])
for idx,on_time in enumerate(ttl1_on):
offset_ontime = on_time - offset_time
#print idx,on_time,offset_ontime
if offset_ontime-int(.250*fs)>0:
if filt_data.shape[0] - offset_ontime > 0:
if offset_ontime+int(.750*fs) <filt_data.shape[0] :
right_led_ephys[idx,:] = filt_data[int(offset_ontime-int(.250*fs)):int(offset_ontime+int(.750*fs))]
left_led_ephys = np.zeros([ttl0_on.shape[0],int(1.0*fs)])
for idx,on_time in enumerate(ttl0_on):
offset_ontime = on_time - offset_time
#print idx,on_time,offset_ontime
if offset_ontime-int(.250*fs)>0:
if filt_data.shape[0] - offset_ontime > 0:
if offset_ontime+int(.750*fs) <filt_data.shape[0] :
left_led_ephys[idx,:] = filt_data[int(offset_ontime-int(.250*fs)):int(offset_ontime+int(.750*fs))]
mean_left_led = np.mean(left_led_ephys,axis=0)
#std_left_led = np.std(left_led_ephys,axis=0)
sem_left_led = stats.sem(left_led_ephys,axis=0)
mean_right_led = np.mean(right_led_ephys,axis=0)
#std_right_led = np.std(right_led_ephys,axis=0)
sem_right_led = stats.sem(right_led_ephys,axis=0)
##### binocularity index for each electrode:
left_max = np.max(abs(mean_left_led))
right_max = np.max(abs(mean_right_led))
binocularity = (left_max - right_max) / (left_max + right_max)
print 'binocularity for ch %i = %6f' % ((ch+1),binocularity)
allchans_binoc[str(ch+1)] = [binocularity]
plot = 1
if plot == 1:
f, axarr = plt.subplots(2, sharex=True,sharey=True) #plt.figure(dpi=600)
f.dpi = 600
f.suptitle('binocularity for ch %i = %6f' % (ch,binocularity))
#ax = sns.tsplot(data=mean_left_led_ch22,time=time_axis, linewidth=0.1)
axarr[0].plot(time_axis,mean_left_led,linewidth=0.1)
axarr[0].fill_between(time_axis, mean_left_led-sem_left_led, mean_left_led+sem_left_led,alpha=0.1)
axarr[0].set_ylabel('Left LED Response (uV)')
axarr[1].plot(time_axis,mean_right_led,linewidth=0.1)
axarr[1].fill_between(time_axis, mean_right_led-sem_right_led, mean_right_led+sem_right_led,alpha=0.1)
axarr[1].set_ylabel('Right LED Response (uV)')
axarr[1].set_xlabel('Time (sec)')
axarr[1].set_xticks
start, end = axarr[1].get_xlim()
stepsize = 0.1
axarr[1].xaxis.set_ticks(np.arange(start, end, stepsize))
# add a square wave showing when the stimulus happened:
plt.plot([-0.25,0,0,0.25,0.25,0.75],[-75,-75,-50,-50,-75,-75],color='black')
f.savefig(save_path + 'ch%i.pdf'%(ch+1),dpi=600)
import sys
sys.path.append(
'/Volumes/Mac HD/Dropbox (coxlab)/Scripts/Repositories/continuous-ephys/open-ephys analysis/'
)
import OpenEphys
import numpy as np
import matplotlib.pyplot as plt
#eventsfile = "/Volumes/labuser/Desktop/EPHYS/test data/10HzDigitalInput/2015-05-11_14-06-08_diginputtest/all_channels.events"
#eventsfile = "/Volumes/GG Data Raid/Ephys/2015-04-29_17-35-04_digitalinputtest/all_channels_4.events"
#eventsfile = "/Volumes/labuser/Desktop/EPHYS/test data/MWorksPixelInput/2015-05-11_15-53-23_digintest/all_channels.events"
#eventsfile = "/Volumes/labuser/Desktop/EPHYS/test data/MWorksPixelInput/2015-05-12_12-00-22_digintest/all_channels.events"
#eventsfile = "/Volumes/labuser/Desktop/EPHYS/test data/MWorksPixelInput/2015-05-12_17-51-29_captest/all_channels_2.events"
eventsfile = "/Volumes/labuser/Desktop/EPHYS/Data/grat03/2015-05-30_12-46-18/all_channels.events"
events_data = OpenEphys.load(eventsfile)
pixel_ch1 = [] # np.zeros((len(events_data['timestamps']),1))
pixel_ch2 = [] #np.zeros((len(events_data['timestamps']),1))
pixel_ch1_time = [] #np.zeros((len(events_data['timestamps']),1))
pixel_ch2_time = []
counter = 0
while counter < len(events_data['timestamps']): # go thru all timestamps
if events_data['channel'][counter] == 3:
if events_data['eventId'][counter] == 1:
pixel_ch1.append(1)
pixel_ch1_time.append(events_data['timestamps'][counter])
elif events_data['eventId'][counter] == 0:
pixel_ch1.append(0)
pixel_ch1_time.append(events_data['timestamps'][counter])
if rank == size-1:
last_val = num_files
else:
last_val = np.floor((rank+1)*num_files/size)
rank_vals = np.arange(st_val, last_val)
rank_vals = rank_vals.astype(int)
data = {}
for i in rank_vals:
print('i', i)
st = time.time()
data[continuous_files[i].replace('.continuous', '')] = OpenEphys.load(folderpath + '/'+continuous_files[i])
en = time.time()
print(en-st)
print('done')
#data = OpenEphys.loadFolder('D:/Open_ephys_testing_ground_data')
#df = pandas.DataFrame(data)
#df.to_hdf('./trialhdf', 'trial')
def loadConOE(path):
raw = OpenEphys.load(path)
return Con(raw)
import OpenEphys as OE
if __name__ == '__main__':
data = OE.load('/Users/fpbatta/dataLisa/rat27_plusmaze_base_II_2016-03-24_14-10-08/100_CH9.continuous')
print(data)
