def load_events(file_path, channels=None):
"""
return time stamps in seconds of each digital channel
:param file_path:
:param channels: name of channels
:return: dictionary, {channel: {'rise':[timestamps of rising events in seconds],
'fall':[timestamps of falling events in seconds}}
"""
print('\n')
if file_path[-7:] != '.events':
raise LookupError('The input file: ' + file_path + ' is not a .events file!')
with open(file_path) as f:
header = oe.readHeader(f)
fs = float(header['sampleRate'])
events = oe.loadEvents(file_path)
detected_channel_number = int(max(events['channel']) + 1)
real_channels = ['ch_' + ft.int2str(c, 3) for c in range(detected_channel_number)]
if channels is not None:
if detected_channel_number != len(channels):
warning_msg = '\nThe number of digital channels detected: ' + str(detected_channel_number) + \
' does not match input channel number: ' + str(len(channels))
warnings.warn(warning_msg)
if len(channels) <= detected_channel_number:
real_channels[0:len(channels)] = channels
else:
real_channels = channels[0:detected_channel_number]
output = {}
for i, ch in enumerate(real_channels):
output.update({ch : {'rise' : [],
'fall' : []}
})
rise = events['timestamps'][np.logical_and(events['channel'] == i, events['eventId'] == 1)]
output[ch]['rise'] = np.array(rise.astype(np.float32) / fs).astype(np.float32)
fall = events['timestamps'][np.logical_and(events['channel'] == i, events['eventId'] == 0)]
output[ch]['fall'] = np.array(fall.astype(np.float32) / fs).astype(np.float32)
print('events loaded.\n')
return output
def load_sync_data_ephys(recording_to_process, prm):
is_found = False
sync_data = None
print('loading sync channel...')
file_path = recording_to_process + '/' + prm.get_sync_channel()
if os.path.exists(file_path):
sync_data = open_ephys_IO.get_data_continuous(prm, file_path)
is_found = True
else:
print(
'Sync data was not found, I will check if Axona sync data is present and convert it if it is.'
)
events_file = recording_to_process + '/all_channels.events'
if os.path.exists(events_file):
events = OpenEphys.load(events_file)
time_stamps = events['timestamps']
channel = events['channel']
pulse_indices = time_stamps[np.where(channel == 0)]
# for sample in pulse_indices: # make pulse wider
# pulse_indices = np.append(pulse_indices, np.arange(sample, (sample + 5000)))
# load any continuous data file to get length of recording
for name in glob.glob(recording_to_process + '/*.continuous'):
if os.path.exists(name):
print(name)
ch = open_ephys_IO.get_data_continuous(prm, name)
length = len(ch)
sync_data = np.zeros(length)
sync_data[np.take(
pulse_indices,
np.where(pulse_indices < len(ch))).astype(int)] = 1
is_found = True
return sync_data, is_found
return sync_data, is_found
def loadTTLdata(rawDataDir): # from the .events file
event_path = str(Path(rawDataDir).joinpath('all_channels.events'))
print('loading: ' + event_path)
eventsDict = OE.load(event_path) ### load TTL dict file
print(eventsDict)
print(eventsDict['eventType'])
print(eventsDict['eventId'])
print(
'\nall the events that are saved have type TTL = 3 ; Network Event = 5'
)
eventIDs = eventsDict['eventType']
print('event IDs: ' + str(eventIDs))
TTL_values = eventsDict['eventId']
print('shape of TTL values' + str(TTL_values.shape))
print('TTL values: ' + str(TTL_values))
TTL_timeStamps = eventsDict['timestamps']
print('shape of timestamps: ' + str(TTL_timeStamps.shape))
print('time stamps: ' + str(TTL_timeStamps))
### DEBUG
pdb.set_trace()
return TTL_timeStamps, TTL_values
def get_data_spike(folder_path, file_path, name):
data = OpenEphys.load(
file_path) # returns a dict with data, timestamps, etc.
timestamps = data['timestamps']
waveforms = data['spikes']
# print('{} waveforms were found in the spike file'.format(waveforms.shape[0]))
waveforms, timestamps = delete_noise(folder_path, name, waveforms,
timestamps)
return waveforms, timestamps
def loadRawOEdir(rawDataDir):
data = OE.loadFolderToArray(rawDataDir,
channels=range(FIRST_CH, LAST_CH + 1),
chprefix='CH',
dtype=float,
session='0',
source='100')
print(np.shape(data))
totalNumSamples = np.shape(data)[
0] #no longer used but available for later
print(data)
return data
def load(path, nchan, cutmin):
ndpoints = cutmin * 60 * 30000
dim = int(math.floor((ndpoints / 1024)))
raw = np.empty((dim * 1024, nchan))
for i in range(nchan):
i += 1
# dump=[]
data = op.loadContinuous(str(path) + '\\100_CH%s.continuous' % i,
stop_record=ndpoints / 1024)
print('loaded %s' % i, 'start cutting')
# cutting
timestamps = data['data'] #[0:cutmin*60*30000]
raw[:, i - 1] = (timestamps)
return raw
def loadContinuous(folder, pluginId):
'''
Loads continuous files and returns a MNE data array.
Currently looses all information stored with the data besides electrode number and sampling rate.
Anything else needs to be handled separately.
Inputs:
folder - The folder where the data is held (needs ending \)
pluginId - from 100_CH1.continuous. Normally raw data is from 100. This can be found from config.xml file.
'''
# Get file names and preform human sorting
files = glob.glob(folder + str(pluginId) + '_CH*.continuous')
files.sort(key=alphanum_key)
# Load in Open Ephys data and save channel names
datas = []
chanNames = []
for dataPath in files:
chanNames.append(re.search('100_CH(.*).continuous', dataPath).group(1))
datas.append(OpenEphys.load(dataPath))
print('loading done! Now putting into MNE format')
# Use eeg because that's the closest mne offers
chanTypes = ['eeg'] * len(files)
info = mne.create_info(ch_names=chanNames,
sfreq=datas[0]['header']['sampleRate'],
ch_types=chanTypes)
# Make a list of the data from the channels
rawData = []
for data in datas:
rawData.append(data['data'])
# Put into mne format
matrix = np.array(rawData)
raw = mne.io.RawArray(matrix, info)
print('MNE conversion complete')
return raw
def loadRawOEdir(rawDataDir):
# print('loading: ' + pathToFile)
# data_dict = OE.load(pathToFile) # returns a dict with data, timestamps, etc.
# print(data_dict)
# # open ephys's BROKEN (float given but expects int error) downsample code
# # def downsample(trace,down):
# # downsampled = scipy.signal.resample(trace,np.shape(trace)[0]/down)
# # return downsampled
data = OE.loadFolderToArray(rawDataDir,
channels=range(FIRST_CH, LAST_CH + 1),
chprefix='CH',
dtype=float,
session='0',
source='100')
print(np.shape(data))
totalNumSamples = np.shape(data)[
0] #no longer used but available for later
print(data)
return data
def get_ttls(data_path):
print 'data_path = ', data_path
events_data = OpenEphys.load('all_channels.events')
########## LOAD THE TTLs!!! #############
#temp = np.intersect1d((np.where(events_data['channel'] == 0)[0]), (np.where(events_data['eventId'] == 1)[0]))
ttl0_on = events_data['timestamps'][np.intersect1d((np.where(events_data['channel'] == 0)[0]), (np.where(events_data['eventId'] == 1)[0]))]
#temp = np.intersect1d((np.where(events_data['channel'] == 0)[0]), (np.where(events_data['eventId'] == 0)[0]))
ttl0_off = events_data['timestamps'][np.intersect1d((np.where(events_data['channel'] == 0)[0]), (np.where(events_data['eventId'] == 0)[0]))]
#temp = np.intersect1d((np.where(events_data['channel'] == 1)[0]), (np.where(events_data['eventId'] == 1)[0]))
ttl1_on = events_data['timestamps'][np.intersect1d((np.where(events_data['channel'] == 1)[0]), (np.where(events_data['eventId'] == 1)[0]))]
#temp = np.intersect1d((np.where(events_data['channel'] == 1)[0]), (np.where(events_data['eventId'] == 0)[0]))
ttl1_off = events_data['timestamps'][np.intersect1d((np.where(events_data['channel'] == 1)[0]), (np.where(events_data['eventId'] == 0)[0]))]
return ttl0_on, ttl0_off, ttl1_on, ttl1_off
def loadTTLdata(self, *RAW_DATA_DIR): # from the .events file
if RAW_DATA_DIR:
print('raw data dir specified by user: '******'all_channels.events'))
eventsDict = OE.load(event_path) ### load TTL dict file
eventIDs = eventsDict['eventType']
print('\nloading: ' + event_path)
print(eventsDict['eventType'])
print(eventsDict['eventId'])
print(
'\nall the events that are saved have type TTL = 3 ; Network Event = 5'
)
print('event IDs: ' + str(eventIDs))
self.sampleRate = float(eventsDict['header']['sampleRate'])
TTL_timesInSamples = eventsDict['timestamps']
self.TTL_values = eventsDict['eventId']
self.TTL_timesInSecs = TTL_timesInSamples / self.sampleRate
print('shape of TTL values' + str(self.TTL_values.shape))
print('TTL values: ' + str(self.TTL_values))
print('shape of timestamps: ' + str(TTL_timesInSamples.shape))
print('time stamps: ' + str(TTL_timesInSamples))
print('sample rate (from all_channels.events): ' +
str(self.sampleRate))
print('converting times from sample number to secs')
### skip weird bad values at the beginning... TO DO: find out why these are so rapid yet still have event id 3. check the other data set
# TTL_timeStamps = TTL_times[4::]
# self.TTL_values = self.TTL_values[4::]
return self.TTL_timesInSecs, self.TTL_values
# RAW_DATA_PATH = '/home/orthogonull/a_MHR/a_Research/a_WIP/2018-04-29_23-20-48'
# pathToFile = '/home/orthogonull/a_MHR/a_Research/a_WIP/2018-04-29_23-20-48/100_CH1.continuous'
# figSaveDir = '/home/orthogonull/a_MHR/a_Research/a_WIP'
### ephys computer paths
pathToFile = 'C:\\Users\\yatang\\Documents\\Ephys\\Open Ephys\\2018-05-04_13-55-49_MR1_r2_pre\\100_CH1.continuous'
RAW_DATA_PATH = 'C:\\Users\\yatang\\Documents\\Ephys\\Open Ephys\\2018-05-04_13-55-49_MR1_r2_pre\\'
figSaveDir = 'C:\\Users\\yatang\\Documents\\Ephys\\Open Ephys\\'
else:
sys.exit(
'ERROR: user parameters in this script failed to find a valid directory of recordings!'
)
print('loading: ' + RAW_DATA_PATH)
data_dict = OE.load(pathToFile) # returns a dict with data, timestamps, etc.
print(data_dict)
# open ephys's BROKEN (float given but expects int error) downsample code
# def downsample(trace,down):
# downsampled = scipy.signal.resample(trace,np.shape(trace)[0]/down)
# return downsampled
data = OE.loadFolderToArray(RAW_DATA_PATH,
channels=range(FIRST_CH, LAST_CH + 1),
chprefix='CH',
dtype=float,
session='0',
source='100')
print(np.shape(data))
totalNumSamples = np.shape(data)[0] #no longer used but available for later
print(data)
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
Just sketches for now.
"""
#%%
import OpenEphys
import matplotlib.pyplot as plt
OpenEphysData = OpenEphys.load('100_CH6.continuous')
OpenEphysEvents = OpenEphys.load('all_channels.events')
RawData = OpenEphysData["data"].tolist()
Events = OpenEphysEvents["timestamps"].tolist()
EventChannels = OpenEphysEvents["channel"].tolist()
EventTypes = OpenEphysEvents["eventType"].tolist()
TTL0 = [0] * len(RawData)
TTL1 = [0] * len(RawData)
#del stim_array[0]
#st= np.array(stim_array)/25000.
##file_name = '/home/cat/neuron/in_vivo/dan/M149873/2015-02-06_13-02-32_flash_CSD/data/stim_times.txt'
#file_name = file_dir+'stim_times.txt'
#np.savetxt(file_name,st, fmt='%8.3f')
#quit()
#******************************* LOAD RAW DATA AND PREP FOR .TSF FILE ***************************
data=[]
for i in range(128):
file_name = file_dir + '100_CH'+ str(i+1) +'.continuous'
data.append(oe.loadContinuous(file_name)['data'])
dt = np.array(data, dtype=np.int16)
print dt.shape
####Plot first 10000 steps from channel 0
#x = np.arange(0,len(dt1),1.)/SampleFrequency
#plt.plot(x,dt1, 'r-', color='black',linewidth=1)
#plt.plot(x,dt2+1000, 'r-', color='black',linewidth=1)
#plt.plot(x,dt[probe128[0]][0:100000], 'r-', color='black',linewidth=1)
#plt.plot(x,dt[probe128[2]][0:100000]-100, 'r-', color='black',linewidth=1)
#plt.show()
def wirtecontinous(filepath,
timestamps,
header,
dtype=float,
start_record=None,
stop_record=None,
ignore_last_record=True):
# This is what the record marker should look like
spec_record_marker = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 255])
recordingNumbers = []
samples = []
samples_read = 0
records_read = 0
# Open the file
with file(filepath, 'wb') as f:
# Read header info, file length, and number of records
header = header
record_length_bytes = 2 * header['blockLength'] + 22
fileLength = os.fstat(f.fileno()).st_size
n_records = op.get_number_of_records(filepath)
# Use this to set start and stop records if not specified
if start_record is None:
start_record = 0
if stop_record is None:
stop_record = n_records
# We'll stop reading after this many records are read
n_records_to_read = stop_record - start_record
# Seek to the start location, relative to the current position
# right after the header.
f.seek(record_length_bytes * start_record, 1)
# Keep reading till the file is finished
while f.tell() < fileLength and records_read < n_records_to_read:
# Skip the last record if requested, which usually contains
# incomplete data
if ignore_last_record and f.tell() == (fileLength -
record_length_bytes):
break
# Read the timestamp for this record
# litte-endian 64-bit signed integer
timestamps.append(np.fromfile(f, np.dtype('<i8'), 1))
# Read the number of samples in this record
# little-endian 16-bit unsigned integer
N = np.fromfile(f, np.dtype('<u2'), 1).item()
# Read and store the recording numbers
# big-endian 16-bit unsigned integer
recordingNumbers.append(np.fromfile(f, np.dtype('>u2'), 1))
# Read the data
# big-endian 16-bit signed integer
data = np.fromfile(f, np.dtype('>i2'), N)
if len(data) != N:
raise IOError("could not load the right number of samples")
# Optionally convert dtype
if dtype == float:
data = data * header['bitVolts']
# Store the data
samples.append(data)
# Extract and test the record marker
record_marker = np.fromfile(f, np.dtype('<u1'), 10)
if np.any(record_marker != spec_record_marker):
raise IOError("corrupted record marker at record %d" %
records_read)
# Update the count
samples_read += len(samples)
records_read += 1
import os
import encode
from mlpy import writemda16i
import sys
cwd=os.getcwd()
sys.path.append(cwd)
sys.path.append('~/repos/mountainlab/old/packages/mountainsort/packages/pyms')
dump = []
for i in range(64):
i+=1
data=[]
data=op.load('2044_2018-06-19_13-42-08/100_CH%s.continuous' %i)
print('loaded %s'%i,'start cutting')
# cutting
timestamps = data['timestamps'][0:30*60*30000]
#writing
# encode, write to another openephys
# encode.writecontinous(filepath='cut_CH%s.continuous'%i,timestamps=timestamps, header=data['header'])
# accumulate to N*M, to mda
dump.append(timestamps) #append to row, need to check if need transpose
#convert to mda,
writemda16i(dump, 'raw30min.mda')
def get_events(prm, file_path):
events = OpenEphys.load(file_path)
return events
import sys
sys.path.append(
'/Volumes/Mac HD/Dropbox (coxlab)/Scripts/Repositories/continuous-ephys/open-ephys analysis/'
)
import OpenEphys
import numpy as np
import matplotlib.pyplot as plt
#eventsfile = "/Volumes/labuser/Desktop/EPHYS/test data/10HzDigitalInput/2015-05-11_14-06-08_diginputtest/all_channels.events"
#eventsfile = "/Volumes/GG Data Raid/Ephys/2015-04-29_17-35-04_digitalinputtest/all_channels_4.events"
#eventsfile = "/Volumes/labuser/Desktop/EPHYS/test data/MWorksPixelInput/2015-05-11_15-53-23_digintest/all_channels.events"
#eventsfile = "/Volumes/labuser/Desktop/EPHYS/test data/MWorksPixelInput/2015-05-12_12-00-22_digintest/all_channels.events"
#eventsfile = "/Volumes/labuser/Desktop/EPHYS/test data/MWorksPixelInput/2015-05-12_17-51-29_captest/all_channels_2.events"
eventsfile = "/Volumes/labuser/Desktop/EPHYS/Data/grat03/2015-05-30_12-46-18/all_channels.events"
events_data = OpenEphys.load(eventsfile)
pixel_ch1 = [] # np.zeros((len(events_data['timestamps']),1))
pixel_ch2 = [] #np.zeros((len(events_data['timestamps']),1))
pixel_ch1_time = [] #np.zeros((len(events_data['timestamps']),1))
pixel_ch2_time = []
counter = 0
while counter < len(events_data['timestamps']): # go thru all timestamps
if events_data['channel'][counter] == 3:
if events_data['eventId'][counter] == 1:
pixel_ch1.append(1)
pixel_ch1_time.append(events_data['timestamps'][counter])
elif events_data['eventId'][counter] == 0:
pixel_ch1.append(0)
pixel_ch1_time.append(events_data['timestamps'][counter])
if rank == size-1:
last_val = num_files
else:
last_val = np.floor((rank+1)*num_files/size)
rank_vals = np.arange(st_val, last_val)
rank_vals = rank_vals.astype(int)
data = {}
for i in rank_vals:
print('i', i)
st = time.time()
data[continuous_files[i].replace('.continuous', '')] = OpenEphys.load(folderpath + '/'+continuous_files[i])
en = time.time()
print(en-st)
print('done')
#data = OpenEphys.loadFolder('D:/Open_ephys_testing_ground_data')
#df = pandas.DataFrame(data)
#df.to_hdf('./trialhdf', 'trial')
#==============================================================================#
ntot = len(dfold_list)
ind = 0
for dfold in dfold_list:
ind = ind + 1
nrem = ntot - ind
print('Converting ' + dfold + ': ' + str(nrem) + ' remaining')
folderpath = ddir + dfold # folderpath where data is stored
#=============== SUBSAMPLE LOWPASS (LFP) SIGNAL AND CONVERT TO DAT ===========#
lfpname = '102_CH2.continuous'
filepath = ddir + dfold + '/' + lfpname
LFP = OpenEphys.loadContinuous(filepath, dtype=float)
LFP = LFP['data']
ssLFP = LFP[::10].copy() #subsampled LFP
# save as dat file
datnameLFP = dfold + '_LFP.dat'
ssLFP.tofile(os.path.join(folderpath, datnameLFP))
#==============================================================================#
#======================== CONVERT HIGHPASS SIGNALS TO DAT ======================#
# Load highpass signals of each sahnk in geometric order (bottom of shank to top)
# then save to flat binary file
# create shank1 and shank2 folders
os.makedirs(folderpath + '/2017/' + shank1_fname)
os.makedirs(folderpath + '/2017/' + shank2_fname)
# -*- coding: utf-8 -*-
"""
Created on Mon Jun 11 13:26:20 2018
@author: behrenslab
"""
import data_import as di
import OpenEphys as op
import Esync as es
import numpy as np
session = di.Session(
'/home/behrenslab/ephys/2018-06-05-reversal_learning_3_tasks_recording/m483-2018-06-07-161545.txt'
)
rsync_events = [
event.time for event in session.events if event.name in ['Rsync']
]
rsync_ephys = op.loadEvents(
'/home/behrenslab/ephys/2018-06-07_16-15-43/all_channels.events')
rsync_timestamps = rsync_ephys['timestamps']
rsync_timestamps = np.array(rsync_timestamps)[::2]
rsync_events = np.array(rsync_events)
rsync_timestamps = rsync_timestamps / 30
aligner = es.Rsync_aligner(rsync_events, rsync_timestamps, plot=True)
times_B = aligner.A_to_B(rsync_events) #A pycontrol
times_A = aligner.B_to_A(rsync_timestamps)
pycontrol_to_ephys = aligner.A_to_B
for file_ephys in files_ephys:
if file_ephys in m486:
match_ephys = re.search(r'\d{4}-\d{2}-\d{2}', file_ephys)
date_ephys = datetime.strptime(match_ephys.group(), '%Y-%m-%d').date()
date_ephys = match_ephys.group()
for file_behaviour in files_behaviour:
match_behaviour = re.search(r'\d{4}-\d{2}-\d{2}', file_behaviour)
date_behaviour = datetime.strptime(match_behaviour.group(), '%Y-%m-%d').date()
date_behaviour = match_behaviour.group()
if date_ephys == date_behaviour:
behaviour_path = behaviour_filename+'/'+file_behaviour
behaviour_session = di.Session(behaviour_path)
ephys_path = ephys_data_folder+'/'+file_ephys + '/' + subject
print(behaviour_path)
print(ephys_path)
ephys_events = op.loadEvents(os.path.join(ephys_path,'all_channels.events'))
data_folder = file_ephys
check_if_kilosort_exists = os.listdir(ephys_path)
check_if_npy_exists = os.listdir(spikes_df_csv_out_folder)
file_ephys_npy = file_ephys +'.npy'
if file_ephys_npy not in check_if_npy_exists:
# For multiple files in one session add the requirement of cluster_groups.csv file
for file in check_if_kilosort_exists:
if fnmatch.fnmatch(file, '*.csv'):
# Get offset in samples of Kilosort spike data relative to TTL events.
with open(os.path.join(ephys_path,'messages.events')) as f:
message = f.read()
recording_start_sample, sampling_rate = [
int(x) for x in message.split('start time: ')[1].split('Hz')[0].split('@')]
def get_data_continuous(prm, file_path):
data = OpenEphys.load(file_path)
signal = data['data']
signal = np.asanyarray(signal)
return signal
def plot_responses(data_path,save_path):
ttl0_on, ttl0_off, ttl1_on, ttl1_off = get_ttls(data_path)
fs = 3e4
time_axis = np.arange(-.250,.750,1/fs)
channels = range(64) # [0,1] #
allchans_binoc = dict()
######### LOAD EPHYS
for ch in channels:
ephys_data = OpenEphys.load('100_CH%i.continuous' % (ch+1))
filt_data = filter(ephys_data['data'],[1,200])
###### take ttl0_on times (in samples), get corresponding chunks from the ephys (250ms before that time to 750ms after)
offset_time = ephys_data['timestamps'][0]
right_led_ephys = np.zeros([ttl1_on.shape[0],int(1.0*fs)])
for idx,on_time in enumerate(ttl1_on):
offset_ontime = on_time - offset_time
#print idx,on_time,offset_ontime
if offset_ontime-int(.250*fs)>0:
if filt_data.shape[0] - offset_ontime > 0:
if offset_ontime+int(.750*fs) <filt_data.shape[0] :
right_led_ephys[idx,:] = filt_data[int(offset_ontime-int(.250*fs)):int(offset_ontime+int(.750*fs))]
left_led_ephys = np.zeros([ttl0_on.shape[0],int(1.0*fs)])
for idx,on_time in enumerate(ttl0_on):
offset_ontime = on_time - offset_time
#print idx,on_time,offset_ontime
if offset_ontime-int(.250*fs)>0:
if filt_data.shape[0] - offset_ontime > 0:
if offset_ontime+int(.750*fs) <filt_data.shape[0] :
left_led_ephys[idx,:] = filt_data[int(offset_ontime-int(.250*fs)):int(offset_ontime+int(.750*fs))]
mean_left_led = np.mean(left_led_ephys,axis=0)
#std_left_led = np.std(left_led_ephys,axis=0)
sem_left_led = stats.sem(left_led_ephys,axis=0)
mean_right_led = np.mean(right_led_ephys,axis=0)
#std_right_led = np.std(right_led_ephys,axis=0)
sem_right_led = stats.sem(right_led_ephys,axis=0)
##### binocularity index for each electrode:
left_max = np.max(abs(mean_left_led))
right_max = np.max(abs(mean_right_led))
binocularity = (left_max - right_max) / (left_max + right_max)
print 'binocularity for ch %i = %6f' % ((ch+1),binocularity)
allchans_binoc[str(ch+1)] = [binocularity]
plot = 1
if plot == 1:
f, axarr = plt.subplots(2, sharex=True,sharey=True) #plt.figure(dpi=600)
f.dpi = 600
f.suptitle('binocularity for ch %i = %6f' % (ch,binocularity))
#ax = sns.tsplot(data=mean_left_led_ch22,time=time_axis, linewidth=0.1)
axarr[0].plot(time_axis,mean_left_led,linewidth=0.1)
axarr[0].fill_between(time_axis, mean_left_led-sem_left_led, mean_left_led+sem_left_led,alpha=0.1)
axarr[0].set_ylabel('Left LED Response (uV)')
axarr[1].plot(time_axis,mean_right_led,linewidth=0.1)
axarr[1].fill_between(time_axis, mean_right_led-sem_right_led, mean_right_led+sem_right_led,alpha=0.1)
axarr[1].set_ylabel('Right LED Response (uV)')
axarr[1].set_xlabel('Time (sec)')
axarr[1].set_xticks
start, end = axarr[1].get_xlim()
stepsize = 0.1
axarr[1].xaxis.set_ticks(np.arange(start, end, stepsize))
# add a square wave showing when the stimulus happened:
plt.plot([-0.25,0,0,0.25,0.25,0.75],[-75,-75,-50,-50,-75,-75],color='black')
f.savefig(save_path + 'ch%i.pdf'%(ch+1),dpi=600)
#file_dir = 'M150566/2015-04-01_16-41-14_flashes_vis/'
#file_dir = 'M150566/2015-04-01_20-55-32_flashes_vis/'
#file_dir = 'M181420/181420_2015-04-07_13-25-07_flashes_vis/'
#file_dir = 'M181420/181420_2015-04-07_18-10-46_flash_vis/'
##********************** LOAD STIMULUS TIMES AND SAVE THEM *****************************
if False:
file_name = '103_PAI7.StimTimeshere.continuous'
data=[]
data = oe.loadContinuous(file_dir+mouse_dir+'/'+exp+dir+'/'+file_name)
dt = np.array(data['data'])
stim_array=[]
state=0
for i in range(len(dt)):
if state==0:
if dt[i]>1:
print "ON"
temp=i
state=1
if state==1:
if dt[i]<1:
print "OFF"
stim_array.append([temp,i])
state=0
import OpenEphys as OE
if __name__ == '__main__':
data = OE.load('/Users/fpbatta/dataLisa/rat27_plusmaze_base_II_2016-03-24_14-10-08/100_CH9.continuous')
print(data)
pathname = f"./plx/{dirname}"
#Ephys形式で記録されたチャネルマップ配置
channel_map = [9, 8, 10, 7, 13, 4, 12, 5, 15, 2, 16, 1, 14, 3, 11, 6]
#刺激方法の設定をSetting.jsonのファイルから読み込む
#jsonファイルは日程-時間で記録されているため、result.jsonに記載された日程を参照する
with open(f"./json/{jsonfile[:14]}Settings.json",
encoding="UTF-8") as json_fp:
stim_setting = json.load(json_fp)
if not (os.path.exists(f"./waveform/{dirname}")):
os.mkdir(f"./waveform/{dirname}")
num = 1
for i in channel_map:
#OpenEphys.pyを使用してcontinuousファイルの読み込み
tmp = OpenEphys.loadContinuous(
f"./plx/{dirname}/101_CH{str(i)}.continuous")
#lowpassフィルタがかけられた波形を抽出
waveform = glia.nucleus.Waveform(lowpass(tmp["data"], 30000, 450, 600,
10, 30),
rate=glia.Frequency(30000))
#tmp(['header', 'timestamps', 'data', 'recordingNumber'])が記載されている
#timestampsはサンプリングレートで記録された記録点
timestamps = tmp["timestamps"][0]
if num < 10:
glia.save(f"./waveform/{dirname}/0{num}.lfp", waveform)
else:
glia.save(f"./waveform/{dirname}/{num}.lfp", waveform)
del tmp, waveform
gc.collect()
num += 1
# continue
# elif ephys_folder.find('m480') == 104:
# if 'm480.dat' not in os.listdir(ephys_folder):
# op.pack_2(ephys_folder,filename =ephys_folder +'/'+'m480'+'.dat', channels =channels_m486_m482_m480_m478, chprefix = 'CH', dref = 'ave')
# elif ephys_folder.find('m478') == 104:
# if 'm478.dat' not in os.listdir(ephys_folder):
# op.pack_2(ephys_folder, filename =ephys_folder +'/'+'m478'+'.dat', channels =channels_m486_m482_m480_m478, chprefix = 'CH', dref = 'ave')
# elif ephys_folder.find('m486') == 104:
# if 'm486.dat' not in os.listdir(ephys_folder):
# op.pack_2(ephys_folder, filename =ephys_folder +'/'+'m486'+'.dat', channels =channels_m486_m482_m480_m478, chprefix ='CH', dref = 'ave')
# elif ephys_folder.find('m482') == 104:
# if 'm482.dat' not in os.listdir(ephys_folder):
# op.pack_2(ephys_folder, filename =ephys_folder +'/'+'m482'+'.dat', channels =channels_m486_m482_m480_m478, chprefix ='CH', dref = 'ave')
# elif ephys_folder.find('m484') == 109:
# if 'm484.dat' not in os.listdir(ephys_folder):
# op.pack_2(ephys_folder, filename =ephys_folder +'/'+'m484'+'.dat', channels =channels_m484_m479, chprefix ='CH', dref = 'ave')
# elif ephys_folder.find('m479') == 109:
# if 'm479.dat' not in os.listdir(ephys_folder):
# op.pack_2(ephys_folder, filename =ephys_folder +'/'+'m479'+'.dat', channels =channels_m484_m479, chprefix ='CH', dref = 'ave')
#
for file in files:
ephys_folder = os.path.join(files_folder, file)
if ephys_folder.find('m480') == 104:
if 'm480.dat' not in os.listdir(ephys_folder):
op.pack_2(ephys_folder,
filename=ephys_folder + '/' + 'm480' + '.dat',
channels=channels_m486_m482_m480_m478_second_box,
chprefix='CH',
dref='ave')
def load_continuous(file_path, dtype=np.float32, start_ind=0):
"""
Jun's wrapper to load .continuous data from OpenEphys data files, this will try to load the whole file into memory,
using np.fromfile. It can also start from any position in the file (defined by the start_ind)
:param file_path:
:param dtype: np.float32 or np.int16
:param start_ind: non-negative int, default 0. start index to extract data.
:return: header: dictionary, standard open ephys header for continuous file
samples: 1D np.array
"""
assert dtype in (np.float32, np.int16), \
'Invalid data type specified for loadContinous, valid types are np.float32 and np.int16'
print("\nLoading continuous data from " + file_path)
bytes_per_block = CONTINUOUS_TIMESTAMP_DTYPE.itemsize + CONTINUOUS_SAMPLE_PER_RECORD_DTYPE.itemsize + \
CONTINUOUS_RECORDING_NUMBER_DTYPE.itemsize + CONTINUOUS_MARKER_BYTES + \
CONTINUOUS_SAMPLE_DTYPE.itemsize * oe.SAMPLES_PER_RECORD
input_array = np.fromfile(file_path, dtype='<u1')
file_length = input_array.shape[0]
print('total length of the file: ', file_length, 'bytes.')
valid_block_start = find_next_valid_block(input_array, bytes_per_block=bytes_per_block, start_index=start_ind)
print('the beginning index of the first valid block after index: ' + str(start_ind) + ' is ' + \
str(valid_block_start))
print('bytes per record block: ', bytes_per_block)
# read in the data
f = open(file_path, 'rb')
header = oe.readHeader(f)
block_num = (file_length - valid_block_start) // bytes_per_block
print('number of potential valid blocks after index', start_ind, ':', block_num)
samples = np.empty(oe.SAMPLES_PER_RECORD * block_num, dtype=dtype)
_ = np.fromfile(f, np.dtype('<u1'), valid_block_start - oe.NUM_HEADER_BYTES)
for i in range(block_num):
if i == 0:
# to get the timestamp of the very first record (block)
# for alignment of the digital event
start_ind = np.fromfile(f, CONTINUOUS_TIMESTAMP_DTYPE, 1)
start_time = float(start_ind) / float(header['sampleRate'])
else:
_ = np.fromfile(f, CONTINUOUS_TIMESTAMP_DTYPE, 1)
N = np.fromfile(f, CONTINUOUS_SAMPLE_PER_RECORD_DTYPE, 1)[0]
if N != oe.SAMPLES_PER_RECORD:
print(('samples per record specified in block ' + str(i) + ' (' + str(N) +
') does not equal to expected value (' + str(oe.SAMPLES_PER_RECORD) + ')!'))
samples = samples[0 : i * oe.SAMPLES_PER_RECORD]
print('samples per record specified in block ' + str(i) + ' (' + str(N) + \
') does not equal to expected value (' + str(oe.SAMPLES_PER_RECORD) + ')!')
break
_ = (np.fromfile(f, CONTINUOUS_RECORDING_NUMBER_DTYPE, 1))
if dtype == np.float32:
curr_samples = (np.fromfile(f, CONTINUOUS_SAMPLE_DTYPE, N) * float(header['bitVolts'])).astype(np.float32)
elif dtype == np.int16:
curr_samples = np.fromfile(f, CONTINUOUS_SAMPLE_DTYPE, N).astype(np.int16)
else:
raise ValueError('Error in reading data of block:' + str(i))
samples[i*oe.SAMPLES_PER_RECORD : (i+1)*oe.SAMPLES_PER_RECORD] = curr_samples
record_mark = np.fromfile(f, dtype=np.dtype('<u1'), count=10)
if not np.array_equal(record_mark, oe.RECORD_MARKER):
print(('record marker specified in block ' + str(i) + ' (' + str(record_mark) +
') does not equal to expected array (oe.RECORD_MARKER)!'))
break
header.update({'start_time': start_time})
print('continuous channel start time (for aligning digital events): ', start_time)
return header, samples
def loadEventsOE(path, tsStart):
raw = OpenEphys.load(path)
return Event(raw, tsStart)
def load_continuous_old(file_path, dtype=np.float32):
"""
Jun's wrapper to load .continuous data from OpenEphys data files
:param file_path:
:param dtype: np.float32 or np.int16
:return: header: dictionary, standard open ephys header for continuous file
samples: 1D np.array
"""
assert dtype in (np.float32, np.int16), \
'Invalid data type specified for loadContinous, valid types are np.float32 and np.int16'
print("\nLoading continuous data from " + file_path)
bytes_per_block = CONTINUOUS_TIMESTAMP_DTYPE.itemsize + CONTINUOUS_SAMPLE_PER_RECORD_DTYPE.itemsize + \
CONTINUOUS_RECORDING_NUMBER_DTYPE.itemsize + CONTINUOUS_MARKER_BYTES + \
CONTINUOUS_SAMPLE_DTYPE.itemsize * oe.SAMPLES_PER_RECORD
# read in the data
f = open(file_path, 'rb')
file_length = os.fstat(f.fileno()).st_size
print('total length of the file: ', file_length, 'bytes.')
print('bytes per record block: ', bytes_per_block)
block_num = (file_length - oe.NUM_HEADER_BYTES) // bytes_per_block
print('total number of valid blocks: ', block_num)
header = oe.readHeader(f)
samples = np.empty(oe.SAMPLES_PER_RECORD * block_num, dtype)
is_break = False
for i in range(block_num):
if i == 0:
# to get the timestamp of the very first record (block)
# for alignment of the digital event
start_ind = np.fromfile(f, CONTINUOUS_TIMESTAMP_DTYPE, 1)
start_time = float(start_ind) / float(header['sampleRate'])
else:
_ = np.fromfile(f, CONTINUOUS_TIMESTAMP_DTYPE, 1)
N = np.fromfile(f, CONTINUOUS_SAMPLE_PER_RECORD_DTYPE, 1)[0]
if N != oe.SAMPLES_PER_RECORD:
print(('samples per record specified in block ' + str(i) + ' (' + str(N) +
') does not equal to expected value (' + str(oe.SAMPLES_PER_RECORD) + ')!'))
samples = samples[0 : i * oe.SAMPLES_PER_RECORD]
is_break = True
break
# raise Exception('samples per record specified in block ' + str(i) + ' (' + str(N) + \
# ') does not equal to expected value (' + str(oe.SAMPLES_PER_RECORD) + ')!')
_ = (np.fromfile(f, CONTINUOUS_RECORDING_NUMBER_DTYPE, 1))
if dtype == np.float32:
curr_samples = (np.fromfile(f, CONTINUOUS_SAMPLE_DTYPE, N) * float(header['bitVolts'])).astype(np.float32)
elif dtype == np.int16:
curr_samples = np.fromfile(f, CONTINUOUS_SAMPLE_DTYPE, N).astype(np.int16)
else:
raise ValueError('Error in reading data of block:' + str(i))
samples[i*oe.SAMPLES_PER_RECORD : (i+1)*oe.SAMPLES_PER_RECORD] = curr_samples
record_mark = np.fromfile(f, dtype=np.dtype('<u1'), count=10)
if not np.array_equal(record_mark, oe.RECORD_MARKER):
print(('record marker specified in block ' + str(i) + ' (' + str(record_mark) +
') does not equal to expected array (' + str(oe.RECORD_MARKER) + ') !'))
is_break = True
break
if is_break:
samples = samples[0: oe.SAMPLES_PER_RECORD * (i -1)]
header.update({'start_time': start_time})
return header, samples
DataFiles = glob.glob(''.join([RecFolder,'/*.continuous']))
DataFiles.sort()
#%% Generate TTLs
os.makedirs('DataArrays', exist_ok=True)
DirList = glob.glob('OpenEphysFiles/*'); DirList.sort()
for RecFolder in DirList:
DataFiles = glob.glob(''.join([RecFolder,'/*.continuous']))
EventsFile = ''.join([RecFolder,'/all_channels.events'])
DataFiles.sort()
TempData = OpenEphys.loadContinuous(DataFiles[0]) # Any ch would work ;)
Events = OpenEphys.loadEvents(EventsFile)
# Draw TTL
print('Drawing TTL channel...')
if len(Events['timestamps']) > 100: # arbitrary value, I never use < 100 stimulation pulses
TTLSound = [0] * len(TempData['data'])
for EvTS in range(len(Events['timestamps'])):
if Events['eventType'][EvTS] == 3: # if event is a TTL
if Events['eventId'][EvTS] == 1: # if TTL is on
TTLSound(find(TempTimestamps==EventsTimestamps(EvTS)): ...
find(TempTimestamps==EventsTimestamps(EvTS+1))) = 1;
end
end
end
else
import numpy as np
import scipy.signal as signal
#%% Specifiy the input and output
#make provisions such that the script can be run independently or through snakemake
if 'snakemake' not in locals():
input = [
f'test/100_CH{x}.continuous' for x in range(1, setting.numChan + 1)
]
input.append(f'test/100_{setting.positionChannel}.continuous')
output = ['test/processing/eeg.npy', 'test/processing/position.npy']
else:
input = snakemake.input
output = snakemake.output
#%% Load and downsample to create data
data = []
for fn in input[:-1]:
d = OpenEphys.loadContinuousFast(fn)['data']
data.append(signal.decimate(d, setting.Fs // setting.eeg_Fs, ftype='fir'))
#%% Convert to numpy array and save
rawSignal = np.vstack(data).T
np.save(output[0], rawSignal)
#%% Load and save position data
d = OpenEphys.loadContinuousFast(input[-1])['data']
np.save(output[1], d)
duration = glia.milliseconds(400)
filename = i
jsonfile = rec_stim_matching[filename]
filename = "plx/" + filename
with open("json/" + jsonfile[:14] + "Settings.json",
encoding="utf-8") as json_fp:
stim_setting = json.load(json_fp)
if not reload and os.path.isfile("./waveform/" + filename[4:-4] +
".lfp") and os.path.isfile(
"./waveform/" + filename[4:-4] +
".trig"):
print("Load lfp, trig")
else:
wave = []
for j in channel_map:
tmp = OpenEphys.loadContinuous(filename + "/101_CH" + str(j) +
".continuous")
#gliaの形式にそってローパスフィルタをかけた波形の出力
wave.append(
glia.nucleus.Waveform(lowpass(tmp["data"], 30000, 450, 600, 10,
30),
rate=glia.Frequency(30000)))
timestamp = tmp["timestamps"][0]
events = OpenEphys.loadEvents(filename + "/all_channels.events")
e_tmp = (events["timestamps"] - timestamp)[0::2]
trigger = []
for i in (e_tmp / 30):
trigger.append(glia.millisecond(i))
trigger = np.array(trigger)
glia.save("./waveform/" + filename[4:] + ".lfp", wave)
glia.save("./waveform/" + filename[4:] + ".trig", trigger)
if not reload and os.path.isfile("./waveform/" + filename[4:-4] +
def loadConOE(path):
raw = OpenEphys.load(path)
return Con(raw)
#==============================================================================#
ntot = len(dfold_list)
ind = 0
for dfold in dfold_list:
ind = ind + 1
nrem = ntot - ind
print('Converting ' + dfold + ': ' + str(nrem) + ' remaining')
folderpath = ddir + dfold # folderpath where data is stored
#=============== SUBSAMPLE LOWPASS (LFP) SIGNAL AND CONVERT TO DAT ===========#
lfpname = '102_CH2.continuous'
filepath = ddir + dfold + '/' + lfpname
LFP = OpenEphys.loadContinuous(filepath, dtype=float)
LFP = LFP['data']
ssLFP = LFP[::10].copy() #subsampled LFP
# save as dat file
datnameLFP = dfold + '_LFP.dat'
ssLFP.tofile(os.path.join(folderpath, datnameLFP))
#==============================================================================#
#======================== CONVERT HIGHPASS SIGNALS TO DAT ======================#
# Load highpass signals of each sahnk in geometric order (bottom of shank to top)
# then save to flat binary file
# create shank1 and shank2 folders
os.makedirs(folderpath + '/shank1')
os.makedirs(folderpath + '/shank2')
# use channels ordered by intan headstage!!!
