def test_eeg_read_keep_buffer(self):
# OLD READERS
events = self.read_ptsa_events()
# in case fancy indexing looses Eventness of events we need to create Events object explicitely
if not isinstance(events, Events):
events = Events(events)
eegs = events.get_data(channels=['002', '003'],
start_time=0.0,
end_time=1.6,
buffer_time=1.0,
eoffset='eegoffset',
keep_buffer=True,
eoffset_in_time=False,
verbose=True)
# NEW READERS
base_events = self.read_base_events()
from ptsa.data.experimental.TimeSeriesEEGReader import TimeSeriesEEGReader
time_series_reader = TimeSeriesEEGReader(events=base_events,
start_time=0.0,
end_time=1.6,
buffer_time=1.0,
keep_buffer=True)
time_series_reader.read(channels=['002', '003'])
base_eegs = time_series_reader.get_output()
# testing
assert_array_equal(eegs, base_eegs.data)
def get_word_events(sub, task, montage=0):
from ptsa.data.readers import BaseEventReader
from ptsa.data.events import Events
from ptsa.data.readers.IndexReader import JsonIndexReader
reader = JsonIndexReader('/protocols/r1.json')
#Get events
evfiles = list(
reader.aggregate_values('task_events',
subject=sub,
experiment=task,
montage=montage))
evs_on = np.array([])
for ef in evfiles:
try:
base_e_reader = BaseEventReader(filename=ef,
eliminate_events_with_no_eeg=True)
base_events = base_e_reader.read()
if len(evs_on) == 0:
evs_on = base_events[base_events.type == 'WORD']
else:
evs_on = np.concatenate(
(evs_on, base_events[base_events.type == 'WORD']), axis=0)
except:
continue
evs = Events(evs_on)
return evs
def test_ptsa_event_ordering(self):
# --------------------OROG PTSA - one raw bin wrapper per event
e_reader = PTSAEventReader(filename=self.e_path,
eliminate_events_with_no_eeg=True,
use_groupped_rawbinwrapper=False)
events = e_reader.read()
events = events[events.type == 'WORD']
events = events[self.event_range]
if not isinstance(events, Events):
events = Events(events)
eegs = events.get_data(channels=['002', '003'],
start_time=0.0,
end_time=1.6,
buffer_time=1.0,
eoffset='eegoffset',
keep_buffer=True,
eoffset_in_time=False,
verbose=True)
eegs = eegs[:, :, :-1]
words_ptsa = eegs['events']['item']
print('words_ptsa=', words_ptsa)
words = self.base_eegs['events'].data['item']
print(words)
assert_array_equal(np.array(words_ptsa), words)
assert_array_equal(eegs, self.base_eegs)
def get_stim_events(sub, task, montage=0):
from ptsa.data.readers import BaseEventReader
from ptsa.data.events import Events
from ptsa.data.readers.IndexReader import JsonIndexReader
reader = JsonIndexReader('/protocols/r1.json')
#Get events
evfiles = list(
reader.aggregate_values(
'task_events', subject=sub, experiment=task,
montage=montage)) #This is supposed to work but often does not
if len(evfiles) < 1:
from glob import glob
sessions = [
s.split('/')[-1]
for s in glob('/protocols/r1/subjects/' + sub + '/experiments/' +
task + '/sessions/*')
]
evfiles = []
for sess in sessions:
evfiles.append('/protocols/r1/subjects/' + sub + '/experiments/' +
task + '/sessions/' + sess +
'/behavioral/current_processed/task_events.json')
evs_on = np.array([])
for ef in evfiles:
try:
base_e_reader = BaseEventReader(filename=ef,
eliminate_events_with_no_eeg=True)
base_events = base_e_reader.read()
if len(evs_on) == 0:
evs_on = base_events[base_events.type == 'STIM_ON']
else:
evs_on = np.concatenate(
(evs_on, base_events[base_events.type == 'STIM_ON']),
axis=0)
except:
continue
evs_on = Events(evs_on)
#Reorganize sessions around stim sites
sessions = reader.sessions(subject=sub, experiment=task, montage=montage)
sess_tags = []
for i in range(len(evs_on)):
stimtag = str(evs_on[i]['stim_params']['anode_number']) + '-' + str(
evs_on[i]['stim_params']['cathode_number'])
sess_tags.append(stimtag)
sess_tags = np.array(sess_tags)
if len(np.unique(sess_tags)) <= len(sessions):
session_array = evs_on['session']
else:
session_array = np.empty(len(evs_on))
for idx, s in enumerate(np.unique(sess_tags)):
session_array[sess_tags == s] = int(idx)
sessions = np.unique(session_array)
return evs_on, session_array
def setUp(self):
self.event_range = range(0, 30, 1)
self.e_path = '/Users/m/data/events/RAM_FR1/R1060M_events.mat'
self.tal_path = '/Users/m/data/eeg/R1060M/tal/R1060M_talLocs_database_bipol.mat'
# --------------- TAL STRUCTS READ
tal_reader = TalReader(filename=self.tal_path)
self.monopolar_channels = tal_reader.get_monopolar_channels()
# ---------------- ORIG PTSA -------------------
e_reader = PTSAEventReader(filename=self.e_path, eliminate_events_with_no_eeg=True)
events = e_reader.read()
events = events[events.type == 'WORD']
events = events[self.event_range]
ev_order = np.argsort(events, order=('session','list','mstime'))
self.events = events[ev_order]
# self.events = self.read_ptsa_events()
# in case fancy indexing looses Eventness of events we need to create Events object explicitely
if not isinstance(self.events, Events):
self.events = Events(self.events)
start_time = 0.0
end_time = 1.6
buffer_time = 1.0
self.eegs = self.events.get_data(channels=['002', '003'], start_time=start_time, end_time=end_time,
buffer_time=buffer_time, eoffset='eegoffset', keep_buffer=True,
eoffset_in_time=False, verbose=True)
# ---------------- NEW STYLE PTSA -------------------
base_e_reader = BaseEventReader(filename=self.e_path, eliminate_events_with_no_eeg=True)
base_events = base_e_reader.read()
base_events = base_events[base_events.type == 'WORD']
base_ev_order = np.argsort(base_events, order=('session','list','mstime'))
base_events = base_events[base_ev_order]
self.base_events = base_events[self.event_range]
# self.base_events = self.read_base_events()
eeg_reader = EEGReader(events=self.base_events, channels=np.array(['002', '003']),
start_time=start_time, end_time=end_time, buffer_time=buffer_time)
self.base_eegs = eeg_reader.read()
def test_full_session_read(self):
# ---------------- ORIG PTSA -------------------
e_reader = PTSAEventReader(filename=self.e_path,
eliminate_events_with_no_eeg=True)
events = e_reader.read()
events = events[events.type == 'WORD']
events = events[self.event_range]
ev_order = np.argsort(events, order=('session', 'list', 'mstime'))
self.events = events[ev_order]
# self.events = self.read_ptsa_events()
# in case fancy indexing looses Eventness of events we need to create Events object explicitely
if not isinstance(self.events, Events):
self.events = Events(self.events)
eegs = self.events.get_data(channels=self.monopolar_channels,
start_time=0.0,
end_time=1.6,
buffer_time=1.0,
eoffset='eegoffset',
keep_buffer=True,
eoffset_in_time=False,
verbose=True)
# removing last entry to match dimensions - ptsa adds one extra element at the end of time axis
eegs = eegs[:, :, :-1]
eeg_reader = EEGReader(events=self.base_events,
channels=self.monopolar_channels,
start_time=0.0,
end_time=1.6,
buffer_time=1.0)
base_eegs = eeg_reader.read()
assert_array_equal(x=eegs, y=base_eegs)
# checking if axes match
assert_array_equal(np.array(eegs['channels']['name']),
base_eegs['channels'].data)
assert_array_almost_equal(np.array(eegs['time']),
base_eegs['time'].data,
decimal=3)
assert_array_equal(np.array(eegs['events']['item']),
base_eegs['events'].data['item'])
print()
def test_eeg_read_keep_buffer_butterworth_filtered(self):
# old readers
events = self.read_ptsa_events()
# in case fancy indexing looses Eventness of events we need to create Events object explicitely
if not isinstance(events, Events):
events = Events(events)
base_events = self.read_base_events()
eegs = events.get_data(channels=['002', '003'],
start_time=0.0,
end_time=1.6,
buffer_time=1.0,
eoffset='eegoffset',
keep_buffer=True,
eoffset_in_time=False,
verbose=True)
# filtering
eegs_filtered = eegs.filtered([58, 62], filt_type='stop', order=4)
# New style reading
from ptsa.data.experimental.TimeSeriesEEGReader import TimeSeriesEEGReader
time_series_reader = TimeSeriesEEGReader(events=base_events,
start_time=0.0,
end_time=1.6,
buffer_time=1.0,
keep_buffer=True)
time_series_reader.read(channels=['002', '003'])
base_eegs = time_series_reader.get_output()
from ptsa.data.filters.ButterworthFilter import ButterworthFilter
b_filter = ButterworthFilter(time_series=base_eegs,
samplerate=base_eegs.attrs['samplerate'],
freq_range=[58, 62],
filt_type='stop',
order=4)
base_eegs_filtered = b_filter.filter()
# testing
assert_array_equal(eegs_filtered, base_eegs_filtered.data)
assert_array_equal(eegs, base_eegs.data)
def test_eeg_read_incomplete_data(self):
e_path_incomplete = '/Volumes/rhino_root/data/events/RAM_PS/R1104D_events.mat'
if sys.platform.startswith('win'):
e_path_incomplete = 'D:/data/events/RAM_PS/R1104D_events.mat'
base_event_reader = BaseEventReader(filename=e_path_incomplete)
base_events = base_event_reader.read()
sess_1 = base_events[base_events.session == 1]
sess_3 = base_events[base_events.session == 3]
sess_5 = base_events[base_events.session == 5]
sess_7 = base_events[base_events.session == 7]
sess_1[440].eegoffset = 2000000000000
sess_1[444].eegoffset = 2000000000000
sess_1[434].eegoffset = 2000000000000
shuffled_sess_events = np.hstack((sess_3, sess_7, sess_1, sess_5)).view(np.recarray)
eeg_reader = EEGReader(events=shuffled_sess_events, channels=np.array(['002', '003']),
start_time=self.start_time, end_time=self.end_time, buffer_time=self.buffer_time)
base_eegs = eeg_reader.read()
if eeg_reader.removed_bad_data():
print('REMOVED BAD DATA !!!!!!!!!!!!!')
events = base_eegs['events'].data.view(np.recarray)
if not isinstance(events, Events):
events = Events(events)
from ptsa.data.rawbinwrapper import RawBinWrapper
events = events.add_fields(esrc=np.dtype(RawBinWrapper))
# ------------- using PTSAevent reader functions to prepare events for reading old-ptsa-style
ptsa_event_reader = PTSAEventReader()
ptsa_event_reader.attach_rawbinwrapper_groupped(events)
eegs = events.get_data(channels=['002', '003'], start_time=self.start_time, end_time=self.end_time,
buffer_time=self.buffer_time, eoffset='eegoffset', keep_buffer=True,
eoffset_in_time=False, verbose=True)
npt.assert_array_equal(eegs[:, :, :-1], base_eegs.data)
def test_eeg_resample(self):
# old readers
events = self.read_ptsa_events()
# in case fancy indexing looses Eventness of events we need to create Events object explicitely
if not isinstance(events, Events):
events = Events(events)
base_events = self.read_base_events()
eegs = events.get_data(channels=['002', '003'],
start_time=0.0,
end_time=1.6,
buffer_time=1.0,
eoffset='eegoffset',
keep_buffer=True,
eoffset_in_time=False,
verbose=True)
# filtering
eegs_resampled = eegs.resampled(50.0)
# New style reading
from ptsa.data.experimental.TimeSeriesEEGReader import TimeSeriesEEGReader
time_series_reader = TimeSeriesEEGReader(events=base_events,
start_time=0.0,
end_time=1.6,
buffer_time=1.0,
keep_buffer=True)
time_series_reader.read(channels=['002', '003'])
base_eegs = time_series_reader.get_output()
from ptsa.data.filters.ResampleFilter import ResampleFilter
resample_filter = ResampleFilter(time_series=base_eegs,
resamplerate=50.0)
base_eegs_resampled = resample_filter.filter()
# testing
assert_array_equal(eegs_resampled, base_eegs_resampled.data)
def get_ps4_events(fn):
from ptsa.data.readers import BaseEventReader
from ptsa.data.events import Events
#Get events
evfiles = [fn]
evs_on = np.array([])
for ef in evfiles:
try:
base_e_reader = BaseEventReader(filename=ef,
eliminate_events_with_no_eeg=True)
base_events = base_e_reader.read()
if len(evs_on) == 0:
evs_on = base_events[base_events.type == 'STIM_ON']
else:
evs_on = np.concatenate(
(evs_on, base_events[base_events.type == 'STIM_ON']),
axis=0)
except:
continue
evs_on = Events(evs_on)
#Reorganize sessions around stim sites
sessions = np.unique(evs_on['session'])
sess_tags = []
for i in range(len(evs_on)):
stimtag = str(evs_on[i]['anode_num']) + '-' + str(
evs_on[i]['cathode_num'])
sess_tags.append(stimtag)
sess_tags = np.array(sess_tags)
if len(np.unique(sess_tags)) <= len(sessions):
session_array = evs_on['session']
else:
session_array = np.empty(len(evs_on))
for idx, s in enumerate(np.unique(sess_tags)):
session_array[sess_tags == s] = int(idx)
sessions = np.unique(session_array)
return evs_on, session_array
def read(self):
'''
Reads Matlab event file , converts it to np.recarray and attaches rawbinwrappers (if appropriate flags indicate so)
:return: Events object. depending on flagg settings the rawbinwrappers may be attached as well
'''
# calling base class read fcn
evs = BaseEventReader.read(self)
# in case evs is simply recarray
if not isinstance(evs, Events):
evs = Events(evs)
if self.attach_rawbinwrapper:
evs = evs.add_fields(esrc=np.dtype(RawBinWrapper))
if self.use_groupped_rawbinwrapper: # this should be default choice - much faster execution
self.attach_rawbinwrapper_groupped(evs)
else: # used for debugging purposes
self.attach_rawbinwrapper_individual(evs)
return evs
def read(self):
from MatlabIO import read_single_matlab_matrix_as_numpy_structured_array
# extract matlab matrix (called 'events') as numpy structured array
struct_array = read_single_matlab_matrix_as_numpy_structured_array(
self.__event_file, 'events')
evs = Events(struct_array)
if self.eliminate_events_with_no_eeg:
# eliminating events that have no eeg file
indicator = np.empty(len(evs), dtype=bool)
indicator[:] = False
for i, ev in enumerate(evs):
indicator[i] = (type(
evs[i].eegfile).__name__.startswith('unicode')) & (len(
str(evs[i].eegfile)) > 3)
evs = evs[indicator]
self.__events = evs
return self.__events
def test_eeg_read(self):
ptsa_reader = PTSAEventReader(filename=self.e_path)
events = ptsa_reader.read()
# in case fancy indexing looses Eventness of events we need to create Events object explicitely
if not isinstance(events, Events):
events = Events(events)
eegs = events.get_data(channels=['002', '003'], start_time=self.start_time, end_time=self.end_time,
buffer_time=self.buffer_time, eoffset='eegoffset', keep_buffer=True,
eoffset_in_time=False, verbose=True)
base_event_reader = BaseEventReader(filename=self.e_path)
base_events = base_event_reader.read()
eeg_reader = EEGReader(events=base_events, channels=np.array(['002', '003']),
start_time=self.start_time, end_time=self.end_time, buffer_time=self.buffer_time)
base_eegs = eeg_reader.read()
npt.assert_array_equal(eegs[:, :, :-1], base_eegs.data)
def run(self):
# PTSA EVENT READER
# e_path = join(self.pipeline.mount_point, 'data/events', self.pipeline.task,self.pipeline.subject+'_events.mat')
e_path = '/Users/m/data/events/RAM_FR1/R1060M_events.mat'
# e_path = '/Users/m/data/events/RAM_FR1/R1056M_events.mat'
e_reader = PTSAEventReader(event_file=e_path,
eliminate_events_with_no_eeg=True)
e_reader.read()
events = e_reader.get_output()
events = events[events.type == 'WORD']
events = events[0:30]
ev_order = np.argsort(events, order=('session', 'list', 'mstime'))
events = events[ev_order]
events = Events(events) # necessary workaround for new numpy
print 'events=', events
eegs = events.get_data(channels=['002', '003'],
start_time=0.0,
end_time=1.6,
buffer_time=1.0,
eoffset='eegoffset',
keep_buffer=False,
eoffset_in_time=False,
verbose=True)
print eegs
# BASE READER
base_e_reader = BaseEventReader(event_file=e_path,
eliminate_events_with_no_eeg=True,
use_ptsa_events_class=False)
base_e_reader.read()
base_events = base_e_reader.get_output()
base_events = base_events[base_events.type == 'WORD']
base_ev_order = np.argsort(base_events,
order=('session', 'list', 'mstime'))
base_events = base_events[base_ev_order]
base_events = base_events[0:30]
print 'base_events=', base_events
from ptsa.data.readers.TimeSeriesEEGReader import TimeSeriesEEGReader
time_series_reader = TimeSeriesEEGReader(base_events)
time_series_reader.start_time = 0.0
time_series_reader.end_time = 1.6
time_series_reader.buffer_time = 1.0
time_series_reader.keep_buffer = False
time_series_reader.read(channels=['002', '003'])
base_eegs = time_series_reader.get_output()
print
e_path = '/Users/m/data/events/RAM_FR1/R1060M_events.mat'
base_e_reader = BaseEventReader(event_file=e_path,
eliminate_events_with_no_eeg=True,
use_ptsa_events_class=False)
base_events = base_e_reader.read()
base_events = base_events[base_events.type == 'WORD']
base_e_reader_ptsa = BaseEventReader(event_file=e_path,
eliminate_events_with_no_eeg=True,
use_ptsa_events_class=False)
base_events_ptsa = base_e_reader_ptsa.read()
base_events_ptsa = base_events_ptsa[base_events_ptsa.type == 'WORD']
base_events_ptsa = Events(base_events_ptsa)
base_events_ptsa = attach_rawbinwrappers(base_events_ptsa)
print base_events_ptsa
base_ev_data_ptsa, base_ev_data_xray = base_events_ptsa.get_data(
channels=channels,
start_time=0.0,
end_time=1.6,
buffer_time=1.0,
eoffset='eegoffset',
keep_buffer=True,
eoffset_in_time=False,
verbose=True,
return_both=True)
newevs_stimParams = np.rec.recarray(len(matevs), dtype=dtypes_stimParams)
# sys.exit()
for field in matevs.dtype.names:
try:
newevs[field] = matevs[field]
except ValueError:
if ((field == 'stimAmp')
and (np.all(np.array(matevs[field], np.str) == 'X'))):
matevs[field].fill(np.nan)
newevs[field] = matevs[field]
else: # not sure what's going on, so raise the same error as above
newevs[field] = matevs[field]
events = Events(newevs)
events = events.add_fields(esrc=np.dtype(RawBinWrapper))
good_indices = np.ones(len(events), np.bool)
for e, event in enumerate(events):
try:
event['esrc'] = RawBinWrapper(event['eegfile'])
except IOError:
print('No EEG files for', event['subject'], event['session'],
event['eegfile'])
good_indices[e] = False
events = events[good_indices]
start_time = -0.6
end_time = 1.6
def get_sub_events(task, sub, eliminate_events_with_no_eeg=True, rhino_prefix='/', ignore_no_params=False):
"""Returns all events for a given task and subject identifier
Keyword arguments:
task -- string of task name
sub -- string of subject identifier
eliminate_events_with_no_eeg -- (default: True) Do not return events for which no corresponding eeg file exists.
rhino_prefix -- (default: '/') To get data if rhino is mounted as an external volume from a local computer.
ignore_no_params -- (default: False) Automatically handles IOError when no params.txt file is found for a subject.
"""
from MatlabIO import read_single_matlab_matrix_as_numpy_structured_array
e_path = rhino_prefix+'/data/events/'+task+'/'+sub+'_events.mat'
# extract matlab matrix (called 'events') as numpy structured array
struct_array = read_single_matlab_matrix_as_numpy_structured_array(e_path, 'events')
evs = Events(struct_array)
if eliminate_events_with_no_eeg:
# eliminating events that have no eeg file
indicator = np.empty(len(evs), dtype=bool)
indicator[:] = False
for i, ev in enumerate(evs):
indicator[i] = (type(evs[i].eegfile).__name__.startswith('unicode')) & (len(str(evs[i].eegfile)) > 3)
#Due to numpy upgrade
evs = Events(evs[indicator])
evs = evs.add_fields(esrc=np.dtype(RawBinWrapper))
import pathlib
#This may throw and error for a few subjects who have no params file. Should be written to handle this elegantly.
if ignore_no_params:
try:
for ev in evs:
try:
eeg_file_path = join(rhino_prefix, str(pathlib.Path(str(ev.eegfile)).parts[1:]))
ev.esrc = RawBinWrapper(eeg_file_path)
#self.raw_data_root=str(eeg_file_path)
except TypeError:
print 'skipping event with eegfile=',ev.eegfile
pass
except IOError:
print('No params.txt or .params file found for '+sub)
else:
for ev in evs:
try:
eeg_file_path = join(rhino_prefix, str(pathlib.Path(str(ev.eegfile)).parts[1:]))
ev.esrc = RawBinWrapper(eeg_file_path)
#self.raw_data_root=str(eeg_file_path)
except TypeError:
print 'skipping event with eegfile=',ev.eegfile
pass
return(evs)
},
'catFR2':{
'output_path': rhino_mount+'/data10/scratch/cweidema/RAM/RAM_catFR/RAM_catFR2_power/hdf5_files_sess',
'ev_file': rhino_mount+'/home1/cweidema/Christoph/Analyses/RAM/RAM_catFR/data/RAM_catFR2_events20151106.npy'
},
'FR1':{
'output_path': rhino_mount+'/data10/scratch/cweidema/RAM/RAM_FR/RAM_FR1_power/hdf5_files_sess',
'ev_file': rhino_mount+'/home1/cweidema/Christoph/Analyses/RAM/RAM_FR/data/RAM_FR1_events20151106.npy'
},
'FR2':{
'output_path': rhino_mount+'/data10/scratch/cweidema/RAM/RAM_FR/RAM_FR2_power/hdf5_files_sess',
'ev_file': rhino_mount+'/home1/cweidema/Christoph/Analyses/RAM/RAM_FR/data/RAM_FR2_events20151106.npy'
}
}
ev = Events(np.load(pow_params[exp]['ev_file']))
ev = ev.add_fields(esrc=np.dtype(RawBinWrapper))
ev = ev[ev['type']=='WORD']
good_indices = np.ones(len(ev),np.bool)
for e,event in enumerate(ev):
try:
event['esrc'] = RawBinWrapper(rhino_mount+event['eegfile'])
except IOError:
print('No EEG files for',event['subject'],event['session'],event['eegfile'])
good_indices[e] = False
ev = ev[good_indices]
start_time = -0.6
end_time = 1.6
buf = 1
