def make(fo):
print('making artifacts, cnn bad epoch, xml files per block')
cnn_model_name = 'cnn_5X5_2048_1'
for i in range(1, 49):
p = e.Participant(i, fid2ort=fo)
p.add_all_sessions()
for s in p.sessions:
for b in s.blocks:
if not os.path.isfile(path.eeg100hz + windower.make_name(b) +
'.npy'):
continue
if b.start_marker_missing or b.end_marker_missing:
d = load_100hz_numpy_block(windower.make_name(b))
w = windower.Windower(b, nsamples=d.shape[1], sf=100)
w = windower.Windower(b, sf=100)
if not os.path.isfile(path.artifact_data_all_pp + w.name +
'_pred.npy'):
print(path.artifact_data_all_pp + w.name + '_pred.npy',
'no prediction file present')
continue
a = xml_cnn.xml_cnn(w)
a.make_bad_epoch()
a.bad_epochs2xml()
a.write_bad_epoch_xml()
print(w.name)
def count_words(save=False):
'''count words in the EEG experiment per block and exp type.
count all presented words (all), count all presented content words (cw-all)
count the subset of words that is useable (ok), count the cw subset (cw-ok)
save whether to save the dict (default false because it exists)
'''
word_counter = {}
bad_blocks = []
output = []
for i in range(1, 49):
p = e.Participant(i)
pp_id = p.pp_id
p.add_all_sessions()
print(p)
for s in p.sessions:
exp_type = s.exp_type
if exp_type not in word_counter.keys(): word_counter[exp_type] = 0
for b in s.blocks:
try:
b.extract_words(content_word=False)
except:
bad_blocks.append(b)
continue
if not hasattr(b, 'extracted_words'):
bad_blocks.append(b)
continue
add_wc(b, word_counter, exp_type)
add_wc(b, word_counter, b.name)
if save:
save_count_ds(word_counter)
save_count_dict(word_counter)
return word_counter, bad_blocks
def check_p(p):
'''ensures p is an participant object and correctly loaded.
p int (1-48) or participant object, if sessions are not loaded this function will do so
'''
if type(p) == int or not hasattr(p, 'so'):
if type(p) != int: p = p.pp_id
p = e.Participant(p)
p.add_all_sessions()
return p
def filename2block(f, fo=None):
'''Return block object that correspond to the bad_epoch.'''
pp_id = int(f.split('pp')[-1].split('_')[0])
exp_type = f.split('exp-')[-1].split('_')[0]
bid = f.split('bid-')[-1].split('.')[0]
p = e.Participant(pp_id, fid2ort=fo)
p.add_session(exp_type)
s = getattr(p, 's' + exp_type)
return getattr(s, 'b' + bid)
def make_n400_word2surprisal(overwrite= True,filename = ''): if filename == '': filename = path.data + 'n400word2surprisal' if overwrite: open(filename,'w') p = e.Participant(1) fo = p.fid2ort for i in range(1,49): p = e.Participant(i,fo) p.add_all_sessions() for s in p.sessions: for b in s.blocks: for i,w in enumerate(b.words): line = make_n400_name(b, i) if not hasattr(w,'ppl'): line += '\t'+'NA' + '\n' else: line += '\t'+str(w.ppl.logprob) + ',' + str(w.ppl.logprob_register) line += ',' + str(w.ppl.logprob_other1) + ',' + str(w.ppl.logprob_other2) + '\n' with open(filename,'a') as fout: fout.write(line)
def name2block(name, fo = None): '''Based on the name made by the windower object, create and return the block object.''' pp_id = name2pp_id(name) exp_type = name2exp_type(name) bid = name2bid(name) p = e.Participant(pp_id,fid2ort = fo) p.add_session(exp_type) s = getattr(p,'s'+exp_type) return getattr(s,'b'+str(bid))
def __init__(self, p=None, history_length=4):
'''Hold all words of all experiments with PMN information (phonological mismatch negativity)
p participant object with all sessions loaded p = e.Participant(1) p.add_all_sessions()
his... number of previous words added to the pmn word (to determine entropy)
'''
self.p = p
self.history_length = history_length
if not p:
self.p = experiment.Participant(1)
self.p.add_all_sessions()
self.make_data()
self.add_index()
def make_matrix(fo, add_pp_info = False,normalize_data = True,save_directory =None,make_data= True):
print('making artifact training np matrices per block, with default 1 sec and 99 perc overlap')
if save_directory == None: save_directory = path.artifact_training_data
print('save directory:',save_directory)
if make_data: fout = open('nrows_per_block_v2','w')
nrows = 0
for i in range(1,49):
p = e.Participant(i,fid2ort = fo)
p.add_all_sessions()
for s in p.sessions:
for b in s.blocks:
if os.path.isfile(path.artifact_training_data+ windower.make_name(b) +'.npy'):
# check whether windowed data is already present
continue
if not os.path.isfile(path.eeg100hz + windower.make_name(b) +'.npy'):
# check whether downsampled data is present to load
continue
print(windower.make_name(b))
d = load_100hz_numpy_block(windower.make_name(b))
if b.start_marker_missing or b.end_marker_missing:
w = windower.Windower(b,nsamples= d.shape[1], sf = 100,window_overlap_percentage = .99)
else:
w = windower.Windower(b,sf = 100,window_overlap_percentage = .99)
if w.fn_annotation == 0:
print('skipping:',w.name,'NO ANNOTATION')
continue # if there is no annotation file skip
print('processing:',w.name,w.fn_annotation)
w.make_info_matrix(add_pp_info = add_pp_info)
if make_data:
d = remove_channels(d)
d = windower.window_data(d,w.windows['sf100'],flatten=True,normalize= normalize_data)
# d = unit_norm(d)
# d = normalize_numpy_matrix(d)
rows = d.shape[0]
nrows += rows
fout.write(w.name + '\t' + str(rows) + '\n')
print (d.shape, w.info_matrix.shape[0])
assert d.shape[0] == w.info_matrix.shape[0]
np.save(save_directory+ w.name + '_data',d)
np.save(save_directory+ w.name + '_info',w.info_matrix)
if make_data:
fout.write('all_blocks\t'+str(nrows)+'\n')
fout.close()
def handle_all(force_make=False):
output = []
bad_blocks = []
for i in range(1, 49):
print(i)
p = e.Participant(i)
p.add_all_sessions()
for s in p.sessions:
for b in s.blocks:
try:
output.extend(handle_block(b, force_make, ppls=s.ppl))
print('-' * 100, '\n' * 9, b.make_name(), '\t\t\tsucces',
'\n' * 9, '-' * 100)
except:
bad_blocks.append(b)
print('!' * 100, '\n' * 9, b.make_name(), '\t\t\tfailed',
'\n' * 9, '!' * 100)
return output, bad_blocks
def make_blinks():
'''Make all blinks objects and store them as pickled blinks objects.'''
for i in range(1, 49):
print('-' * 50)
print('-' * 50)
print('make blinks participant:', i)
print('-' * 50)
print('-' * 50)
if not os.path.isfile(path.blinks + 'pp' + str(i) + '.done'):
p = experiment.Participant(i)
p.add_all_sessions()
for s in p.sessions:
print(s)
for b in s.blocks:
b.make_blinks(False)
fout = open(path.blinks + 'pp' + str(i) + '.done', 'w')
fout.close()
else:
print('skipping participent:', i, 'already done')
def make(fo):
print('making event info xml files per block')
nartifacts = 0
nclean = 0
fout = open('artifact_info.txt', 'w')
for i in range(1, 49):
p = e.Participant(i, fid2ort=fo)
p.add_all_sessions()
for s in p.sessions:
for b in s.blocks:
if not os.path.isfile(path.eeg100hz + windower.make_name(b) +
'.npy'):
continue
if b.start_marker_missing or b.end_marker_missing:
d = load_100hz_numpy_block(windower.make_name(b))
w = windower.Windower(b, nsamples=d.shape[1], sf=100)
w = windower.Windower(b, sf=100)
if not os.path.isfile(path.artifact_data_all_pp + w.name +
'_pred.npy'):
print(path.artifact_data_all_pp + w.name + '_pred.npy',
'no prediction file present')
continue
if b.exp_type == 'k': nepoch = 20
if b.exp_type == 'o': nepoch = 60
if b.exp_type == 'ifadv': nepoch = 80
ii_xml = xml_cnn.xml_cnn(w,
select_nartifact=nepoch,
select_nclean=nepoch,
cnn_model_name='cnn_5X5_2048_1')
ii_xml.make_index_info()
ii_xml.make_selection()
ii_xml.write()
nartifacts += ii_xml.nartifact_indices
nclean += ii_xml.nclean_indices
print(w.name + '\t' + ii_xml.nclean + '\t' + ii_xml.nartifact +
'\n')
fout.write(w.name + '\t' + ii_xml.nclean + '\t' +
ii_xml.nartifact + '\n')
fout.write('all_blocks\t' + str(nclean) + '\t' + str(nartifacts) + '\n')
fout.close()
import experiment as e
import video_log as vl
for i in range(1,49):
print('Loading participant:',i)
p = e.Participant(i)
p.add_all_sessions()
for s in p.sessions:
v = vl.video_log(s.log)
if v.ok:
for marker in v.marker2time.keys():
if marker % 10 == 0:
v.extract_frames(marker,5)
else: print('skipping:',v.pp_id,v.exp_type,'no marker file/or log file')
def make_matrix(fo,
add_pp_info=False,
normalize_data=True,
save_directory=None,
make_data=True,
start_pp=1,
overwrite=False):
print(
'making artifact training np matrices per block, with default 1 sec and 99 perc overlap'
)
if save_directory == None:
save_directory = path.channel_artifact_training_data
print('save directory:', save_directory)
if make_data: fout = open(path.bak + 'channel_nrows_per_block-p', 'w')
nrows = 0
for i in range(start_pp, 49):
p = e.Participant(i, fid2ort=fo)
p.add_all_sessions()
for s in p.sessions:
for b in s.blocks:
if os.path.isfile(path.channel_artifact_training_data +
windower.make_name(b) +
'_data.npy') and not overwrite:
# check whether windowed data is already present
continue
if not os.path.isfile(path.eeg100hz + windower.make_name(b) +
'.npy'):
# check whether downsampled data is present to load
continue
print(windower.make_name(b))
d = load_100hz_numpy_block(windower.make_name(b))
if b.start_marker_missing or b.end_marker_missing:
w = windower.Windower(b,
nsamples=d.shape[1] * 10,
sf=100,
window_overlap_percentage=.99)
else:
w = windower.Windower(b,
sf=100,
window_overlap_percentage=.99)
f = windower.block2channel_fn_annotation(
w.b, path.channel_artifacts_clean)
if f == 0:
print('skipping:', w.name, 'NO ANNOTATION')
continue # if there is no annotation file skip
print('processing:', w.name, w.fn_annotation)
w.make_channel_ca_info_matrix(add_pp_info=add_pp_info)
if make_data:
d = remove_channels(d)
d = windower.window_data(d,
w.windows['sf100'],
flatten=True,
normalize=normalize_data,
cut_off=300)
rows = d.shape[0]
nrows += rows
fout.write(w.name + '\t' + str(rows) + '\n')
print(d.shape, w.info_matrix.shape[0])
assert d.shape[0] == w.info_matrix.shape[0]
# d = insert_target_channel_rows(d,nchannels=26,kernel_size=6)
# IMPORTANT: this was not commented when I checked the file
# however resulting files look like it was commented during execution
# insert target channel is done later in pipeline
# not sure what happened
np.save(save_directory + w.name + '_data', d)
np.save(save_directory + w.name + '_info', w.info_matrix)
if make_data:
fout.write(path.bak + 'all_blocks\t' + str(nrows) + '\n')
fout.close()
def bad_epoch2block(be,fo = None): '''Return block object that correspond to the bad_epoch.''' p = e.Participant(be.pp_id,fid2ort = fo) p.add_session(be.exp_type) s = getattr(p,'s' + be.exp_type) return getattr(s, 'b' + str(be.bid))
import numpy as np
import os
import path
'''Downsample all data from 1000hz to 100hz sf
The code uses MNE function to downsample
Visually checked time domain and frequency domain both looked reasonable.
100hz -> nyquist at 50hz, which is ac frequency (spike in the spectrum),
however this is comfortably far from 30hz lowpass filter on the data
I do not know whether MNE uses zerophase filtering before donwsample should check.
'''
new_sf = 100
error = []
p = e.Participant(1)
fo = p.fid2ort
for i in range(1, 49):
print('loading participant:', i)
p = e.Participant(i, fid2ort=fo)
p.add_all_sessions()
for s in p.sessions:
print(s.exp_type)
for b in s.blocks:
print(b.bid)
name = path.eeg100hz + b.block2name()
print(name, name + '.npy')
if not os.path.isfile(name + '.npy') and not b.block_missing:
b.load_eeg_data(sf=new_sf)
if b.raw != 0:
print('saving data to:', name)
def _load_participant(self,number): '''Load a specific participant, participant numbers 1 - 48.''' self.current_p = e.Participant(number) self.current_p.add_all_sessions() self.pp.append(self.current_p)
