class Automator(object):
EXPERIMENTS = {'r1': ('FR1', 'FR2', 'FR3',
'catFR1', 'catFR2', 'catFR3',
'PAL1', 'PAL2', 'PAL3',
'TH1', 'TH2', 'TH3', 'THR',
'PS1', 'PS2', 'PS3')}
MATH_TASKS = ('FR1', 'FR2', 'FR3', 'catFR1', 'catFR2', 'catFR3', 'PAL1', 'PAL2', 'PAL3', 'ltpFR', 'ltpFR2')
INCLUDE_TRANSFERRED = False
def __init__(self, protocol):
self.protocol = protocol
self.index = JsonIndexReader(os.path.join(paths.db_root, 'protocols', '{}.json'.format(protocol)))
self.importers = []
def populate_importers(self):
self.add_existing_events_importers()
self.add_existing_montage_importers()
self.add_future_events_importers()
def add_existing_montage_importers(self):
subjects = self.index.subjects()
for subject in subjects:
montages = self.index.montages(subject=subject)
for montage in montages:
code = self.index.get_value('subject_alias', subject=subject, montage=montage.split('.')[1])
importer = Importer(Importer.CONVERT_MONTAGE,
subject=subject, montage=montage, protocol=self.protocol, code=code)
if importer.check() or importer.errored or self.INCLUDE_TRANSFERRED:
self.importers.append(importer)
def session_indexes(self):
experiments = self.index.experiments()
for experiment in experiments:
exp_index = self.index.filtered(experiment=experiment)
subjects = exp_index.subjects()
for subject in subjects:
subj_index = exp_index.filtered(subject=subject)
sessions = subj_index.sessions()
for session in sessions:
sess_index = subj_index.filtered(session=session)
yield subject, experiment, session, sess_index
def build_existing_event_importer_kwargs(self, subject, experiment, session, index, do_compare=True):
montage = index.montages()[0]
do_math = experiment in self.MATH_TASKS
code = index.get_value('subject_alias')
try:
original_session = int(index.get_value('original_session'))
except KeyError:
original_session = int(session)
try:
original_experiment = index.get_value('original_experiment')
except KeyError:
original_experiment = experiment
kwargs = dict(subject=subject, montage=montage, experiment=original_experiment, session=int(session),
new_experiment=experiment, original_session=original_session,
do_math=do_math, protocol=self.protocol, code=code, do_compare=do_compare)
return kwargs
def add_existing_events_importers(self):
for subject, experiment, session, index in self.session_indexes():
kwargs = self.build_existing_event_importer_kwargs(subject, experiment, session, index)
importer = Importer(Importer.BUILD_EVENTS, **kwargs)
if not importer.check() and not importer.errored:
pass
elif not importer.errored:
self.importers.append(importer)
else:
importer2 = Importer(Importer.CONVERT_EVENTS, **kwargs)
if not importer2.check() and not importer2.errored:
pass
else:
if not importer.errored:
self.importers.append(importer)
elif not importer2.errored:
self.importers.append(importer2)
else:
self.importers.append(importer)
def add_future_events_importers(self):
subjects = self.index.subjects()
for subject in subjects:
subj_index = self.index.filtered(subject=subject)
montages = subj_index.montages()
max_montage = max(montages)
for experiment in self.EXPERIMENTS:
exp_index = subj_index.filtered(montage=max_montage, experiment=experiment)
sessions = exp_index.sessions()
if sessions:
max_session = max(sessions)
try:
original_session = exp_index.get_value('original_session', session=max_session) + 1
except KeyError:
original_session = max_session+1
try:
original_experiment = exp_index.get_value('original_experiment', session=max_session)
except KeyError:
original_experiment = experiment
session = max_session + 1
else:
session = 0
original_session = 0
original_experiment = experiment
do_math = experiment in self.MATH_TASKS
code = subj_index.get_value('subject_alias', montage=max_montage.split('.')[1])
kwargs = dict(subject=subject, montage=max_montage, experiment=original_experiment, session=session,
new_experiment=experiment, original_session=original_session,
do_math=do_math, protocol=self.protocol, code=code, do_compare=False)
importer = Importer(Importer.BUILD_EVENTS, **kwargs)
if importer.check():
self.importers.append(importer)
def run_all_imports(self):
for importer in self.importers:
importer.run()
def sorted_importers(self):
order = 'initialized', 'errored', '_should_transfer', 'transferred', 'processed', 'subject'
return sorted(self.importers, key=lambda imp: [imp.__dict__[o] for o in order])
def describe(self):
descriptions = []
if not self.importers:
return 'No Importers'
for importer in self.sorted_importers():
descriptions.append(importer.describe())
return '\n---------------\n'.join(descriptions)
def __init__(self, protocol):
self.protocol = protocol
self.index = JsonIndexReader(os.path.join(paths.db_root, 'protocols', '{}.json'.format(protocol)))
self.importers = []
def test_as_dataframe(self):
index = osp.join(get_rhino_root(), "protocols", "r1.json")
reader = JsonIndexReader(index)
df = reader.as_dataframe()
assert isinstance(df, pd.DataFrame)
def get_subject_sessions_by_experiment(experiment, protocol='r1', include_montage_changes=False):
"""
:param experiment:
:param protocol:
:param include_montage_changes:
:return: subject, subject_code, session, original_session, experiment, version
"""
json_reader = JsonIndexReader(os.path.join(paths.rhino_root,'protocols','%s.json'%protocol))
if experiment in json_reader.experiments():
subjects = json_reader.subjects(experiment=experiment)
for subject_no_montage in subjects:
for montage in json_reader.montages(subject=subject_no_montage, experiment=experiment):
subject = subject_no_montage if montage == '0' else '%s_%s' % (subject_no_montage, montage)
sessions = json_reader.sessions(subject=subject_no_montage, montage=montage, experiment=experiment)
for session in sessions:
try:
original_session = json_reader.get_value('original_session',
subject=subject_no_montage,experiment=experiment,
session=session)
except ValueError:
original_session = session # not necessarily robust
yield subject_no_montage, subject,session, original_session, experiment, '0'
else:
if re.match('catFR[0-4]', experiment):
ram_exp = 'RAM_{}'.format(experiment[0].capitalize() + experiment[1:])
else:
ram_exp = 'RAM_{}'.format(experiment)
events_dir = os.path.join(paths.data_root,'events',ram_exp)
events_files = sorted(glob.glob(os.path.join(events_dir, '{}*_events.mat'.format(protocol.upper()))),
key=lambda f: f.split('_')[:-1])
seen_experiments = defaultdict(list)
for events_file in events_files:
subject = '_'.join(os.path.basename(events_file).split('_')[:-1])
subject_no_montage = subject.split('_')[0]
if '_' in subject:
if not include_montage_changes:
continue
mat_events_reader = BaseEventReader(filename=events_file, common_root=paths.data_root)
logger.debug('Loading matlab events {exp}: {subj}'.format(exp=experiment, subj=subject))
try:
mat_events = mat_events_reader.read()
sessions = np.unique(mat_events['session'])
version_str = mat_events[-5]['expVersion'] if 'expVersion' in mat_events.dtype.names else '0'
version = -1
try:
version = float(version_str.split('_')[-1])
except:
try:
version = float(version_str.split('v')[-1])
except:
pass
for i, session in enumerate(sessions):
if 'experiment' in mat_events.dtype.names:
experiments = np.unique(mat_events[mat_events['session'] == session]['experiment'])
else:
experiments = [experiment]
for this_experiment in experiments:
n_sessions = seen_experiments[subject_no_montage].count(this_experiment)
yield subject_no_montage, subject, n_sessions, session, this_experiment, version
seen_experiments[subject_no_montage].append(this_experiment)
except AttributeError:
traceback.print_exc()
logger.error('Could not get session from {}'.format(events_file))
class Automator(object):
EXPERIMENTS = {
'r1': ('FR1', 'FR2', 'FR3', 'catFR1', 'catFR2', 'catFR3', 'PAL1',
'PAL2', 'PAL3', 'TH1', 'TH2', 'TH3', 'THR', 'PS1', 'PS2', 'PS3')
}
MATH_TASKS = ('FR1', 'FR2', 'FR3', 'catFR1', 'catFR2', 'catFR3', 'PAL1',
'PAL2', 'PAL3', 'ltpFR', 'ltpFR2')
INCLUDE_TRANSFERRED = False
def __init__(self, protocol):
self.protocol = protocol
self.index = JsonIndexReader(
os.path.join(paths.db_root, 'protocols',
'{}.json'.format(protocol)))
self.importers = []
def populate_importers(self):
self.add_existing_events_importers()
self.add_existing_montage_importers()
self.add_future_events_importers()
def add_existing_montage_importers(self):
subjects = self.index.subjects()
for subject in subjects:
montages = self.index.montages(subject=subject)
for montage in montages:
code = self.index.get_value('subject_alias',
subject=subject,
montage=montage.split('.')[1])
importer = Importer(Importer.CONVERT_MONTAGE,
subject=subject,
montage=montage,
protocol=self.protocol,
code=code)
if importer.check(
) or importer.errored or self.INCLUDE_TRANSFERRED:
self.importers.append(importer)
def session_indexes(self):
experiments = self.index.experiments()
for experiment in experiments:
exp_index = self.index.filtered(experiment=experiment)
subjects = exp_index.subjects()
for subject in subjects:
subj_index = exp_index.filtered(subject=subject)
sessions = subj_index.sessions()
for session in sessions:
sess_index = subj_index.filtered(session=session)
yield subject, experiment, session, sess_index
def build_existing_event_importer_kwargs(self,
subject,
experiment,
session,
index,
do_compare=True):
montage = index.montages()[0]
do_math = experiment in self.MATH_TASKS
code = index.get_value('subject_alias')
try:
original_session = int(index.get_value('original_session'))
except KeyError:
original_session = int(session)
try:
original_experiment = index.get_value('original_experiment')
except KeyError:
original_experiment = experiment
kwargs = dict(subject=subject,
montage=montage,
experiment=original_experiment,
session=int(session),
new_experiment=experiment,
original_session=original_session,
do_math=do_math,
protocol=self.protocol,
code=code,
do_compare=do_compare)
return kwargs
def add_existing_events_importers(self):
for subject, experiment, session, index in self.session_indexes():
kwargs = self.build_existing_event_importer_kwargs(
subject, experiment, session, index)
importer = Importer(Importer.BUILD_EVENTS, **kwargs)
if not importer.check() and not importer.errored:
pass
elif not importer.errored:
self.importers.append(importer)
else:
importer2 = Importer(Importer.CONVERT_EVENTS, **kwargs)
if not importer2.check() and not importer2.errored:
pass
else:
if not importer.errored:
self.importers.append(importer)
elif not importer2.errored:
self.importers.append(importer2)
else:
self.importers.append(importer)
def add_future_events_importers(self):
subjects = self.index.subjects()
for subject in subjects:
subj_index = self.index.filtered(subject=subject)
montages = subj_index.montages()
max_montage = max(montages)
for experiment in self.EXPERIMENTS:
exp_index = subj_index.filtered(montage=max_montage,
experiment=experiment)
sessions = exp_index.sessions()
if sessions:
max_session = max(sessions)
try:
original_session = exp_index.get_value(
'original_session', session=max_session) + 1
except KeyError:
original_session = max_session + 1
try:
original_experiment = exp_index.get_value(
'original_experiment', session=max_session)
except KeyError:
original_experiment = experiment
session = max_session + 1
else:
session = 0
original_session = 0
original_experiment = experiment
do_math = experiment in self.MATH_TASKS
code = subj_index.get_value('subject_alias',
montage=max_montage.split('.')[1])
kwargs = dict(subject=subject,
montage=max_montage,
experiment=original_experiment,
session=session,
new_experiment=experiment,
original_session=original_session,
do_math=do_math,
protocol=self.protocol,
code=code,
do_compare=False)
importer = Importer(Importer.BUILD_EVENTS, **kwargs)
if importer.check():
self.importers.append(importer)
def run_all_imports(self):
for importer in self.importers:
importer.run()
def sorted_importers(self):
order = 'initialized', 'errored', '_should_transfer', 'transferred', 'processed', 'subject'
return sorted(self.importers,
key=lambda imp: [imp.__dict__[o] for o in order])
def describe(self):
descriptions = []
if not self.importers:
return 'No Importers'
for importer in self.sorted_importers():
descriptions.append(importer.describe())
return '\n---------------\n'.join(descriptions)
params = p_reader.read()
bo_recon_vec = []
epsilon = 1.0e-3
except:
print("not in lucy.owen")
t = time.time()
pow_vec_orig = []
pow_vec_recon = []
print(len(recon_files))
print(len(orig_files))
assert (len(recon_files) == len(events))
experiment = 'FR1'
jr = JsonIndexReader(rhino_root +
'/protocols/r1.json') # Build JSON reader
pairs_path = jr.get_value('pairs',
subject=subject,
experiment=experiment)
tal_reader = TalReader(filename=pairs_path)
monopolar_channels = tal_reader.get_monopolar_channels()
bipolar_channels = tal_reader.get_bipolar_pairs()
recon_files = np.sort(recon_files)
orig_files = np.sort(orig_files)
data_orig_vec = []
data_recon_vec = []
for i, (recon_file,
orig_file) in enumerate(zip(recon_files, orig_files)):
def get_subject_sessions_by_experiment(experiment,
protocol='r1',
include_montage_changes=False):
"""
:param experiment:
:param protocol:
:param include_montage_changes:
:return: subject, subject_code, session, original_session, experiment, version
"""
json_reader = JsonIndexReader(
os.path.join(paths.rhino_root, 'protocols', '%s.json' % protocol))
if experiment in json_reader.experiments():
subjects = json_reader.subjects(experiment=experiment)
for subject_no_montage in subjects:
for montage in json_reader.montages(subject=subject_no_montage,
experiment=experiment):
subject = subject_no_montage if montage == '0' else '%s_%s' % (
subject_no_montage, montage)
sessions = json_reader.sessions(subject=subject_no_montage,
montage=montage,
experiment=experiment)
for session in sessions:
try:
original_session = json_reader.get_value(
'original_session',
subject=subject_no_montage,
experiment=experiment,
session=session)
except ValueError:
original_session = session # not necessarily robust
yield subject_no_montage, subject, session, original_session, experiment, '0'
else:
if re.match('catFR[0-4]', experiment):
ram_exp = 'RAM_{}'.format(experiment[0].capitalize() +
experiment[1:])
else:
ram_exp = 'RAM_{}'.format(experiment)
events_dir = os.path.join(paths.data_root, 'events', ram_exp)
events_files = sorted(glob.glob(
os.path.join(events_dir,
'{}*_events.mat'.format(protocol.upper()))),
key=lambda f: f.split('_')[:-1])
seen_experiments = defaultdict(list)
for events_file in events_files:
subject = '_'.join(os.path.basename(events_file).split('_')[:-1])
subject_no_montage = subject.split('_')[0]
if '_' in subject:
if not include_montage_changes:
continue
mat_events_reader = BaseEventReader(filename=events_file,
common_root=paths.data_root)
logger.debug('Loading matlab events {exp}: {subj}'.format(
exp=experiment, subj=subject))
try:
mat_events = mat_events_reader.read()
sessions = np.unique(mat_events['session'])
version_str = mat_events[-5][
'expVersion'] if 'expVersion' in mat_events.dtype.names else '0'
version = -1
try:
version = float(version_str.split('_')[-1])
except:
try:
version = float(version_str.split('v')[-1])
except:
pass
for i, session in enumerate(sessions):
if 'experiment' in mat_events.dtype.names:
experiments = np.unique(mat_events[
mat_events['session'] == session]['experiment'])
else:
experiments = [experiment]
for this_experiment in experiments:
n_sessions = seen_experiments[
subject_no_montage].count(this_experiment)
yield subject_no_montage, subject, n_sessions, session, this_experiment, version
seen_experiments[subject_no_montage].append(
this_experiment)
except AttributeError:
traceback.print_exc()
logger.error(
'Could not get session from {}'.format(events_file))
os.symlink(osp.join(reference_index_root,'protocols','r1','subjects',subject,'localizations','0','neuroradiology'),
osp.join(test_index_root,'protocols','r1','subjects',subject,'localizations','0','neuroradiology'))
except OSError as ose:
if ose.errno != 17:
raise ose
return run_montage_import(inputs,force=True)
if __name__ == '__main__':
for subject in bipolar_subjects:
logger.set_subject(subject,'R1')
sucess, _ =make_bipolar_montage(subject)
if sucess:
tal = TalReader(filename=pairs_path_template.format(protocols_root=test_index_root,
subject= subject))
new_bipolar_pairs = tal.get_bipolar_pairs()
jr = JsonIndexReader(osp.join(reference_index_root,'protocols','r1.json'))
task_event_files = list(jr.aggregate_values('task_events',subject=subject,montage=0))
events = CMLEventReader(filename=task_event_files[0]).read()
eeg = EEGReader(events=events[:1],channels = np.array([]),start_time=0.0,end_time=0.1).read()
hdf_bipolar_pairs = eeg.bipolar_pairs.values
new_bipolar_pairs = new_bipolar_pairs.astype(hdf_bipolar_pairs.dtype)
if not np.in1d(hdf_bipolar_pairs,new_bipolar_pairs).all():
logger.info('\n\n%s missing from new_bipolar_pairs\n\n'%str(hdf_bipolar_pairs[~np.in1d(hdf_bipolar_pairs,new_bipolar_pairs)]))
if not np.in1d(new_bipolar_pairs,hdf_bipolar_pairs).all():
logger.info('\n\n%s missing from new_bipolar_pairs\n\n'%str(new_bipolar_pairs[~np.in1d(new_bipolar_pairs,hdf_bipolar_pairs)]))
beh_sess_root = os.path.join(paths.rhino_root, 'data', 'eeg', '{subject}',
'behavioral', '{experiment}', 'session_{session}')
session_log = os.path.join(beh_sess_root, 'session.log')
session_sql = os.path.join(beh_sess_root, 'session.sqlite')
ann_files = os.path.join(beh_sess_root, '*.ann')
fr_wordpool = os.path.join(os.path.dirname(beh_sess_root), 'RAM_wordpool.txt')
catfr_wordpool = os.path.join(os.path.dirname(beh_sess_root),
'CatFR_WORDS.txt')
reader = JsonIndexReader(os.path.join(paths.rhino_root, 'protocols',
'r1.json'))
fields_to_skip = ['eegfile', 'msoffset', 'eegoffset', 'stim_params', 'montage']
def make_files(wordpool, **kwargs):
if 'original_experiment' not in kwargs:
kwargs['original_experiment'] = ''
return {
'session_log':
session_log.format(**kwargs),
'session_sqlite':
session_sql.format(**kwargs),
'wordpool':
wordpool.format(**kwargs),
'matlab_events':