def compare_equal_events(subject, experiment, session, config, db_root='/'):
from event_creation.submission.parsers.base_log_parser import EventComparator
success, msg = run_session_import(subject, experiment, session, config,
db_root)
if not success:
return False, 'Import Failed for %s %s_%s : %s' % (subject, experiment,
session, msg)
new_jr = JsonIndexReader(os.path.join(db_root, 'protocols', 'r1.json'))
new_events = CMLEventReader(filename=new_jr.get_value(
'task_events', subject=subject, experiment=experiment,
session=session)).read()
old_jr = JsonIndexReader(
os.path.join(config.paths.rhino_root, 'protocols', 'r1.json'))
old_events = CMLEventReader(filename=old_jr.get_value(
'task_events', subject=subject, experiment=experiment,
session=session)).read()
flat_comparison = EventComparator(events1=old_events,
events2=new_events,
same_fields=False,
field_ignore=[
'stim_params', 'test', 'eegfile',
'msoffset', 'exp_version'
],
verbose=True)
failure, msg = flat_comparison.compare()
return not failure, msg
def read_session(subject, root, experiment, rhino_root):
jr = JsonIndexReader(rhino_root + '/protocols/r1.json') # Build JSON reader
pairs_path = jr.get_value('pairs', subject=subject, experiment=experiment)
tal_reader = TalReader(filename=pairs_path)
monopolar_channels = tal_reader.get_monopolar_channels()
bipolar_channels = tal_reader.get_bipolar_pairs()
jr = JsonIndexReader(rhino_root + '/protocols/r1.json') # Build JSON reader
pairs_path = jr.get_value('pairs', subject=subject, experiment=experiment)
tal_reader = TalReader(filename=pairs_path)
monopolar_channels = tal_reader.get_monopolar_channels()
p_reader = ParamsReader(dataroot=root)
params = p_reader.read()
samplerate = params['samplerate']
print samplerate
try:
raw_eeg = EEGReader(session_dataroot=root, channels = monopolar_channels)
raw_eeg = raw_eeg.read()
print raw_eeg.shape
except:
with h5py.File(root, 'r') as hfile:
raw_eeg = H5RawReader.H5RawReader.read_h5file(hfile, monopolar_channels,[0])[0]
print raw_eeg.shape
raw_eeg = TimeSeriesX.TimeSeriesX(raw_eeg,
dims=['channels', 'start_offsets', 'time'],
coords={
'channels': monopolar_channels,
'start_offsets': [0],
'time': np.arange(raw_eeg.shape[2])*1000/samplerate,
'offsets': ('time',np.arange(raw_eeg.shape[2])),
'samplerate': samplerate})
return raw_eeg
def load_index(protocol):
if protocol not in LOADED_INDEXES:
index_file = os.path.join(paths.db_root, 'protocols', '{}.json'.format(protocol))
if not os.path.exists(index_file):
with open(index_file, 'w') as f:
json.dump({}, f)
LOADED_INDEXES[protocol] = JsonIndexReader(index_file)
return LOADED_INDEXES[protocol]
def compare_contains(subject, experiment, session, config):
assert run_session_import(subject, experiment, session, config)
new_jr = JsonIndexReader(os.path.join(db_root, 'protocols', 'r1.json'))
new_events = CMLEventReader(filename=new_jr.get_value(
'all_events', subject=subject, experiment=experiment,
session=session)).read()
old_jr = JsonIndexReader(
os.path.join(config.paths.rhino_root, 'protocols', 'r1.json'))
old_events = CMLEventReader(filename=old_jr.get_value(
'all_events', subject=subject, experiment=experiment,
session=session)).read()
fields_match = all(
[n in new_events.dtype.names for n in old_events.dtype.names])
if fields_match:
return all([(old_events[f] == new_events[f]).all()
for f in old_events.dtype.names])
else:
return False
def _test_eeg_with_tal_struct(self, subject, experiment, session, struct_type='mono'):
from ptsa.data.readers import JsonIndexReader,TalReader,EEGReader,BaseEventReader
import os
tal_file = 'contacts' if struct_type=='mono' else 'pairs'
jr = JsonIndexReader(os.path.join(get_rhino_root(),'protocols','r1.json'))
events = BaseEventReader(filename=jr.get_value('task_events',subject=subject,experiment=experiment,session=session)).read()
tal_struct = TalReader(filename=jr.get_value(tal_file,subject=subject),struct_type=struct_type).read()
eeg_reader = EEGReader(events=events[:10],channels=tal_struct[:10],start_time=0.0,end_time=0.1)
eeg = eeg_reader.read()
print(eeg)
channel_name = eeg_reader.channel_name
for col in tal_struct.dtype.names:
if col != 'atlases': # Because I don't want to deal with NaNs at the moment
assert_array_equal(eeg[channel_name].data[col],tal_struct[:10][col])
def test_eeg_reader(self, subject, session):
jr = JsonIndexReader(
osp.join(get_rhino_root(), 'protocols', 'ltp.json'))
events = BaseEventReader(filename=jr.get_value(
'task_events', subject=subject, session=session)).read()
# hack to make the scalp events work with Rhino mounted
events['eegfile'] = get_rhino_root() + events[0]['eegfile']
channels = np.array([0, 1])
eeg = EEGReader(events=events[:10],
channels=channels,
start_time=0.0,
end_time=0.1).read()
assert (eeg['channels'].values['index'] == channels).all()
assert (eeg['channels'].values['label'] == np.array(
['A1', 'A2'], dtype=bytes)).all()
def __init__(self, protocol):
self.protocol = protocol
self.index = JsonIndexReader(os.path.join(paths.db_root, 'protocols', '{}.json'.format(protocol)))
self.importers = []
def get_subject_sessions_by_experiment(experiment, protocol='r1', include_montage_changes=False):
"""
:param experiment:
:param protocol:
:param include_montage_changes:
:return: subject, subject_code, session, original_session, experiment, version
"""
json_reader = JsonIndexReader(os.path.join(paths.rhino_root,'protocols','%s.json'%protocol))
if experiment in json_reader.experiments():
subjects = json_reader.subjects(experiment=experiment)
for subject_no_montage in subjects:
for montage in json_reader.montages(subject=subject_no_montage, experiment=experiment):
subject = subject_no_montage if montage == '0' else '%s_%s' % (subject_no_montage, montage)
sessions = json_reader.sessions(subject=subject_no_montage, montage=montage, experiment=experiment)
for session in sessions:
try:
original_session = json_reader.get_value('original_session',
subject=subject_no_montage,experiment=experiment,
session=session)
except ValueError:
original_session = session # not necessarily robust
yield subject_no_montage, subject,session, original_session, experiment, '0'
else:
if re.match('catFR[0-4]', experiment):
ram_exp = 'RAM_{}'.format(experiment[0].capitalize() + experiment[1:])
else:
ram_exp = 'RAM_{}'.format(experiment)
events_dir = os.path.join(paths.data_root,'events',ram_exp)
events_files = sorted(glob.glob(os.path.join(events_dir, '{}*_events.mat'.format(protocol.upper()))),
key=lambda f: f.split('_')[:-1])
seen_experiments = defaultdict(list)
for events_file in events_files:
subject = '_'.join(os.path.basename(events_file).split('_')[:-1])
subject_no_montage = subject.split('_')[0]
if '_' in subject:
if not include_montage_changes:
continue
mat_events_reader = BaseEventReader(filename=events_file, common_root=paths.data_root)
logger.debug('Loading matlab events {exp}: {subj}'.format(exp=experiment, subj=subject))
try:
mat_events = mat_events_reader.read()
sessions = np.unique(mat_events['session'])
version_str = mat_events[-5]['expVersion'] if 'expVersion' in mat_events.dtype.names else '0'
version = -1
try:
version = float(version_str.split('_')[-1])
except:
try:
version = float(version_str.split('v')[-1])
except:
pass
for i, session in enumerate(sessions):
if 'experiment' in mat_events.dtype.names:
experiments = np.unique(mat_events[mat_events['session'] == session]['experiment'])
else:
experiments = [experiment]
for this_experiment in experiments:
n_sessions = seen_experiments[subject_no_montage].count(this_experiment)
yield subject_no_montage, subject, n_sessions, session, this_experiment, version
seen_experiments[subject_no_montage].append(this_experiment)
except AttributeError:
traceback.print_exc()
logger.error('Could not get session from {}'.format(events_file))
if __name__ == "__main__":
from argparse import ArgumentParser
from collections import namedtuple
from ptsa.data.readers import JsonIndexReader
os.environ['RHINO_ROOT'] = "~/mnt/rhino"
Dataset = namedtuple("Dataset", "subject,experiment,session")
parser = ArgumentParser()
parser.add_argument("outfile", type=str, help="path to output HDF5 file")
args = parser.parse_args()
reader = JsonIndexReader('/Users/depalati/mnt/rhino/protocols/r1.json')
df = reader.as_dataframe().reset_index()
subjects = df[df.experiment.isin(['FR5', 'catFR5'])].subject.unique()
datasets = [
# Dataset('R1395M', "catFR5", 4),
]
for subject in subjects:
for session in range(4):
datasets.append(Dataset(subject, 'FR5', session))
datasets.append(Dataset(subject, 'catFR5', session))
with h5py.File(args.outfile, 'w') as hfile:
count = 0
'localizations', '0', 'neuroradiology'))
except OSError as ose:
if ose.errno != 17:
raise ose
return run_montage_import(inputs, force=True)
if __name__ == '__main__':
for subject in bipolar_subjects:
logger.set_subject(subject, 'R1')
sucess, _ = make_bipolar_montage(subject)
if sucess:
tal = TalReader(filename=pairs_path_template.format(
protocols_root=test_index_root, subject=subject))
new_bipolar_pairs = tal.get_bipolar_pairs()
jr = JsonIndexReader(
osp.join(reference_index_root, 'protocols', 'r1.json'))
task_event_files = list(
jr.aggregate_values('task_events', subject=subject, montage=0))
events = CMLEventReader(filename=task_event_files[0]).read()
eeg = EEGReader(events=events[:1],
channels=np.array([]),
start_time=0.0,
end_time=0.1).read()
hdf_bipolar_pairs = eeg.bipolar_pairs.values
new_bipolar_pairs = new_bipolar_pairs.astype(
hdf_bipolar_pairs.dtype)
if not np.in1d(hdf_bipolar_pairs, new_bipolar_pairs).all():
logger.info(
'\n\n%s missing from new_bipolar_pairs\n\n' %
str(hdf_bipolar_pairs[
~np.in1d(hdf_bipolar_pairs, new_bipolar_pairs)]))
def test_as_dataframe(self):
index = osp.join(get_rhino_root(), "protocols", "r1.json")
reader = JsonIndexReader(index)
df = reader.as_dataframe()
assert isinstance(df, pd.DataFrame)
params = p_reader.read()
bo_recon_vec = []
epsilon = 1.0e-3
except:
print("not in lucy.owen")
t = time.time()
pow_vec_orig = []
pow_vec_recon = []
print(len(recon_files))
print(len(orig_files))
assert (len(recon_files) == len(events))
experiment = 'FR1'
jr = JsonIndexReader(rhino_root +
'/protocols/r1.json') # Build JSON reader
pairs_path = jr.get_value('pairs',
subject=subject,
experiment=experiment)
tal_reader = TalReader(filename=pairs_path)
monopolar_channels = tal_reader.get_monopolar_channels()
bipolar_channels = tal_reader.get_bipolar_pairs()
recon_files = np.sort(recon_files)
orig_files = np.sort(orig_files)
data_orig_vec = []
data_recon_vec = []
for i, (recon_file,
orig_file) in enumerate(zip(recon_files, orig_files)):
beh_sess_root = os.path.join(paths.rhino_root, 'data', 'eeg', '{subject}',
'behavioral', '{experiment}', 'session_{session}')
session_log = os.path.join(beh_sess_root, 'session.log')
session_sql = os.path.join(beh_sess_root, 'session.sqlite')
ann_files = os.path.join(beh_sess_root, '*.ann')
fr_wordpool = os.path.join(os.path.dirname(beh_sess_root), 'RAM_wordpool.txt')
catfr_wordpool = os.path.join(os.path.dirname(beh_sess_root),
'CatFR_WORDS.txt')
reader = JsonIndexReader(os.path.join(paths.rhino_root, 'protocols',
'r1.json'))
fields_to_skip = ['eegfile', 'msoffset', 'eegoffset', 'stim_params', 'montage']
def make_files(wordpool, **kwargs):
if 'original_experiment' not in kwargs:
kwargs['original_experiment'] = ''
return {
'session_log':
session_log.format(**kwargs),
'session_sqlite':
session_sql.format(**kwargs),
'wordpool':
wordpool.format(**kwargs),
'matlab_events':