def test_tsv_io():
"""Test tsv I/O"""
names = ['A', 'B', 'rm', 'intvar', 'fltvar', 'fltvar2', 'index']
# get Dataset
ds = datasets.get_uv()
ds['fltvar'][5:10] = np.nan
ds[:4, 'rm'] = ''
# save and load
tempdir = tempfile.mkdtemp()
try:
dst = os.path.join(tempdir, 'ds.txt')
ds.save_txt(dst)
ds1 = load.tsv(dst)
ds2 = load.tsv(dst, skiprows=1, names=names)
assert_dataset_equal(ds1, ds, "TSV write/read test failed", 10)
assert_dataset_equal(ds2, ds, "TSV write/read test failed", 10)
# guess data types with missing
intvar2 = ds['intvar'].as_factor()
intvar2[10:] = ''
ds_intvar = Dataset((intvar2, ))
ds_intvar.save_txt(dst)
ds_intvar1 = load.tsv(dst, empty='nan')
assert_dataobj_equal(ds_intvar1['intvar', :10], ds['intvar', :10])
assert_array_equal(ds_intvar1['intvar', 10:], np.nan)
finally:
shutil.rmtree(tempdir)
def test_tsv_io():
"""Test tsv I/O"""
names = ['A', 'B', 'rm', 'intvar', 'fltvar', 'fltvar2', 'index']
# get Dataset
ds = datasets.get_uv()
ds['fltvar'][5:10] = np.nan
ds[:4, 'rm'] = ''
# save and load
tempdir = tempfile.mkdtemp()
try:
dst = os.path.join(tempdir, 'ds.txt')
ds.save_txt(dst)
ds1 = load.tsv(dst)
ds2 = load.tsv(dst, skiprows=1, names=names)
assert_dataset_equal(ds1, ds, "TSV write/read test failed", 10)
assert_dataset_equal(ds2, ds, "TSV write/read test failed", 10)
# guess data types with missing
intvar2 = ds['intvar'].as_factor()
intvar2[10:] = ''
ds_intvar = Dataset((intvar2,))
ds_intvar.save_txt(dst)
ds_intvar1 = load.tsv(dst, empty='nan')
assert_dataobj_equal(ds_intvar1['intvar', :10], ds['intvar', :10])
assert_array_equal(ds_intvar1['intvar', 10:], np.nan)
finally:
shutil.rmtree(tempdir)
def test_r_tsv_io():
"Test reading output of write.table"
path = file_path('r-write.table.txt')
ds = load.tsv(path, types={'row': 'f'})
assert_array_equal(ds['row'], ['1', '2'])
assert_array_equal(ds['participant'], [1, 1])
assert_array_equal(ds['condition'], ['3B', '3B'])
assert_array_equal(ds['bin'], [0, 0])
def test_tsv_io():
"""Test tsv I/O"""
names = ['A', 'B', 'rm', 'intvar', 'fltvar', 'index']
# get Dataset
ds = datasets.get_uv()
ds['fltvar'][5:10] = np.nan
ds[:4, 'rm'] = ''
# save and load
tempdir = tempfile.mkdtemp()
try:
dst = os.path.join(tempdir, 'ds.txt')
ds.save_txt(dst)
ds1 = load.tsv(dst)
ds2 = load.tsv(dst, skiprows=1, names=names)
finally:
shutil.rmtree(tempdir)
assert_dataset_equal(ds1, ds, "TSV write/read test failed", 10)
assert_dataset_equal(ds2, ds, "TSV write/read test failed", 10)
def load_log(self):
"""Loads the log of what stimulus screen was presented and combines the log with
stimuli variables and behavioral statistics.
"""
subject = self.get('subject')
# load the log file
path = self.get('log-file', fmatch=True)
log = load.tsv(path)
log = log[log['BeforeOrAfter'].isnot(128)] # remove triggers to button press
# TODO keep button press triggers. Requires mangling rt to repeat each row
# load the processed behavioral file + stimulus variables
rt_path = os.path.join('/Volumes', 'BackUp', 'sufAmb_behavioral', 'combined', subject+'.txt')
try:
rt = load.tsv(rt_path)
except IOError:
raise IOError('Could not find behavioral file for %s.' % subject)
# combine the log file and behavioral/stim variable files
word_map = dict((w, i) for i, w in enumerate(list(log['Stimulus']))) # note order of words in log
index = []
for case in range(rt.n_cases):
w = rt[case]['Word']
i = word_map[w]
index.append(i)
rt['index'] = Var(index)
rt.sort('index') # align rows of log with rt
log.update(rt, replace=False, info=False) # combine
log = log[log['RealWord'] == 1] # event file currently does not contain nonwords
log.sort('AbsTime') # make sure log file is sorted by time
return log
def test_tsv_io():
"""Test tsv I/O"""
tempdir = TempDir()
names = ['A', 'B', 'rm', 'intvar', 'fltvar', 'fltvar2', 'index']
ds = datasets.get_uv()
ds['fltvar'][5:10] = np.nan
ds[:4, 'rm'] = ''
# save and load
dst = Path(tempdir) / 'ds.txt'
ds.save_txt(dst)
ds1 = load.tsv(dst, random='rm')
assert_dataset_equal(ds1, ds, decimal=10)
ds1 = load.tsv(dst, skiprows=1, names=names, random='rm')
assert_dataset_equal(ds1, ds, decimal=10)
# delimiter
for delimiter in [' ', ',']:
ds.save_txt(dst, delimiter=delimiter)
ds1 = load.tsv(dst, delimiter=delimiter, random='rm')
assert_dataset_equal(ds1, ds, decimal=10)
# guess data types with missing
intvar2 = ds['intvar'].as_factor()
intvar2[10:] = ''
ds_intvar = Dataset((intvar2, ))
ds_intvar.save_txt(dst)
ds_intvar1 = load.tsv(dst, empty='nan')
assert_dataobj_equal(ds_intvar1['intvar', :10], ds['intvar', :10])
assert_array_equal(ds_intvar1['intvar', 10:], np.nan)
# str with space
ds[:5, 'A'] = 'a 1'
ds.save_txt(dst)
ds1 = load.tsv(dst, random='rm')
assert_dataset_equal(ds1, ds, decimal=10)
ds.save_txt(dst, delimiter=' ')
ds1 = load.tsv(dst, delimiter=' ', random='rm')
assert_dataset_equal(ds1, ds, decimal=10)
# Fixed column width
path = file_path('fox-prestige')
ds = load.tsv(path, delimiter=' ', skipinitialspace=True)
assert ds[1] == {
'id': 'GENERAL.MANAGERS',
'education': 12.26,
'income': 25879,
'women': 4.02,
'prestige': 69.1,
'census': 1130,
'type': 'prof'
}
def test_io_txt():
"Test Dataset io as text"
ds = datasets.get_uv()
# Var that has integer values as float
ds['intflt'] = ds.eval('intvar * 1.')
ds['intflt'].name = 'intflt'
# io test
tempdir = tempfile.mkdtemp()
try:
dest = os.path.join(tempdir, 'test.txt')
ds.save_txt(dest)
ds2 = load.tsv(dest)
finally:
shutil.rmtree(tempdir)
assert_dataset_equal(ds, ds2, decimal=6)
def test_io_txt():
"Test Dataset io as text"
ds = datasets.get_uv()
# Var that has integer values as float
ds['intflt'] = ds.eval('intvar * 1.')
ds['intflt'].name = 'intflt'
# io test
tempdir = tempfile.mkdtemp()
try:
dest = os.path.join(tempdir, 'test.txt')
ds.save_txt(dest)
ds2 = load.tsv(dest)
finally:
shutil.rmtree(tempdir)
assert_dataset_equal(ds, ds2, decimal=6)
def load_logs(log_path=logPath):
log_names = os.listdir(log_path)
# remove other files in the directory
if '.DS_Store' in log_names:
log_names.remove('.DS_Store')
if 'old' in log_names:
log_names.remove('old')
logs = []
for s in log_names:
path = os.path.join(logPath, s)
log = load.tsv(path)
log.info['name'] = log.name # store name here b/c log.name gets reset
# fix 'spread(s)'
log = log[log['Stimulus'].isnot('spread', 'spreads')]
if log.info['name'] == 'R0823_s1_stimuli_data.txt': # this subject doesn't have the same nonword stimuli
log = log[log['Correct'].isnot('NotWord')]
log.name = s
logs.append(log)
print 'Done reading log files.'
return logs
def test_r_tsv_io():
"Test reading output of write.table"
ds = load.tsv(file_path('r-write.table.txt'))
assert_array_equal(ds['row'], ['1', '2'])
assert_array_equal(ds['bin'], [0, 0])
def reformat_logs():
# first, format the logs in /meg directory
meg_path = os.path.join('/Volumes', 'Backup', 'sufAmb', 'meg')
subjects = os.listdir(meg_path)
if '.DS_Store' in subjects:
subjects.remove('.DS_Store')
for s in subjects:
log_path = os.path.join(meg_path, s, 'log', 'old')
file_names = os.listdir(log_path)
if '.DS_Store' in file_names:
file_names.remove('.DS_Store')
for f in file_names:
f_path = os.path.join(log_path, f)
if 'data' in f:
# 'data' file
ds = load.tsv(f_path, names=['Key','Button','RT',
'Abs_RT','Shown','Tag',
'Experiment', 'Trial','Condition',
'Stimulus', 'Trigger', 'Correct'],
start_tag='START',
ignore_missing=True)
# filter unanalyzed screens
ds = ds[ds['Shown'].isnot('Paragraph')]
ds = ds[ds['Experiment'].isnot('practice')]
ds = ds[ds['Stimulus'].isnot('')]
del ds['Condition'] # shadows 'Condition' from rt file and causes weird problem with update()
if f == 'R0823_s1_stimuli_data.txt': # change british spelling for this subject
brit = {'favour':'favor', 'favours':'favors', 'favoured':'favored',
'honour':'honor', 'honours':'honors', 'honoured':'honored'}
for w in brit:
i = ds['Stimulus'].index(w)
ds['Stimulus'][i] = brit[w]
# remove 'spread' and 'spreads' if they are in ds
ds = ds[ds['Stimulus'].isnot('spread', 'spreads')]
elif 'log' in f:
# 'log' files
ds = load.tsv(f_path, names=['StimOrTrig','ScreenType','BeforeOrAfter',
'AbsTime','PracOrExper','TrialNum',
'Condition', 'Stimulus','Trigger',
'WordOrNonword'], start_tag='START',
ignore_missing=True)
# filter unanlayzed screens
ds = ds[ds['StimOrTrig'].isnot('STIM')]
ds = ds[ds['BeforeOrAfter'].isnot('', 'Tag')] # intermissions
del ds['Condition'] # shadows 'Condition' from rt file and causes weird problem with update()
if f == 'R0823_s1_stimuli_log.txt': # change british spelling for this subject
brit = {'favour':'favor', 'favours':'favors', 'favoured':'favored',
'honour':'honor', 'honours':'honors', 'honoured':'honored'}
for w in brit:
i = ds['Stimulus'].index(w)
ds['Stimulus'][i] = brit[w]
# remove 'spread' and 'spreads' if they are in ds
ds = ds[ds['Stimulus'].isnot('spread', 'spreads')]
else:
raise IOError('Could not find log file.')
ds.save_txt(os.path.join(meg_path, s, 'log', f))
# second, format the logs in /SufAmb_behavioral directory
rt_path = os.path.join('/Volumes', 'Backup', 'sufAmb_behavioral', 'raw', 'old')
file_names = os.listdir(rt_path)
if '.DS_Store' in file_names:
file_names.remove('.DS_Store')
for f in file_names:
f_path = os.path.join(rt_path, f)
ds = load.tsv(f_path, names=['Key','Button','RT',
'Abs_RT','Shown','Tag',
'Experiment', 'Trial','Condition',
'Stimulus', 'Trigger', 'Correct'],
start_tag='START',
ignore_missing=True)
# unanalyzed screens
ds = ds[ds['Shown'].isnot('Paragraph')]
ds = ds[ds['Experiment'].isnot('practice')]
ds = ds[ds['Stimulus'].isnot('')]
del ds['Condition'] # shadows 'Condition' from rt file and causes weird problem with update()
if f == 'R0823_s1_stimuli_data.txt': # change british spelling for this subject
brit = {'favour':'favor', 'favours':'favors', 'favoured':'favored',
'honour':'honor', 'honours':'honors', 'honoured':'honored'}
for w in brit:
i = ds['Stimulus'].index(w)
ds['Stimulus'][i] = brit[w]
# remove 'spread' and 'spreads' if they are in ds
ds = ds[ds['Stimulus'].isnot('spread', 'spreads')]
ds.save_txt(os.path.join('/Volumes', 'Backup', 'sufAmb_behavioral', 'raw', f))
file_names.remove('.DS_Store')
for f in file_names:
f_path = os.path.join(rt_path, f)
ds = load.tsv(f_path, names=['Key','Button','RT',
'Abs_RT','Shown','Tag',
'Experiment', 'Trial','Condition',
'Stimulus', 'Trigger', 'Correct'],
start_tag='START',
ignore_missing=True)
# unanalyzed screens
ds = ds[ds['Shown'].isnot('Paragraph')]
ds = ds[ds['Experiment'].isnot('practice')]
ds = ds[ds['Stimulus'].isnot('')]
del ds['Condition'] # shadows 'Condition' from rt file and causes weird problem with update()
if f == 'R0823_s1_stimuli_data.txt': # change british spelling for this subject
brit = {'favour':'favor', 'favours':'favors', 'favoured':'favored',
'honour':'honor', 'honours':'honors', 'honoured':'honored'}
for w in brit:
i = ds['Stimulus'].index(w)
ds['Stimulus'][i] = brit[w]
# remove 'spread' and 'spreads' if they are in ds
ds = ds[ds['Stimulus'].isnot('spread', 'spreads')]
ds.save_txt(os.path.join('/Volumes', 'Backup', 'sufAmb_behavioral', 'raw', f))
if __name__ == '__main__':
reformat_logs()
print 'Done'
path = os.path.join('/Volumes', 'Backup', 'sufAmb_behavioral', 'raw', 'R0823_s1_stimuli_data.txt')
ds = load.tsv(path)
import os
import numpy as np
from eelbrain import load, Var, Factor, combine
logPath = os.path.join('/Volumes','BackUp','sufAmb_behavioral', 'raw')
masterPath = os.path.join('/Volumes','BackUp','sufAmb_stimuli','stimuli',
'stim_properties','master_list35_update6.txt')
masterRT = load.tsv(masterPath) # complete set of stimuli and properties
def load_logs(log_path=logPath):
log_names = os.listdir(log_path)
# remove other files in the directory
if '.DS_Store' in log_names:
log_names.remove('.DS_Store')
if 'old' in log_names:
log_names.remove('old')
logs = []
for s in log_names:
path = os.path.join(logPath, s)
log = load.tsv(path)
log.info['name'] = log.name # store name here b/c log.name gets reset
# fix 'spread(s)'
log = log[log['Stimulus'].isnot('spread', 'spreads')]
if log.info['name'] == 'R0823_s1_stimuli_data.txt': # this subject doesn't have the same nonword stimuli
log = log[log['Correct'].isnot('NotWord')]
log.name = s
logs.append(log)
def test_r_tsv_io():
"Test reading output of write.table"
ds = load.tsv(file_path('r-write.table.txt'))
assert_array_equal(ds['row'], ['1', '2'])
assert_array_equal(ds['bin'], [0, 0])