def test_io():
refdm = DataMatrix(length=3)
refdm[u'tést'] = 1, 2, u''
refdm.B = u'mathôt', u'b', u'x'
refdm.C = u'a,\\b"\'c', 8, u''
testdm = io.readtxt('testcases/data/data.csv')
check_dm(refdm, testdm)
io.writetxt(testdm, 'tmp.csv')
testdm = io.readtxt('tmp.csv')
check_dm(refdm, testdm)
refdm = io.readtxt('testcases/data/line-ending-cr.csv')
check_dm(refdm, testdm)
refdm = io.readtxt('testcases/data/line-ending-crlf.csv')
check_dm(refdm, testdm)
refdm = io.readtxt('testcases/data/data-with-bom.csv')
check_dm(refdm, testdm)
io.writepickle(testdm, 'tmp.pickle')
testdm = io.readpickle('tmp.pickle')
check_dm(refdm, testdm)
io.writexlsx(testdm, 'tmp.xlsx')
with pytest.warns(UserWarning): # Not all rows have column C
testdm = io.readxlsx('tmp.xlsx')
check_dm(refdm, testdm)
io.writexlsx(testdm, 'tmp.xlsx')
with pytest.warns(UserWarning): # Not all rows have column C
testdm = io.readxlsx('tmp.xlsx')
check_dm(refdm, testdm)
def test_io():
refdm = DataMatrix(length=3)
refdm[u'tést'] = 1, 2, u''
refdm.B = u'mathôt', u'b', u'x'
refdm.C = u'a,\\b"\'c', 8, u''
testdm = io.readtxt('testcases/data/data.csv')
check_dm(refdm, testdm)
io.writetxt(testdm, 'tmp.csv')
testdm = io.readtxt('tmp.csv')
check_dm(refdm, testdm)
refdm = io.readtxt('testcases/data/line-ending-cr.csv')
check_dm(refdm, testdm)
refdm = io.readtxt('testcases/data/line-ending-crlf.csv')
check_dm(refdm, testdm)
io.writepickle(testdm, 'tmp.pickle')
testdm = io.readpickle('tmp.pickle')
check_dm(refdm, testdm)
io.writexlsx(testdm, 'tmp.xlsx')
testdm = io.readxlsx('tmp.xlsx')
check_dm(refdm, testdm)
def get_trace_data(sub, run, tracename):
"""A generic function to get a simple trace."""
dm = io.readtxt(TRACE_SRC.format(sub=sub, run=run))
dm = ops.auto_type(dm)
dm[tracename] @= lambda i: np.nan if i == 0 else i
dm[tracename] = srs._interpolate(ops.z(dm[tracename]))
return deconv_hrf(flt(dm[tracename]))
def get_arousal_data(run):
"""Get arousal data for one avmovie run"""
dm = io.readtxt(SRC_EMOTION)
t0, t1 = EMOTION_SEGMENTATION[run - 1]
a = list(dm.arousal[t0 // 2:t1 // 2]) # Align with 2s fMRI data
if run == 8: # One sample missing from the emotion timeseries in run 8!
a.append(a[-1])
return deconv_hrf(a)
def test_io():
refdm = DataMatrix(length=3)
refdm[u'tést'] = 1, 2, u''
refdm.B = u'mathôt', u'b', u'x'
refdm.C = u'a,\\b"\'c', 8, u''
testdm = io.readtxt('testcases/data/data.csv')
check_dm(refdm, testdm)
io.writetxt(testdm, 'tmp.csv')
testdm = io.readtxt('tmp.csv')
check_dm(refdm, testdm)
io.writepickle(testdm, 'tmp.pickle')
testdm = io.readpickle('tmp.pickle')
check_dm(refdm, testdm)
io.writexlsx(testdm, 'tmp.xlsx')
testdm = io.readxlsx('tmp.xlsx')
check_dm(refdm, testdm)
def parsefile(path):
"""
desc:
Parse a single .txt data file, and return it as a DataMatrix.
arguments:
path:
desc: The path to the data file.
type: str
returns:
type: DataMatrix
"""
print('Reading %s' % path)
src = io.readtxt(path, delimiter='\t')
dm = DataMatrix(length=0)
trialdm = None
for row in src:
if isinstance(row.state, str):
if 'start_trial' in row.state:
if trialdm is not None:
print('%d: %d samples (%d invalid) %s' \
% (trialdm.trialid[0], sample, invalid, \
trialdm.snelheid[0]))
dm <<= trialdm
trialdm = DataMatrix(length=1)
trialdm.pupil = SeriesColumn(1000)
trialdm.trialid = int(row.state.split()[1])
trialdm.path = path
sample = 0
invalid = 0
continue
if 'var' in row.state:
l = row.state[4:].split(maxsplit=1)
if len(l) == 1:
continue
var, val = tuple(l)
# Length is a special property for datamatrix
if var == 'length':
var = '_length'
trialdm[var] = val
continue
continue
if trialdm is not None:
if row.Rpsize != '' and row.Rpsize > 0:
trialdm.pupil[0, sample] = row.Rpsize
else:
trialdm.pupil[0, sample] = np.nan
invalid += 1
sample += 1
dm <<= trialdm
return dm
def video_timeseries(subject, run, frames):
dm = io.readtxt(SRC_LUMINANCE.format(subject=subject, run=run))
dm = dm.sub == subject
luminance = np.empty(frames.shape)
luminance[:] = np.nan
change = np.empty(frames.shape)
change[:] = np.nan
for row in dm:
i = frames == row.frame
luminance[i] = row.luminance
change[i] = row.change
return luminance, change
def get_pupil_data(sub, run):
"""Get a preprocessed pupil trace for one subject and one avmovie run """
dm = io.readtxt(TRACE_SRC.format(sub=sub, run=run))
dm = ops.auto_type(dm)
dm.pupil_size @= lambda i: np.nan if i == 0 else i
dm.luminance @= lambda i: np.nan if i == 0 else i
dm.pupil_size = srs._interpolate(ops.z(dm.pupil_size))
dm.luminance = srs._interpolate(ops.z(dm.luminance))
if REMOVE_LUMINANCE_FROM_PUPIL:
i, s1, s2 = np.polyfit(dm.pupil_size, dm.luminance, deg=2)
dm.pupil_size -= i + s1 * dm.luminance + s2 * dm.luminance ** 2
return deconv_hrf(flt(dm.pupil_size))
def test_convert():
refdm = DataMatrix(length=3)
refdm[u'tést'] = 1, 2, u''
refdm.B = u'mathôt', u'b', u'x'
refdm.C = u'a,\\b"\'c', 8, u''
testdm = io.readtxt('testcases/data/data.csv')
check_dm(refdm, testdm)
testdm = cnv.from_json(cnv.to_json(testdm))
check_dm(refdm, testdm)
testdf = cnv.to_pandas(testdm)
testdm = cnv.from_pandas(testdf)
check_dm(refdm, testdm)
print(np.array(testdf))
print(np.array(testdm))
assert (np.array(testdf, dtype=str) == np.array(testdm, dtype=str)).all()
def _launchr(dm, cmd):
dm = dm[:]
# SeriesColumns cannot be saved to a csv file, so we delete those first.
for name, col in dm.columns:
if isinstance(col, _SeriesColumn):
del dm[name]
# Write the data to an input file
io.writetxt(dm, u'.r-in.csv')
# Launch R, read the data, and communicate the commands
proc = subprocess.Popen( ['R', '--vanilla'], stdin=subprocess.PIPE)
# proc = subprocess.Popen( ['R', '--vanilla'], stdin=subprocess.PIPE,
# stdout=subprocess.PIPE, stderr=subprocess.PIPE)
cmd = u'data <- read.csv(".r-in.csv")\nattach(data)\n%s' % cmd
proc.communicate(safe_encode(cmd, u'ascii'))
# Wait until the output file has been generated and return it
while not os.path.exists(u'.r-out.csv'):
time.sleep(.5)
dm = io.readtxt(u'.r-out.csv')
return dm
def _read_file(self):
"""
desc:
Reads a source file and raises an osexception if this fails.
returns:
type: DataMatrix
"""
from datamatrix import io
src = self.experiment.pool[self.var.source_file]
if src.endswith(u'.xlsx'):
try:
return io.readxlsx(src)
except Exception as e:
raise osexception(u'Failed to read .xlsx file: %s' % src,
exception=e)
try:
return io.readtxt(src)
except Exception as e:
raise osexception(
(u'Failed to read text file (perhaps it has the '
u'wrong format or it is not utf-8 encoded): %s') % src,
exception=e)
def _launchr(dm, cmd):
dm = dm[:]
# SeriesColumns cannot be saved to a csv file, so we delete those first.
for name, col in dm.columns:
if isinstance(col, _SeriesColumn):
del dm[name]
# Write the data to an input file
io.writetxt(dm, u'.r-in.csv')
# Launch R, read the data, and communicate the commands
if verbose:
proc = subprocess.Popen(['R', '--vanilla'], stdin=subprocess.PIPE)
else:
proc = subprocess.Popen(['R', '--vanilla'],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
cmd = u'data <- read.csv(".r-in.csv")\nattach(data)\n%s' % cmd
proc.communicate(safe_encode(cmd, u'ascii'))
# Wait until the output file has been generated and return it
while not os.path.exists(u'.r-out.csv'):
time.sleep(.5)
dm = io.readtxt(u'.r-out.csv')
return dm
def _create_live_datamatrix(self): """ desc: Builds a live DataMatrix. That is, it takes the orignal DataMatrix and applies all the operations as specified. returns: desc: A live DataMatrix. type: DataMatrix """ if self.var.source == u'table': src_dm = self.dm else: from datamatrix import io src = self.experiment.pool[self.var.source_file] if src.endswith(u'.xlsx'): try: src_dm = io.readxlsx(src) except Exception as e: raise osexception(u'Failed to read .xlsx file: %s' % src, exception=e) else: try: src_dm = io.readtxt(src) except Exception as e: raise osexception(u'Failed to read text file (perhaps it has the wrong format or it is not utf-8 encoded): %s' % src, exception=e) for column_name in src_dm.column_names: if not self.syntax.valid_var_name(column_name): raise osexception( u'The loop table contains an invalid column name: 'u'\'%s\'' \ % column_name) # The number of repeats should be numeric. If not, then give an error. # This can also occur when generating a preview of a loop table if # repeat is variable. if not isinstance(self.var.repeat, (int, float)): raise osexception( u'Don\'t know how to generate a DataMatrix for "%s" repeats' \ % self.var.repeat) length = int(len(src_dm) * self.var.repeat) dm = DataMatrix(length=0) while len(dm) < length: i = min(length-len(dm), len(src_dm)) if self.var.order == u'random': dm <<= operations.shuffle(src_dm)[:i] else: dm <<= src_dm[:i] if self.var.order == u'random': dm = operations.shuffle(dm) if self.ef is not None: self.ef.dm = dm dm = self.ef.enforce() for cmd, arglist in self.operations: # The column name is always specified last, or not at all if arglist: try: colname = arglist[-1] col = dm[colname] except: raise osexception( u'Column %s does not exist' % arglist[-1]) if cmd == u'fullfactorial': dm = operations.fullfactorial(dm) elif cmd == u'shuffle': if not arglist: dm = operations.shuffle(dm) else: dm[colname] = operations.shuffle(col) elif cmd == u'shuffle_horiz': if not arglist: dm = operations.shuffle_horiz(dm) else: dm = operations.shuffle_horiz( *[dm[_colname] for _colname in arglist]) elif cmd == u'slice': self._require_arglist(cmd, arglist, minlen=2) dm = dm[arglist[0]: arglist[1]] elif cmd == u'sort': self._require_arglist(cmd, arglist) dm[colname] = operations.sort(col) elif cmd == u'sortby': self._require_arglist(cmd, arglist) dm = operations.sort(dm, by=col) elif cmd == u'reverse': if not arglist: dm = dm[::-1] else: dm[colname] = col[::-1] elif cmd == u'roll': self._require_arglist(cmd, arglist) steps = arglist[0] if not isinstance(steps, int): raise osexception(u'roll steps should be numeric') if len(arglist) == 1: dm = dm[-steps:] << dm[:-steps] else: dm[colname] = list(col[-steps:]) + list(col[:-steps]) elif cmd == u'weight': self._require_arglist(cmd, arglist) dm = operations.weight(col) return dm
TNTfMRI = neurodesign.classes.experiment(
TR=2,rho=0, # rho (assumed auotcorrelation coefficient) greatly impacts Fd i.e. higher rho (0.3 = default) = much lower Fd / 0 is best
n_stimuli=5, n_trials=124,
P=[0.20,0.20,0.20,0.20,0.20],
C=[[1,1,-1,-1,0],[1,0,-1,0,0],[0,1,0,-1,0]],
t_pre=0,stim_duration=3,t_post=1, # since there are only 20% NULL events incl here I am ok with the intrusion ratings being modled with the NULL events, to add a bit more time to them (does that reasoning work? Or is it the frequency of NULL events that important i.e. not their duration?)
#hardprob=True,maxrep=3, # (maxrep=3 is not true for NULL events)
restnum=0,restdur=0,
ITImodel="exponential",resolution=0.1, # resolution greatly impacts Fe i.e. higher (0.4) = much lower Fe / 0.1 is best
ITImin=1.4,ITImean=2,ITImax=2.6,
FeMax=1,FdMax=1,FcMax=1,FfMax=1,
confoundorder=3) # lower confound order seems to more consistently yield better Fe (3 is default, but 1 seems to be best - however, still possible to get a good result with loop if set to 3)
# Read a datafile with Pandas and convert it to a pseudorandom DataFrame.
TNTpseudM = io.readtxt('/Users/claudius/PycharmProjects2/EXPdesignOP/filepool/TNTrun2csv.csv')
# optimisation loop
Fe_threshold = 13#14.3 #(15)
Fd_threshold = 6#6.3 #(6.5)
Ff_threshold = 0.9 #(1)
Fc_threshold = 0.8 #(0.8)
timeout = time.time() + 60*5 # 5 minutes from now
while True:
# Create an Enforce object and add constraints
ef = Enforce(TNTpseudM)
ef.add_constraint(MaxRep,cols=[TNTpseudM.T_ID],maxrep=3)
# and negative distractor 2
L_NEG_DIST2 = list(neg_dm.Scene)
# Make a list of potential scenes for neutral distractor 1
L_NEU_DIST1 = list(neu_dm.Scene)
# and 2
L_NEU_DIST2 = list(neu_dm.Scene)
new_dm = combine(dm, L_NEG_DIST1[:], L_NEG_DIST2[:], L_NEU_DIST1[:],
L_NEU_DIST2[:])
# Make sure target, distractor 1 and distractor 2 are different scenes:
while any(row.distractor_scene_1 == row.distractor_scene_2
or row.distractor_scene_1 == row.Scene
or row.distractor_scene_2 == row.Scene for row in new_dm):
# If not, try again:
print("try again")
new_dm = combine(dm, L_NEG_DIST1[:], L_NEG_DIST2[:], L_NEU_DIST1[:],
L_NEU_DIST2[:])
return new_dm
if __name__ == "__main__":
# Read dm containing stimuli without distractors:
dm_exp = io.readtxt('dm_exp.csv')
dm_final = addDistractors(dm_exp)
io.writetxt(dm_final, "./trial_list.csv")
You should have received a copy of the GNU General Public License
along with P0005.1. If not, see <http://www.gnu.org/licenses/>.
"""
import os
from datamatrix import io, DataMatrix, FloatColumn, plot
import numpy as np
# First merge all ratings into a single DataMatrix
dm = None
for path in os.listdir('data-ratings'):
if not path.endswith('.csv'):
continue
print('Reading %s' % path)
if dm is None:
dm = io.readtxt('data-ratings/%s' % path)
else:
dm <<= io.readtxt('data-ratings/%s' % path)
# This word was misspelled
dm = dm.word != 'térébrant'
# There are 30 participants who provided 2 ratings for 122 words
assert(len(dm) == 30*2*102)
rm = DataMatrix(length=101)
rm.word = ''
rm._type = ''
rm.rating_brightness = FloatColumn
rm.rating_valence = FloatColumn
rm.rating_intensity = FloatColumn
for row, word in zip(rm, dm.word.unique):
def setUp(self):
self.dm = io.readtxt('examples/data.csv')
def filter_(dm):
"""
desc:
Applies basic filtering to the parsed data, by removing unnecessary
columns, filtering out practice trials, etc. This function does not do
pupil-size preprocessing; this is handled by pupil.preprocess()
arguments:
dm:
type: DataMatrix
returns:
type: DataMatrix
"""
# Keep only relevant columns to speed up processing
ops.keep_only(dm, ['trialid', 'word', 'type', 'ptrace_target', 'category',
'ptrace_fixation', 'subject_nr', 'response_time_keyboard_response',
'correct_keyboard_response', 'practice'])
dm.rename('response_time_keyboard_response', 'rt')
dm.rename('correct_keyboard_response', 'correct')
# Remove practice trials, and the misspelled word
print('Filtering')
if constants.EXP == 'control':
dm = dm.practice == 'no'
else:
dm = dm.trialid >= 10
dm = dm.word != 'trbrant'
# Show all words and trialcounts
for word in dm.word.unique:
print('%s\t%d' % (word, len(dm.word == word)))
# First show error information, and then remove error trials
errors = len(dm.correct == 0)
error_percent = 100. * errors / len(dm)
n_word = len(dm.word.unique)
print('N(word) = %d Errors(Total) = %d (%.2f %%) of %d' \
% (n_word, errors, error_percent, len(dm)))
categories = ('positive', 'negative', 'animal') \
if constants.EXP == 'control' else ('light', 'dark', 'ctrl', 'animal')
print(dm.category.unique)
for type_ in categories:
if constants.EXP == 'control':
dm_ = dm.category == type_
else:
dm_ = dm.type == type_
total = len(dm_)
errors = len((dm_) & (dm.correct == 0))
error_percent = 100. * errors / total
n_word = len(dm_.word.unique)
print('N(word) = %d, Errors(%s) = %d (%.2f %%) of %d' \
% (n_word, type_, errors, error_percent, total))
dm = dm.correct == 1
if constants.EXP == 'control':
return dm
# Add new columns
print('Adding columns')
dm.rating_brightness = FloatColumn
dm.rating_valence = FloatColumn
dm.rating_intensity = FloatColumn
dm.word_len = IntColumn
dm.word_len = [len(word) for word in dm.word]
# Integrate the normative ratings and French lexicon project into the data
print('Integrating normative ratings')
ratings = io.readtxt('ratings.csv')
ratings.ascii_word = [word.encode('ascii', errors='ignore') \
for word in ratings.word]
for row in dm:
# The animal names haven't been rated
if row.type == 'animal':
continue
# The special characters have been stripped in the EDF file, but not from
# the ratings data. We assert that we have exactly one match between the
# ascii-fied and original word to make sure that we map things correctly.
# In addition, in the auditory experiment the special characters were
# converted to html hex notation. So we strip those characters.
word = ''.join([l for l in row.word if l.isalpha()])
rating = ratings.ascii_word == word
assert(len(rating) == 1)
row.rating_brightness = rating.rating_brightness[0]
row.rating_valence = rating.rating_valence[0]
row.rating_intensity = rating.rating_intensity[0]
return dm
final_dm = dm_fillers_slice_first << main_dm
# And to the end fo the final dm:
dm_fillers_slice_last = dm_fillers[6:]
final_dm = final_dm << dm_fillers_slice_last
return final_dm
if __name__ == "__main__":
src_file = "trial list postscan"
dst_file = "trial list postscan/block loops IMDF postscan"
# And experimental trials (without distractors)
dm_exp = io.readtxt(os.path.join(src_file, 'IMDF_pairs_exp.csv'))
dm_exp = getBlockLoopsScanner.addValenceColumn(dm_exp)
dm_exp["Exp_ID"] = "IMDF_postscan"
# Read dm containing fillers (without distractors)
dm_fillers = io.readtxt(os.path.join(src_file, 'IMDF_pairs_fillers.csv'))
dm_fillers["Exp_ID"] = "IMDF_postscan"
for subject_ID in range(1, 71):
pp_dm = applyConstraints(dm_exp, dm_fillers)
io.writetxt(
pp_dm,
os.path.join(dst_file,
"block_loop_IMDF_postscan_PP_%s.csv" % subject_ID))
dm_fillers_slice_last = dm_fillers[10:]
final_dm = final_dm << dm_fillers_slice_last
# Append to the dm list:
list_dms.append(final_dm)
# Unpack the list so we can return two separate dms
dm1, dm2 = list_dms
return dm1, dm2
if __name__ == "__main__":
src_path = "trial list prescan"
dst_path = "trial list prescan"
# And experimental trials (without distractors)
dm_exp = io.readtxt(os.path.join(src_path, 'dm_exp.csv'))
# Read dm containing fillers (without distractors)
dm_fillers = io.readtxt(os.path.join(src_path, 'dm_fillers.csv'))
for subject_ID in range(1, 71):
dm1, dm2 = applyConstraints(dm_exp, dm_fillers)
io.writetxt(dm1, os.path.join(dst_path, "block_loop_test_feedback_PP_%s.csv" % subject_ID))
io.writetxt(dm2, os.path.join(dst_path, "block_loop_criterion_test_PP_%s.csv" % subject_ID))
def _create_live_datamatrix(self):
"""
desc:
Builds a live DataMatrix. That is, it takes the orignal DataMatrix
and applies all the operations as specified.
returns:
desc: A live DataMatrix.
type: DataMatrix
"""
if self.var.source == u'table':
src_dm = self.dm
else:
from datamatrix import io
src = self.experiment.pool[self.var.source_file]
if src.endswith(u'.xlsx'):
try:
src_dm = io.readxlsx(src)
except Exception as e:
raise osexception(u'Failed to read .xlsx file: %s' % src,
exception=e)
else:
try:
src_dm = io.readtxt(src)
except Exception as e:
raise osexception(
u'Failed to read text file (perhaps it has the wrong format or it is not utf-8 encoded): %s'
% src,
exception=e)
for column_name in src_dm.column_names:
if not self.syntax.valid_var_name(column_name):
raise osexception(
u'The loop table contains an invalid column name: 'u'\'%s\'' \
% column_name)
# The number of repeats should be numeric. If not, then give an error.
# This can also occur when generating a preview of a loop table if
# repeat is variable.
if not isinstance(self.var.repeat, (int, float)):
raise osexception(
u'Don\'t know how to generate a DataMatrix for "%s" repeats' \
% self.var.repeat)
length = int(len(src_dm) * self.var.repeat)
dm = DataMatrix(length=0)
while len(dm) < length:
i = min(length - len(dm), len(src_dm))
if self.var.order == u'random':
dm <<= operations.shuffle(src_dm)[:i]
else:
dm <<= src_dm[:i]
if self.var.order == u'random':
dm = operations.shuffle(dm)
if self.ef is not None:
self.ef.dm = dm
dm = self.ef.enforce()
for cmd, arglist in self.operations:
# The column name is always specified last, or not at all
if arglist:
try:
colname = arglist[-1]
col = dm[colname]
except:
raise osexception(u'Column %s does not exist' %
arglist[-1])
if cmd == u'fullfactorial':
dm = operations.fullfactorial(dm)
elif cmd == u'shuffle':
if not arglist:
dm = operations.shuffle(dm)
else:
dm[colname] = operations.shuffle(col)
elif cmd == u'shuffle_horiz':
if not arglist:
dm = operations.shuffle_horiz(dm)
else:
dm = operations.shuffle_horiz(
*[dm[_colname] for _colname in arglist])
elif cmd == u'slice':
self._require_arglist(cmd, arglist, minlen=2)
dm = dm[arglist[0]:arglist[1]]
elif cmd == u'sort':
self._require_arglist(cmd, arglist)
dm[colname] = operations.sort(col)
elif cmd == u'sortby':
self._require_arglist(cmd, arglist)
dm = operations.sort(dm, by=col)
elif cmd == u'reverse':
if not arglist:
dm = dm[::-1]
else:
dm[colname] = col[::-1]
elif cmd == u'roll':
self._require_arglist(cmd, arglist)
steps = arglist[0]
if not isinstance(steps, int):
raise osexception(u'roll steps should be numeric')
if len(arglist) == 1:
dm = dm[-steps:] << dm[:-steps]
else:
dm[colname] = list(col[-steps:]) + list(col[:-steps])
elif cmd == u'weight':
self._require_arglist(cmd, arglist)
dm = operations.weight(col)
return dm
from datamatrix import io
from pseudorandom import Enforce, MaxRep, MinDist
# Read a datafile with Pandas and convert it to a pseudorandom DataFrame.
dm = io.readtxt('examples/data.csv')
print(dm)
# Create an Enforce object, and add two constraints
ef = Enforce(dm)
ef.add_constraint(MaxRep, cols=[dm.category], maxrep=1)
ef.add_constraint(MinDist, cols=[dm.word], mindist=4)
# Enforce the constraints
dm = ef.enforce()
# See the resulting DataFrame and a report of how long the enforcement took.
print(dm)
print(ef.report)
def _create_live_datamatrix(self): """ desc: Builds a live DataMatrix. That is, it takes the orignal DataMatrix and applies all the operations as specified. returns: desc: A live DataMatrix. type: DataMatrix """ if self.var.source == u'table': src_dm = self.dm else: from datamatrix import io src = self.experiment.pool[self.var.source_file] if src.endswith(u'.xlsx'): try: src_dm = io.readxlsx(src) except Exception as e: raise osexception(u'Failed to read .xlsx file: %s' % src, exception=e) else: try: src_dm = io.readtxt(src) except Exception as e: raise osexception(u'Failed to read text file: %s' % src, exception=e) length = int(len(src_dm) * self.var.repeat) dm = DataMatrix(length=0) while len(dm) < length: i = min(length-len(dm), len(src_dm)) if self.var.order == u'random': dm <<= operations.shuffle(src_dm)[:i] else: dm <<= src_dm[:i] if self.var.order == u'random': dm = operations.shuffle(dm) if self.ef is not None: self.ef.dm = dm dm = self.ef.enforce() for cmd, arglist in self.operations: # The column name is always specified last, or not at all if arglist: try: colname = arglist[-1] col = dm[colname] except: raise osexception( u'Column %s does not exist' % arglist[-1]) if cmd == u'fullfactorial': dm = operations.fullfactorial(dm) elif cmd == u'shuffle': if not arglist: dm = operations.shuffle(dm) else: dm[colname] = operations.shuffle(col) elif cmd == u'shuffle_horiz': if not arglist: dm = operations.shuffle_horiz(dm) else: dm = operations.shuffle_horiz( *[dm[_colname] for _colname in arglist]) elif cmd == u'slice': self._require_arglist(cmd, arglist, minlen=2) dm = dm[arglist[0]: arglist[1]] elif cmd == u'sort': self._require_arglist(cmd, arglist) dm[colname] = operations.sort(col) elif cmd == u'sortby': self._require_arglist(cmd, arglist) dm = operations.sort(dm, by=col) elif cmd == u'reverse': if not arglist: dm = dm[::-1] else: dm[colname] = col[::-1] elif cmd == u'roll': self._require_arglist(cmd, arglist) steps = arglist[0] if not isinstance(steps, int): raise osexception(u'roll steps should be numeric') if len(arglist) == 1: dm = dm[-steps:] << dm[:-steps] else: dm[colname] = list(col[-steps:]) + list(col[:-steps]) elif cmd == u'weight': self._require_arglist(cmd, arglist) dm = operations.weight(col) return dm
def lc_data(sub, run):
dm = io.readtxt(PUPIL_SRC.format(sub=sub, run=run))
return signal.detrend(srs._interpolate(dm.LC_l_nonconf + dm.LC_r_nonconf))
def _create_live_datamatrix(self):
"""
desc:
Builds a live DataMatrix. That is, it takes the orignal DataMatrix
and applies all the operations as specified.
returns:
desc: A live DataMatrix.
type: DataMatrix
"""
if self.var.source == u'table':
src_dm = self.dm
else:
from datamatrix import io
src = self.experiment.pool[self.var.source_file]
if src.endswith(u'.xlsx'):
try:
src_dm = io.readxlsx(src)
except Exception as e:
raise osexception(u'Failed to read .xlsx file: %s' % src,
exception=e)
else:
try:
src_dm = io.readtxt(src)
except Exception as e:
raise osexception(u'Failed to read text file: %s' % src,
exception=e)
length = int(len(src_dm) * self.var.repeat)
dm = DataMatrix(length=0)
while len(dm) < length:
i = min(length - len(dm), len(src_dm))
if self.var.order == u'random':
dm <<= operations.shuffle(src_dm)[:i]
else:
dm <<= src_dm[:i]
if self.var.order == u'random':
dm = operations.shuffle(dm)
if self.ef is not None:
self.ef.dm = dm
dm = self.ef.enforce()
for cmd, arglist in self.operations:
# The column name is always specified last, or not at all
if arglist:
try:
colname = arglist[-1]
col = dm[colname]
except:
raise osexception(u'Column %s does not exist' %
arglist[-1])
if cmd == u'fullfactorial':
dm = operations.fullfactorial(dm)
elif cmd == u'shuffle':
if not arglist:
dm = operations.shuffle(dm)
else:
dm[colname] = operations.shuffle(col)
elif cmd == u'shuffle_horiz':
if not arglist:
dm = operations.shuffle_horiz(dm)
else:
dm = operations.shuffle_horiz(
*[dm[_colname] for _colname in arglist])
elif cmd == u'slice':
self._require_arglist(cmd, arglist, minlen=2)
dm = dm[arglist[0]:arglist[1]]
elif cmd == u'sort':
self._require_arglist(cmd, arglist)
dm[colname] = operations.sort(col)
elif cmd == u'sortby':
self._require_arglist(cmd, arglist)
dm = operations.sort(dm, by=col)
elif cmd == u'reverse':
if not arglist:
dm = dm[::-1]
else:
dm[colname] = col[::-1]
elif cmd == u'roll':
self._require_arglist(cmd, arglist)
steps = arglist[0]
if not isinstance(steps, int):
raise osexception(u'roll steps should be numeric')
if len(arglist) == 1:
dm = dm[-steps:] << dm[:-steps]
else:
dm[colname] = list(col[-steps:]) + list(col[:-steps])
elif cmd == u'weight':
self._require_arglist(cmd, arglist)
dm = operations.weight(col)
return dm
src_exp = "TNT_pairs_exp.csv"
src_fillers = "trial list postscan/TNT_pairs_fillers.csv"
for block in [
"eval_TNT_prescan", "learning_TNT_prescan", "refresher_TNT_prescan",
"refresher_TNT_scanner", "TNT_test_postscan", "eval_TNT_postscan"
]:
# Get path to folder beloning to the current experimental phase
# (prescan, scanner, postscan)
exp_phase = block.split("_")[-1]
dst = "trial list %s/block loops TNT %s" % (exp_phase, exp_phase)
for pp_ID in range(1, 71):
main_dm = io.readtxt(src_exp)
# Read in fillers:
dm = io.readtxt(src_fillers)
dm_first, dm_last, dm_remaining = helpers.splitFillers(dm)
# Merge the remaining fillers with the main dm:
merged_dm = main_dm << dm_remaining
# Shuffle the merged dm:
merged_dm = ops.shuffle(merged_dm)
# Create an Enforce object, and add constraint
ef = Enforce(merged_dm)
ef.add_constraint(MaxRep, cols=[merged_dm.Emotion], maxrep=3)
ef.add_constraint(MinDist, cols=[merged_dm.Trial_ID], mindist=2)
def pupil_data(sub, run):
return signal.detrend(
srs._interpolate(
io.readtxt(PUPIL_SRC.format(sub=sub, run=run)).pupil_size))
