def test_constraintMinDist(self):
self.setUp()
ef = Enforce(self.dm)
ef.add_constraint(MinDist, cols=[self.dm.word], mindist=3)
dm = ef.enforce()
for row in range(3, len(dm)):
s = dm.word[row - 3:row].unique
self.assertTrue(len(s) == 3)
def test_constraintMaxRep(self):
self.setUp()
ef = Enforce(self.dm)
ef.add_constraint(MaxRep, maxrep=1)
dm = ef.enforce()
for row in range(len(dm)):
for cell1, cell2 in zip(dm[row], dm[row - 1]):
self.assertTrue(cell1 != cell2)
def applyConstraints(dm_exp, dm_fillers):
"""
Arguments:
dm_exp --- dm containing exp scene-object combinations
dm_fillers --- dm containing filler scene-object combinations
Returns:
final_dm_test_feedback --- final datamatrix for the test feedback blockloop
final_dm_criterion_test --- final datamatrix for the test feedback blockloop
"""
# Add distractors:
dm_exp = addDistractors.addDistractors(dm_exp)
dm_fillers = addDistractors.addDistractors(dm_fillers)
# Make sure we will collect two dms
list_dms = []
for block in ["test_feedback", "criterion_test"]:
# Shuffle the fillers:
dm_fillers = ops.shuffle(dm_fillers)
# Merge first 8 fillers to the main dm
# NOTE: we leave out four fillers because we will use them for the
# very first and the very last trials
dm_fillers_slice = dm_fillers[0:8]
main_dm = dm_exp << dm_fillers_slice
# Shuffle the main dm:
main_dm = ops.shuffle(main_dm)
# Create an Enforce object, and add two constraints
ef = Enforce(main_dm)
ef.add_constraint(MaxRep, cols=[main_dm.Emotion], maxrep=3)
ef.add_constraint(MinDist, cols=[main_dm.Trial_ID], mindist=2)
# Enforce the constraints
main_dm = ef.enforce()
# See the resulting DataFrame and a report of how long the enforcement took.
# Add two fillers to the beginning of the final dm:
dm_fillers_slice_first = dm_fillers[8:10]
final_dm = dm_fillers_slice_first << main_dm
# And to the end fo the final dm:
dm_fillers_slice_last = dm_fillers[10:]
final_dm = final_dm << dm_fillers_slice_last
# Append to the dm list:
list_dms.append(final_dm)
# Unpack the list so we can return two separate dms
dm1, dm2 = list_dms
return dm1, dm2
def applyConstraints(dm_exp, dm_fillers):
"""
Arguments:
dm_exp --- dm containing exp scene-object combinations
dm_fillers --- dm containing filler scene-object combinations
Returns:
final_dm --- final datamatrix for the IMDF MC memory task
"""
# Add distractors:
dm_exp = addDistractorsImdf.addDistractors(dm_exp)
dm_fillers = addDistractorsImdf.addDistractors(dm_fillers)
# Shuffle the fillers:
dm_fillers = ops.shuffle(dm_fillers)
# Merge first 4 fillers to the main dm
# NOTE: we leave out four fillers because we will use them for the
# very first and the very last trials
# TODO: select 2 neg en 2 neutral
dm_fillers_slice = dm_fillers[0:4]
main_dm = dm_exp << dm_fillers_slice
# Shuffle the main dm:
main_dm = ops.shuffle(main_dm)
# Create an Enforce object, and add two constraints
ef = Enforce(main_dm)
ef.add_constraint(MaxRep, cols=[main_dm.Emotion], maxrep=3)
ef.add_constraint(MinDist, cols=[main_dm.Trial_ID], mindist=2)
# Enforce the constraints
main_dm = ef.enforce()
# See the resulting DataFrame and a report of how long the enforcement took.
# Add two fillers to the beginning of the final dm:
dm_fillers_slice_first = dm_fillers[4:6]
final_dm = dm_fillers_slice_first << main_dm
# And to the end fo the final dm:
dm_fillers_slice_last = dm_fillers[6:]
final_dm = final_dm << dm_fillers_slice_last
return final_dm
for pp_ID in range(1, 71):
main_dm = io.readtxt(src_exp)
# Read in fillers:
dm = io.readtxt(src_fillers)
dm_first, dm_last, dm_remaining = helpers.splitFillers(dm)
# Merge the remaining fillers with the main dm:
merged_dm = main_dm << dm_remaining
# Shuffle the merged dm:
merged_dm = ops.shuffle(merged_dm)
# Create an Enforce object, and add constraint
ef = Enforce(merged_dm)
ef.add_constraint(MaxRep, cols=[merged_dm.Emotion], maxrep=3)
ef.add_constraint(MinDist, cols=[merged_dm.Trial_ID], mindist=2)
# Enforce the constraints
merged_dm = ef.enforce()
# Add the first fillers:
merged_dm = dm_first << merged_dm
# Add the last fillers:
merged_dm = merged_dm << dm_last
# Add exp ID to the dm:
merged_dm["Exp_ID"] = block
io.writetxt(merged_dm, os.path.join(dst, "blockloop_%s_PP_%s.csv" \
% (block, pp_ID)))
# Read a datafile with Pandas and convert it to a pseudorandom DataFrame.
TNTpseudM = io.readtxt('/Users/claudius/PycharmProjects2/EXPdesignOP/filepool/TNTrun2csv.csv')
# optimisation loop
Fe_threshold = 13#14.3 #(15)
Fd_threshold = 6#6.3 #(6.5)
Ff_threshold = 0.9 #(1)
Fc_threshold = 0.8 #(0.8)
timeout = time.time() + 60*5 # 5 minutes from now
while True:
# Create an Enforce object and add constraints
ef = Enforce(TNTpseudM)
ef.add_constraint(MaxRep,cols=[TNTpseudM.T_ID],maxrep=3)
ef.add_constraint(MaxRep,cols=[TNTpseudM.NT_ID],maxrep=3)
ef.add_constraint(MaxRep,cols=[TNTpseudM.neu_ID],maxrep=3)
ef.add_constraint(MaxRep,cols=[TNTpseudM.neg_ID],maxrep=3)
ef.add_constraint(MinDist,cols=[TNTpseudM.null_ID],mindist=2)
# Enforce the constraints
TNTpseudM = ef.enforce()
# convert new datamatrix to a list as input for DES1
TNTorder = list(TNTpseudM.order)
TNTdES = neurodesign.classes.design(
order = TNTorder,
ITI = generate.iti(ntrials=124, model="exponential", min=1.4, mean=2, max=2.6,
def from_string(self, string): """See item.""" self.var.clear() self.comments = [] self.reset() if string is None: return for i in string.split(u'\n'): self.parse_variable(i) cmd, arglist, kwdict = self.syntax.parse_cmd(i) if cmd == u'run': if len(arglist) != 1 or kwdict: raise osexception(u'Invalid run command: %s' % i) self._item = arglist[0] continue if cmd == u'setcycle': if self.ef is not None or self.operations: raise osexception( u'setcycle must come before constraints and operations') if len(arglist) != 3 or kwdict: raise osexception(u'Invalid setcycle command: %s' % i) row, var, val = tuple(arglist) if row >= len(self.dm): self.dm.length = row+1 if var not in self.dm: self.dm[var] = u'' self.dm[row][var] = val continue if cmd == u'constrain': if self.operations: raise osexception( u'constraints must come before operations') if self.ef is None: self.ef = Enforce(self.dm) if len(arglist) != 1: raise osexception(u'Invalid constrain command: %s' % i) colname = arglist[0] try: col = self.dm[colname] except: raise osexception(u'Invalid column name: %s' % colname) for constraint, value in kwdict.items(): try: if constraint == u'maxrep': self.ef.add_constraint(MaxRep, cols=[col], maxrep=value) continue if constraint == u'mindist': self.ef.add_constraint(MinDist, cols=[col], mindist=value) continue except InvalidConstraint as e: raise osexception(e) raise osexception(u'Invalid constrain command: %s' % i) continue if cmd in self.commands: self.operations.append( (cmd, arglist) ) if len(self.dm) == 0: self.dm.length = 1 if len(self.dm.columns) == 0: self.dm.empty_column = u'' # Backwards compatibility: Older version of OpenSesame can specify the # number of cycles through the cycles variable. If the specified number # of cycles doesn't match the length of the datamatrix, we change the # length of the datamatrix. if u'cycles' in self.var and isinstance(self.var.cycles, int) \ and self.var.cycles != len(self.dm): self.dm.length = self.var.cycles
class loop(item.item):
"""A loop item runs a single other item multiple times"""
description = u'Repeatedly runs another item'
valid_orders = u'sequential', u'random'
commands = [
u'fullfactorial',
u'shuffle',
u'shuffle_horiz',
u'slice',
u'sort',
u'sortby',
u'reverse',
u'roll',
u'weight',
]
def reset(self):
"""See item."""
self.ef = None
self.dm = DataMatrix(length=0)
self.live_dm = None
self.live_row = None
self.operations = []
self._item = u''
self.var.repeat = 1
self.var.continuous = u'no'
self.var.order = u'random'
self.var.break_if = u'never'
self.var.break_if_on_first = u'yes'
self.var.source = u'table' # file or table
self.var.source_file = u''
def from_string(self, string):
"""See item."""
self.var.clear()
self.comments = []
self.reset()
if string is None:
return
for i in string.split(u'\n'):
self.parse_variable(i)
cmd, arglist, kwdict = self.syntax.parse_cmd(i)
if cmd == u'run':
if len(arglist) != 1 or kwdict:
raise osexception(u'Invalid run command: %s' % i)
self._item = arglist[0]
continue
if cmd == u'setcycle':
if self.ef is not None or self.operations:
raise osexception(
u'setcycle must come before constraints and operations')
if len(arglist) != 3 or kwdict:
raise osexception(u'Invalid setcycle command: %s' % i)
row, var, val = tuple(arglist)
if row >= len(self.dm):
self.dm.length = row+1
if var not in self.dm:
self.dm[var] = u''
self.dm[row][var] = val
continue
if cmd == u'constrain':
if self.operations:
raise osexception(
u'constraints must come before operations')
if self.ef is None:
self.ef = Enforce(self.dm)
if len(arglist) != 1:
raise osexception(u'Invalid constrain command: %s' % i)
colname = arglist[0]
try:
col = self.dm[colname]
except:
raise osexception(u'Invalid column name: %s' % colname)
for constraint, value in kwdict.items():
try:
if constraint == u'maxrep':
self.ef.add_constraint(MaxRep, cols=[col],
maxrep=value)
continue
if constraint == u'mindist':
self.ef.add_constraint(MinDist, cols=[col],
mindist=value)
continue
except InvalidConstraint as e:
raise osexception(e)
raise osexception(u'Invalid constrain command: %s' % i)
continue
if cmd in self.commands:
self.operations.append( (cmd, arglist) )
if len(self.dm) == 0:
self.dm.length = 1
if len(self.dm.columns) == 0:
self.dm.empty_column = u''
# Backwards compatibility: Older version of OpenSesame can specify the
# number of cycles through the cycles variable. If the specified number
# of cycles doesn't match the length of the datamatrix, we change the
# length of the datamatrix.
if u'cycles' in self.var and isinstance(self.var.cycles, int) \
and self.var.cycles != len(self.dm):
self.dm.length = self.var.cycles
def to_string(self):
"""See item."""
# Older versions of OpenSesame use an explicit cycles variable, so we
# set this here for backwards compatibility.
self.var.cycles = len(self.dm)
s = item.item.to_string(self)
for i, row in enumerate(self.dm):
for name, val in row:
s += u'\t%s\n' % \
self.syntax.create_cmd(u'setcycle', [i, name, val])
if self.ef is not None:
d = {}
for constraint in self.ef.constraints:
col = constraint.cols[0]
if col not in d:
d[col] = {}
if isinstance(constraint, MaxRep):
d[col][u'maxrep'] = constraint.maxrep
elif isinstance(constraint, MinDist):
d[col][u'mindist'] = constraint.mindist
for col, kwdict in d.items():
s += u'\t%s\n' % self.syntax.create_cmd(u'constrain', [col],
kwdict)
for cmd, arglist in self.operations:
s += u'\t%s\n' % self.syntax.create_cmd(cmd, arglist)
s += u'\t%s\n' % self.syntax.create_cmd(u'run', [self._item])
return s
def _require_arglist(self, cmd, arglist, minlen=1):
"""
desc:
Checks whether a non-empty argument list has been specified, and
raises an Exception otherwise.
arguments:
cmd:
desc: The command to check for.
type: str
arglist:
desc: The argument list to check.
type: list
"""
if len(arglist) < minlen:
raise osexception(u'Invalid argument list for %s' % cmd)
def _create_live_datamatrix(self):
"""
desc:
Builds a live DataMatrix. That is, it takes the orignal DataMatrix
and applies all the operations as specified.
returns:
desc: A live DataMatrix.
type: DataMatrix
"""
if self.var.source == u'table':
src_dm = self.dm
else:
from datamatrix import io
src = self.experiment.pool[self.var.source_file]
if src.endswith(u'.xlsx'):
try:
src_dm = io.readxlsx(src)
except Exception as e:
raise osexception(u'Failed to read .xlsx file: %s' % src,
exception=e)
else:
try:
src_dm = io.readtxt(src)
except Exception as e:
raise osexception(u'Failed to read text file (perhaps it has the wrong format or it is not utf-8 encoded): %s' % src,
exception=e)
for column_name in src_dm.column_names:
if not self.syntax.valid_var_name(column_name):
raise osexception(
u'The loop table contains an invalid column name: 'u'\'%s\'' \
% column_name)
# The number of repeats should be numeric. If not, then give an error.
# This can also occur when generating a preview of a loop table if
# repeat is variable.
if not isinstance(self.var.repeat, (int, float)):
raise osexception(
u'Don\'t know how to generate a DataMatrix for "%s" repeats' \
% self.var.repeat)
length = int(len(src_dm) * self.var.repeat)
dm = DataMatrix(length=0)
while len(dm) < length:
i = min(length-len(dm), len(src_dm))
if self.var.order == u'random':
dm <<= operations.shuffle(src_dm)[:i]
else:
dm <<= src_dm[:i]
if self.var.order == u'random':
dm = operations.shuffle(dm)
if self.ef is not None:
self.ef.dm = dm
dm = self.ef.enforce()
for cmd, arglist in self.operations:
# The column name is always specified last, or not at all
if arglist:
try:
colname = arglist[-1]
col = dm[colname]
except:
raise osexception(
u'Column %s does not exist' % arglist[-1])
if cmd == u'fullfactorial':
dm = operations.fullfactorial(dm)
elif cmd == u'shuffle':
if not arglist:
dm = operations.shuffle(dm)
else:
dm[colname] = operations.shuffle(col)
elif cmd == u'shuffle_horiz':
if not arglist:
dm = operations.shuffle_horiz(dm)
else:
dm = operations.shuffle_horiz(
*[dm[_colname] for _colname in arglist])
elif cmd == u'slice':
self._require_arglist(cmd, arglist, minlen=2)
dm = dm[arglist[0]: arglist[1]]
elif cmd == u'sort':
self._require_arglist(cmd, arglist)
dm[colname] = operations.sort(col)
elif cmd == u'sortby':
self._require_arglist(cmd, arglist)
dm = operations.sort(dm, by=col)
elif cmd == u'reverse':
if not arglist:
dm = dm[::-1]
else:
dm[colname] = col[::-1]
elif cmd == u'roll':
self._require_arglist(cmd, arglist)
steps = arglist[0]
if not isinstance(steps, int):
raise osexception(u'roll steps should be numeric')
if len(arglist) == 1:
dm = dm[-steps:] << dm[:-steps]
else:
dm[colname] = list(col[-steps:]) + list(col[:-steps])
elif cmd == u'weight':
self._require_arglist(cmd, arglist)
dm = operations.weight(col)
return dm
def prepare(self):
"""See item."""
item.item.prepare(self)
# Deprecation errors. The direct reference to __vars__ prevents a lookup
# in the parent var store.
if (u'skip' in self.var.__vars__ and self.var.skip != 0) \
or (u'offset' in self.var.__vars__ and self.var.offset != u'no'):
raise osexception(
u'The skip and offset options have been removed. Please refer to the documentation of the loop item for more information.')
# Compile break-if statement
break_if = self.var.get(u'break_if', _eval=False)
if break_if not in (u'never', u''):
self._break_if = self.syntax.compile_cond(break_if)
else:
self._break_if = None
# Create a keyboard to flush responses between cycles
self._keyboard = openexp.keyboard.keyboard(self.experiment)
# Make sure the item to run exists
if self._item not in self.experiment.items:
raise osexception(
u"Could not find item '%s', which is called by loop item '%s'" \
% (self._item, self.name))
def run(self):
"""See item."""
self.set_item_onset()
if self.live_dm is None or self.var.continuous == u'no':
self.live_dm = self._create_live_datamatrix()
self.live_row = 0
first = True
while self.live_row < len(self.live_dm):
self.experiment.var.repeat_cycle = 0
self.experiment.var.live_row = self.live_row
self.experiment.var.set('live_row_%s' % self.name, self.live_row)
for name, val in self.live_dm[self.live_row]:
if isinstance(val, basestring) and val.startswith(u'='):
val = self.python_workspace._eval(val[1:])
self.experiment.var.set(name, val)
# Evaluate the run if statement
if self._break_if is not None and \
(not first or self.var.break_if_on_first == u'yes'):
self.python_workspace[u'self'] = self
if self.python_workspace._eval(self._break_if):
break
# Run the item!
self.experiment.items.execute(self._item)
# If the repeat_cycle flag was set, run the item again later
if self.experiment.var.repeat_cycle:
self.live_dm <<= self.live_dm[self.live_row:self.live_row+1]
if self.var.order == u'random':
self.live_dm = self.live_dm[:self.live_row+1] \
<< operations.shuffle(self.live_dm[self.live_row+1:])
self.live_row += 1
first = False
else:
# If the loop finished without breaking, it needs to be reset on
# the next run of the loop item
self.live_row = None
self.live_dm = None
def var_info(self):
"""See item."""
return item.item.var_info(self) + [ (colname, safe_decode(col)) \
for colname, col in self.dm.columns]
def from_string(self, string):
"""See item."""
self.var.clear()
self.comments = []
self.reset()
if string is None:
return
for i in string.split(u'\n'):
self.parse_variable(i)
cmd, arglist, kwdict = self.syntax.parse_cmd(i)
if cmd == u'run':
if len(arglist) != 1 or kwdict:
raise osexception(u'Invalid run command: %s' % i)
self._item = arglist[0]
continue
if cmd == u'setcycle':
if self.ef is not None or self.operations:
raise osexception(
u'setcycle must come before constraints and operations'
)
if len(arglist) != 3 or kwdict:
raise osexception(u'Invalid setcycle command: %s' % i)
row, var, val = tuple(arglist)
if row >= len(self.dm):
self.dm.length = row + 1
if var not in self.dm:
self.dm[var] = u''
self.dm[row][var] = val
continue
if cmd == u'constrain':
if self.operations:
raise osexception(
u'constraints must come before operations')
if self.ef is None:
self.ef = Enforce(self.dm)
if len(arglist) != 1:
raise osexception(u'Invalid constrain command: %s' % i)
colname = arglist[0]
try:
col = self.dm[colname]
except:
raise osexception(u'Invalid column name: %s' % colname)
for constraint, value in kwdict.items():
try:
if constraint == u'maxrep':
self.ef.add_constraint(MaxRep,
cols=[col],
maxrep=value)
continue
if constraint == u'mindist':
self.ef.add_constraint(MinDist,
cols=[col],
mindist=value)
continue
except InvalidConstraint as e:
raise osexception(e)
raise osexception(u'Invalid constrain command: %s' % i)
continue
if cmd in self.commands:
self.operations.append((cmd, arglist))
if len(self.dm) == 0:
self.dm.length = 1
if len(self.dm.columns) == 0:
self.dm.empty_column = u''
# Backwards compatibility: Older version of OpenSesame can specify the
# number of cycles through the cycles variable. If the specified number
# of cycles doesn't match the length of the datamatrix, we change the
# length of the datamatrix.
if u'cycles' in self.var and isinstance(self.var.cycles, int) \
and self.var.cycles != len(self.dm):
self.dm.length = self.var.cycles
class loop(item.item):
"""A loop item runs a single other item multiple times"""
description = u'Repeatedly runs another item'
valid_orders = u'sequential', u'random'
commands = [
u'fullfactorial',
u'shuffle',
u'shuffle_horiz',
u'slice',
u'sort',
u'sortby',
u'reverse',
u'roll',
u'weight',
]
def reset(self):
"""See item."""
self.ef = None
self.dm = DataMatrix(length=0)
self.dm.sorted = False
self.live_dm = None
self.live_row = None
self.operations = []
self._item = u''
self.var.repeat = 1
self.var.continuous = u'no'
self.var.order = u'random'
self.var.break_if = u'never'
self.var.break_if_on_first = u'yes'
self.var.source = u'table' # file or table
self.var.source_file = u''
def from_string(self, string):
"""See item."""
self.var.clear()
self.comments = []
self.reset()
if string is None:
return
for i in string.split(u'\n'):
self.parse_variable(i)
cmd, arglist, kwdict = self.syntax.parse_cmd(i)
if cmd == u'run':
if len(arglist) != 1 or kwdict:
raise osexception(u'Invalid run command: %s' % i)
self._item = arglist[0]
continue
if cmd == u'setcycle':
if self.ef is not None or self.operations:
raise osexception(
u'setcycle must come before constraints and operations'
)
if len(arglist) != 3 or kwdict:
raise osexception(u'Invalid setcycle command: %s' % i)
row, var, val = tuple(arglist)
if row >= len(self.dm):
self.dm.length = row + 1
if var not in self.dm:
self.dm[var] = u''
self.dm[row][var] = val
continue
if cmd == u'constrain':
if self.operations:
raise osexception(
u'constraints must come before operations')
if self.ef is None:
self.ef = Enforce(self.dm)
if len(arglist) != 1:
raise osexception(u'Invalid constrain command: %s' % i)
colname = arglist[0]
try:
col = self.dm[colname]
except:
raise osexception(u'Invalid column name: %s' % colname)
for constraint, value in kwdict.items():
try:
if constraint == u'maxrep':
self.ef.add_constraint(MaxRep,
cols=[col],
maxrep=value)
continue
if constraint == u'mindist':
self.ef.add_constraint(MinDist,
cols=[col],
mindist=value)
continue
except InvalidConstraint as e:
raise osexception(e)
raise osexception(u'Invalid constrain command: %s' % i)
continue
if cmd in self.commands:
self.operations.append((cmd, arglist))
if len(self.dm) == 0:
self.dm.length = 1
if len(self.dm.columns) == 0:
self.dm.empty_column = u''
# Backwards compatibility: Older version of OpenSesame can specify the
# number of cycles through the cycles variable. If the specified number
# of cycles doesn't match the length of the datamatrix, we change the
# length of the datamatrix.
if u'cycles' in self.var and isinstance(self.var.cycles, int) \
and self.var.cycles != len(self.dm):
self.dm.length = self.var.cycles
def to_string(self):
"""See item."""
# Older versions of OpenSesame use an explicit cycles variable, so we
# set this here for backwards compatibility.
self.var.cycles = len(self.dm)
s = item.item.to_string(self)
for i, row in enumerate(self.dm):
for name, val in row:
s += u'\t%s\n' % \
self.syntax.create_cmd(u'setcycle', [i, name, val])
if self.ef is not None:
d = {}
for constraint in self.ef.constraints:
col = constraint.cols[0]
if col not in d:
d[col] = {}
if isinstance(constraint, MaxRep):
d[col][u'maxrep'] = constraint.maxrep
elif isinstance(constraint, MinDist):
d[col][u'mindist'] = constraint.mindist
for col, kwdict in d.items():
s += u'\t%s\n' % self.syntax.create_cmd(
u'constrain', [col], kwdict)
for cmd, arglist in self.operations:
s += u'\t%s\n' % self.syntax.create_cmd(cmd, arglist)
s += u'\t%s\n' % self.syntax.create_cmd(u'run', [self._item])
return s
def _require_arglist(self, cmd, arglist, minlen=1):
"""
desc:
Checks whether a non-empty argument list has been specified, and
raises an Exception otherwise.
arguments:
cmd:
desc: The command to check for.
type: str
arglist:
desc: The argument list to check.
type: list
"""
if len(arglist) < minlen:
raise osexception(u'Invalid argument list for %s' % cmd)
def _create_live_datamatrix(self):
"""
desc:
Builds a live DataMatrix. That is, it takes the orignal DataMatrix
and applies all the operations as specified.
returns:
desc: A live DataMatrix.
type: DataMatrix
"""
if self.var.source == u'table':
src_dm = self.dm
else:
from datamatrix import io
src = self.experiment.pool[self.var.source_file]
if src.endswith(u'.xlsx'):
try:
src_dm = io.readxlsx(src)
except Exception as e:
raise osexception(u'Failed to read .xlsx file: %s' % src,
exception=e)
else:
try:
src_dm = io.readtxt(src)
except Exception as e:
raise osexception(
u'Failed to read text file (perhaps it has the wrong format or it is not utf-8 encoded): %s'
% src,
exception=e)
for column_name in src_dm.column_names:
if not self.syntax.valid_var_name(column_name):
raise osexception(
u'The loop table contains an invalid column name: 'u'\'%s\'' \
% column_name)
# The number of repeats should be numeric. If not, then give an error.
# This can also occur when generating a preview of a loop table if
# repeat is variable.
if not isinstance(self.var.repeat, (int, float)):
raise osexception(
u'Don\'t know how to generate a DataMatrix for "%s" repeats' \
% self.var.repeat)
length = int(len(src_dm) * self.var.repeat)
dm = DataMatrix(length=0)
while len(dm) < length:
i = min(length - len(dm), len(src_dm))
if self.var.order == u'random':
dm <<= operations.shuffle(src_dm)[:i]
else:
dm <<= src_dm[:i]
if self.var.order == u'random':
dm = operations.shuffle(dm)
if self.ef is not None:
self.ef.dm = dm
dm = self.ef.enforce()
for cmd, arglist in self.operations:
# The column name is always specified last, or not at all
if arglist:
try:
colname = arglist[-1]
col = dm[colname]
except:
raise osexception(u'Column %s does not exist' %
arglist[-1])
if cmd == u'fullfactorial':
dm = operations.fullfactorial(dm)
elif cmd == u'shuffle':
if not arglist:
dm = operations.shuffle(dm)
else:
dm[colname] = operations.shuffle(col)
elif cmd == u'shuffle_horiz':
if not arglist:
dm = operations.shuffle_horiz(dm)
else:
dm = operations.shuffle_horiz(
*[dm[_colname] for _colname in arglist])
elif cmd == u'slice':
self._require_arglist(cmd, arglist, minlen=2)
dm = dm[arglist[0]:arglist[1]]
elif cmd == u'sort':
self._require_arglist(cmd, arglist)
dm[colname] = operations.sort(col)
elif cmd == u'sortby':
self._require_arglist(cmd, arglist)
dm = operations.sort(dm, by=col)
elif cmd == u'reverse':
if not arglist:
dm = dm[::-1]
else:
dm[colname] = col[::-1]
elif cmd == u'roll':
self._require_arglist(cmd, arglist)
steps = arglist[0]
if not isinstance(steps, int):
raise osexception(u'roll steps should be numeric')
if len(arglist) == 1:
dm = dm[-steps:] << dm[:-steps]
else:
dm[colname] = list(col[-steps:]) + list(col[:-steps])
elif cmd == u'weight':
self._require_arglist(cmd, arglist)
dm = operations.weight(col)
return dm
def prepare(self):
"""See item."""
item.item.prepare(self)
# Deprecation errors. The direct reference to __vars__ prevents a lookup
# in the parent var store.
if (u'skip' in self.var.__vars__ and self.var.skip != 0) \
or (u'offset' in self.var.__vars__ and self.var.offset != u'no'):
raise osexception(
u'The skip and offset options have been removed. Please refer to the documentation of the loop item for more information.'
)
# Compile break-if statement
break_if = self.var.get(u'break_if', _eval=False)
if break_if not in (u'never', u''):
self._break_if = self.syntax.compile_cond(break_if)
else:
self._break_if = None
# Create a keyboard to flush responses between cycles
self._keyboard = openexp.keyboard.keyboard(self.experiment)
# Make sure the item to run exists
if self._item not in self.experiment.items:
raise osexception(
u"Could not find item '%s', which is called by loop item '%s'" \
% (self._item, self.name))
def run(self):
"""See item."""
self.set_item_onset()
if self.live_dm is None or self.var.continuous == u'no':
self.live_dm = self._create_live_datamatrix()
self.live_row = 0
first = True
while self.live_row < len(self.live_dm):
self.experiment.var.repeat_cycle = 0
self.experiment.var.live_row = self.live_row
self.experiment.var.set('live_row_%s' % self.name, self.live_row)
for name, val in self.live_dm[self.live_row]:
if isinstance(val, basestring) and val.startswith(u'='):
val = self.python_workspace._eval(val[1:])
self.experiment.var.set(name, val)
# Evaluate the run if statement
if self._break_if is not None and \
(not first or self.var.break_if_on_first == u'yes'):
self.python_workspace[u'self'] = self
if self.python_workspace._eval(self._break_if):
break
# Run the item!
self.experiment.items.execute(self._item)
# If the repeat_cycle flag was set, run the item again later
if self.experiment.var.repeat_cycle:
self.live_dm <<= self.live_dm[self.live_row:self.live_row + 1]
if self.var.order == u'random':
self.live_dm = self.live_dm[:self.live_row+1] \
<< operations.shuffle(self.live_dm[self.live_row+1:])
self.live_row += 1
first = False
else:
# If the loop finished without breaking, it needs to be reset on
# the next run of the loop item
self.live_row = None
self.live_dm = None
def var_info(self):
"""See item."""
return item.item.var_info(self) + [ (colname, safe_decode(col)) \
for colname, col in self.dm.columns]
from datamatrix import io
from pseudorandom import Enforce, MaxRep, MinDist
# Read a datafile with Pandas and convert it to a pseudorandom DataFrame.
dm = io.readtxt('examples/data.csv')
print(dm)
# Create an Enforce object, and add two constraints
ef = Enforce(dm)
ef.add_constraint(MaxRep, cols=[dm.category], maxrep=1)
ef.add_constraint(MinDist, cols=[dm.word], mindist=4)
# Enforce the constraints
dm = ef.enforce()
# See the resulting DataFrame and a report of how long the enforcement took.
print(dm)
print(ef.report)
def from_string(self, string):
"""See item."""
self.variables = {}
self.comments = []
self.reset()
if string is None:
return
for i in string.split(u'\n'):
self.parse_variable(i)
cmd, arglist, kwdict = self.syntax.parse_cmd(i)
if cmd == u'run':
if len(arglist) != 1 or kwdict:
raise osexception(u'Invalid run command: %s' % i)
self._item = arglist[0]
continue
if cmd == u'setcycle':
if self.ef is not None or self.operations:
raise osexception(
u'setcycle must come before constraints and operations')
if len(arglist) != 3 or kwdict:
raise osexception(u'Invalid setcycle command: %s' % i)
row, var, val = tuple(arglist)
if row >= len(self.dm):
self.dm.length = row+1
if var not in self.dm:
self.dm[var] = u''
self.dm[row][var] = val
continue
if cmd == u'constrain':
if self.operations:
raise osexception(
u'constraints must come before operations')
if self.ef is None:
self.ef = Enforce(self.dm)
if len(arglist) != 1:
raise osexception(u'Invalid constrain command: %s' % i)
colname = arglist[0]
try:
col = self.dm[colname]
except:
raise osexception(u'Invalid column name: %s' % colname)
for constraint, value in kwdict.items():
try:
if constraint == u'maxrep':
self.ef.add_constraint(MaxRep, cols=[col],
maxrep=value)
continue
if constraint == u'mindist':
self.ef.add_constraint(MinDist, cols=[col],
mindist=value)
continue
except InvalidConstraint as e:
raise osexception(e)
raise osexception(u'Invalid constrain command: %s' % i)
continue
if cmd in self.commands:
self.operations.append( (cmd, arglist) )
if len(self.dm) == 0:
self.dm.length = 1
if len(self.dm.columns) == 0:
self.dm.empty_column = u''
def from_string(self, string):
"""See item."""
self.var.clear()
self.comments = []
self.reset()
if string is None:
return
for i in string.split(u'\n'):
self.parse_variable(i)
cmd, arglist, kwdict = self.syntax.parse_cmd(i)
if cmd == u'run':
if len(arglist) != 1 or kwdict:
raise osexception(u'Invalid run command: %s' % i)
self._item = arglist[0]
continue
if cmd == u'setcycle':
if self.ef is not None or self.operations:
raise osexception(
u'setcycle must come before constraints and operations'
)
if len(arglist) != 3 or kwdict:
raise osexception(u'Invalid setcycle command: %s' % i)
row, var, val = tuple(arglist)
if row >= len(self.dm):
self.dm.length = row + 1
if var not in self.dm:
self.dm[var] = u''
self.dm[row][var] = val
continue
if cmd == u'constrain':
if self.operations:
raise osexception(
u'constraints must come before operations')
if self.ef is None:
self.ef = Enforce(self.dm)
if len(arglist) != 1:
raise osexception(u'Invalid constrain command: %s' % i)
colname = arglist[0]
try:
col = self.dm[colname]
except:
raise osexception(u'Invalid column name: %s' % colname)
for constraint, value in kwdict.items():
try:
if constraint == u'maxrep':
self.ef.add_constraint(MaxRep,
cols=[col],
maxrep=value)
continue
if constraint == u'mindist':
self.ef.add_constraint(MinDist,
cols=[col],
mindist=value)
continue
except InvalidConstraint as e:
raise osexception(e)
raise osexception(u'Invalid constrain command: %s' % i)
continue
if cmd in self.commands:
self.operations.append((cmd, arglist))
if len(self.dm) == 0:
self.dm.length = 1
if len(self.dm.columns) == 0:
self.dm.empty_column = u''
def _create_live_datamatrix(self):
"""
desc:
Builds a live DataMatrix. That is, it takes the orignal DataMatrix
and applies all the operations as specified.
returns:
desc: A live DataMatrix.
type: DataMatrix
"""
src_dm = self.dm if self.var.source == u'table' else self._read_file()
for column_name in src_dm.column_names:
if not self.syntax.valid_var_name(column_name):
raise osexception(
u'The loop table contains an invalid column name: '
u'\'%s\'' % column_name)
# The number of repeats should be numeric. If not, then give an error.
# This can also occur when generating a preview of a loop table if
# repeat is variable.
if not isinstance(self.var.repeat, (int, float)):
raise osexception(
u'Don\'t know how to generate a DataMatrix for "%s" repeats' %
self.var.repeat)
length = int(len(src_dm) * self.var.repeat)
dm = DataMatrix(length=0)
while len(dm) < length:
i = min(length - len(dm), len(src_dm))
if self.var.order == u'random':
dm <<= operations.shuffle(src_dm)[:i]
else:
dm <<= src_dm[:i]
if self.var.order == u'random':
dm = operations.shuffle(dm)
# Constraints come before loop operations
if self._constraints:
self.ef = Enforce(dm)
for constraint_cls, colname, kwargs in self._constraints:
self.ef.add_constraint(constraint_cls,
cols=dm[colname],
**kwargs)
dm = self.ef.enforce()
# Operations come last
for cmd, arglist in self._operations:
# The column name is always specified last, or not at all
if arglist:
try:
colname = arglist[-1]
col = dm[colname]
except:
raise osexception(u'Column %s does not exist' %
arglist[-1])
if cmd == u'fullfactorial':
dm = operations.fullfactorial(dm)
elif cmd == u'shuffle':
if not arglist:
dm = operations.shuffle(dm)
else:
dm[colname] = operations.shuffle(col)
elif cmd == u'shuffle_horiz':
if not arglist:
dm = operations.shuffle_horiz(dm)
else:
# There can be multiple column names, so we need to check
# if all of them exist, rather than only the last one as
# we did above.
for _colname in arglist:
try:
dm[_colname]
except:
raise osexception(u'Column %s does not exist' %
_colname)
dm = operations.shuffle_horiz(
*[dm[_colname] for _colname in arglist])
elif cmd == u'slice':
self._require_arglist(cmd, arglist, minlen=2)
dm = dm[arglist[0]:arglist[1]]
elif cmd == u'sort':
self._require_arglist(cmd, arglist)
dm[colname] = operations.sort(col)
elif cmd == u'sortby':
self._require_arglist(cmd, arglist)
dm = operations.sort(dm, by=col)
elif cmd == u'reverse':
if not arglist:
dm = dm[::-1]
else:
dm[colname] = col[::-1]
elif cmd == u'roll':
self._require_arglist(cmd, arglist)
steps = arglist[0]
if not isinstance(steps, int):
raise osexception(u'roll steps should be numeric')
if len(arglist) == 1:
dm = dm[-steps:] << dm[:-steps]
else:
dm[colname] = list(col[-steps:]) + list(col[:-steps])
elif cmd == u'weight':
self._require_arglist(cmd, arglist)
try:
dm = operations.weight(col)
except TypeError:
raise osexception(
u'weight values should be non-negative numeric values')
return dm