def test_io():
refdm = DataMatrix(length=3)
refdm[u'tést'] = 1, 2, u''
refdm.B = u'mathôt', u'b', u'x'
refdm.C = u'a,\\b"\'c', 8, u''
testdm = io.readtxt('testcases/data/data.csv')
check_dm(refdm, testdm)
io.writetxt(testdm, 'tmp.csv')
testdm = io.readtxt('tmp.csv')
check_dm(refdm, testdm)
refdm = io.readtxt('testcases/data/line-ending-cr.csv')
check_dm(refdm, testdm)
refdm = io.readtxt('testcases/data/line-ending-crlf.csv')
check_dm(refdm, testdm)
refdm = io.readtxt('testcases/data/data-with-bom.csv')
check_dm(refdm, testdm)
io.writepickle(testdm, 'tmp.pickle')
testdm = io.readpickle('tmp.pickle')
check_dm(refdm, testdm)
io.writexlsx(testdm, 'tmp.xlsx')
with pytest.warns(UserWarning): # Not all rows have column C
testdm = io.readxlsx('tmp.xlsx')
check_dm(refdm, testdm)
io.writexlsx(testdm, 'tmp.xlsx')
with pytest.warns(UserWarning): # Not all rows have column C
testdm = io.readxlsx('tmp.xlsx')
check_dm(refdm, testdm)
def test_io():
refdm = DataMatrix(length=3)
refdm[u'tést'] = 1, 2, u''
refdm.B = u'mathôt', u'b', u'x'
refdm.C = u'a,\\b"\'c', 8, u''
testdm = io.readtxt('testcases/data/data.csv')
check_dm(refdm, testdm)
io.writetxt(testdm, 'tmp.csv')
testdm = io.readtxt('tmp.csv')
check_dm(refdm, testdm)
refdm = io.readtxt('testcases/data/line-ending-cr.csv')
check_dm(refdm, testdm)
refdm = io.readtxt('testcases/data/line-ending-crlf.csv')
check_dm(refdm, testdm)
io.writepickle(testdm, 'tmp.pickle')
testdm = io.readpickle('tmp.pickle')
check_dm(refdm, testdm)
io.writexlsx(testdm, 'tmp.xlsx')
testdm = io.readxlsx('tmp.xlsx')
check_dm(refdm, testdm)
def test_io():
refdm = DataMatrix(length=3)
refdm[u'tést'] = 1, 2, u''
refdm.B = u'mathôt', u'b', u'x'
refdm.C = u'a,\\b"\'c', 8, u''
testdm = io.readtxt('testcases/data/data.csv')
check_dm(refdm, testdm)
io.writetxt(testdm, 'tmp.csv')
testdm = io.readtxt('tmp.csv')
check_dm(refdm, testdm)
io.writepickle(testdm, 'tmp.pickle')
testdm = io.readpickle('tmp.pickle')
check_dm(refdm, testdm)
io.writexlsx(testdm, 'tmp.xlsx')
testdm = io.readxlsx('tmp.xlsx')
check_dm(refdm, testdm)
def _read_file(self):
"""
desc:
Reads a source file and raises an osexception if this fails.
returns:
type: DataMatrix
"""
from datamatrix import io
src = self.experiment.pool[self.var.source_file]
if src.endswith(u'.xlsx'):
try:
return io.readxlsx(src)
except Exception as e:
raise osexception(u'Failed to read .xlsx file: %s' % src,
exception=e)
try:
return io.readtxt(src)
except Exception as e:
raise osexception(
(u'Failed to read text file (perhaps it has the '
u'wrong format or it is not utf-8 encoded): %s') % src,
exception=e)
def _create_live_datamatrix(self): """ desc: Builds a live DataMatrix. That is, it takes the orignal DataMatrix and applies all the operations as specified. returns: desc: A live DataMatrix. type: DataMatrix """ if self.var.source == u'table': src_dm = self.dm else: from datamatrix import io src = self.experiment.pool[self.var.source_file] if src.endswith(u'.xlsx'): try: src_dm = io.readxlsx(src) except Exception as e: raise osexception(u'Failed to read .xlsx file: %s' % src, exception=e) else: try: src_dm = io.readtxt(src) except Exception as e: raise osexception(u'Failed to read text file (perhaps it has the wrong format or it is not utf-8 encoded): %s' % src, exception=e) for column_name in src_dm.column_names: if not self.syntax.valid_var_name(column_name): raise osexception( u'The loop table contains an invalid column name: 'u'\'%s\'' \ % column_name) # The number of repeats should be numeric. If not, then give an error. # This can also occur when generating a preview of a loop table if # repeat is variable. if not isinstance(self.var.repeat, (int, float)): raise osexception( u'Don\'t know how to generate a DataMatrix for "%s" repeats' \ % self.var.repeat) length = int(len(src_dm) * self.var.repeat) dm = DataMatrix(length=0) while len(dm) < length: i = min(length-len(dm), len(src_dm)) if self.var.order == u'random': dm <<= operations.shuffle(src_dm)[:i] else: dm <<= src_dm[:i] if self.var.order == u'random': dm = operations.shuffle(dm) if self.ef is not None: self.ef.dm = dm dm = self.ef.enforce() for cmd, arglist in self.operations: # The column name is always specified last, or not at all if arglist: try: colname = arglist[-1] col = dm[colname] except: raise osexception( u'Column %s does not exist' % arglist[-1]) if cmd == u'fullfactorial': dm = operations.fullfactorial(dm) elif cmd == u'shuffle': if not arglist: dm = operations.shuffle(dm) else: dm[colname] = operations.shuffle(col) elif cmd == u'shuffle_horiz': if not arglist: dm = operations.shuffle_horiz(dm) else: dm = operations.shuffle_horiz( *[dm[_colname] for _colname in arglist]) elif cmd == u'slice': self._require_arglist(cmd, arglist, minlen=2) dm = dm[arglist[0]: arglist[1]] elif cmd == u'sort': self._require_arglist(cmd, arglist) dm[colname] = operations.sort(col) elif cmd == u'sortby': self._require_arglist(cmd, arglist) dm = operations.sort(dm, by=col) elif cmd == u'reverse': if not arglist: dm = dm[::-1] else: dm[colname] = col[::-1] elif cmd == u'roll': self._require_arglist(cmd, arglist) steps = arglist[0] if not isinstance(steps, int): raise osexception(u'roll steps should be numeric') if len(arglist) == 1: dm = dm[-steps:] << dm[:-steps] else: dm[colname] = list(col[-steps:]) + list(col[:-steps]) elif cmd == u'weight': self._require_arglist(cmd, arglist) dm = operations.weight(col) return dm
def _create_live_datamatrix(self):
"""
desc:
Builds a live DataMatrix. That is, it takes the orignal DataMatrix
and applies all the operations as specified.
returns:
desc: A live DataMatrix.
type: DataMatrix
"""
if self.var.source == u'table':
src_dm = self.dm
else:
from datamatrix import io
src = self.experiment.pool[self.var.source_file]
if src.endswith(u'.xlsx'):
try:
src_dm = io.readxlsx(src)
except Exception as e:
raise osexception(u'Failed to read .xlsx file: %s' % src,
exception=e)
else:
try:
src_dm = io.readtxt(src)
except Exception as e:
raise osexception(
u'Failed to read text file (perhaps it has the wrong format or it is not utf-8 encoded): %s'
% src,
exception=e)
for column_name in src_dm.column_names:
if not self.syntax.valid_var_name(column_name):
raise osexception(
u'The loop table contains an invalid column name: 'u'\'%s\'' \
% column_name)
# The number of repeats should be numeric. If not, then give an error.
# This can also occur when generating a preview of a loop table if
# repeat is variable.
if not isinstance(self.var.repeat, (int, float)):
raise osexception(
u'Don\'t know how to generate a DataMatrix for "%s" repeats' \
% self.var.repeat)
length = int(len(src_dm) * self.var.repeat)
dm = DataMatrix(length=0)
while len(dm) < length:
i = min(length - len(dm), len(src_dm))
if self.var.order == u'random':
dm <<= operations.shuffle(src_dm)[:i]
else:
dm <<= src_dm[:i]
if self.var.order == u'random':
dm = operations.shuffle(dm)
if self.ef is not None:
self.ef.dm = dm
dm = self.ef.enforce()
for cmd, arglist in self.operations:
# The column name is always specified last, or not at all
if arglist:
try:
colname = arglist[-1]
col = dm[colname]
except:
raise osexception(u'Column %s does not exist' %
arglist[-1])
if cmd == u'fullfactorial':
dm = operations.fullfactorial(dm)
elif cmd == u'shuffle':
if not arglist:
dm = operations.shuffle(dm)
else:
dm[colname] = operations.shuffle(col)
elif cmd == u'shuffle_horiz':
if not arglist:
dm = operations.shuffle_horiz(dm)
else:
dm = operations.shuffle_horiz(
*[dm[_colname] for _colname in arglist])
elif cmd == u'slice':
self._require_arglist(cmd, arglist, minlen=2)
dm = dm[arglist[0]:arglist[1]]
elif cmd == u'sort':
self._require_arglist(cmd, arglist)
dm[colname] = operations.sort(col)
elif cmd == u'sortby':
self._require_arglist(cmd, arglist)
dm = operations.sort(dm, by=col)
elif cmd == u'reverse':
if not arglist:
dm = dm[::-1]
else:
dm[colname] = col[::-1]
elif cmd == u'roll':
self._require_arglist(cmd, arglist)
steps = arglist[0]
if not isinstance(steps, int):
raise osexception(u'roll steps should be numeric')
if len(arglist) == 1:
dm = dm[-steps:] << dm[:-steps]
else:
dm[colname] = list(col[-steps:]) + list(col[:-steps])
elif cmd == u'weight':
self._require_arglist(cmd, arglist)
dm = operations.weight(col)
return dm
def _create_live_datamatrix(self): """ desc: Builds a live DataMatrix. That is, it takes the orignal DataMatrix and applies all the operations as specified. returns: desc: A live DataMatrix. type: DataMatrix """ if self.var.source == u'table': src_dm = self.dm else: from datamatrix import io src = self.experiment.pool[self.var.source_file] if src.endswith(u'.xlsx'): try: src_dm = io.readxlsx(src) except Exception as e: raise osexception(u'Failed to read .xlsx file: %s' % src, exception=e) else: try: src_dm = io.readtxt(src) except Exception as e: raise osexception(u'Failed to read text file: %s' % src, exception=e) length = int(len(src_dm) * self.var.repeat) dm = DataMatrix(length=0) while len(dm) < length: i = min(length-len(dm), len(src_dm)) if self.var.order == u'random': dm <<= operations.shuffle(src_dm)[:i] else: dm <<= src_dm[:i] if self.var.order == u'random': dm = operations.shuffle(dm) if self.ef is not None: self.ef.dm = dm dm = self.ef.enforce() for cmd, arglist in self.operations: # The column name is always specified last, or not at all if arglist: try: colname = arglist[-1] col = dm[colname] except: raise osexception( u'Column %s does not exist' % arglist[-1]) if cmd == u'fullfactorial': dm = operations.fullfactorial(dm) elif cmd == u'shuffle': if not arglist: dm = operations.shuffle(dm) else: dm[colname] = operations.shuffle(col) elif cmd == u'shuffle_horiz': if not arglist: dm = operations.shuffle_horiz(dm) else: dm = operations.shuffle_horiz( *[dm[_colname] for _colname in arglist]) elif cmd == u'slice': self._require_arglist(cmd, arglist, minlen=2) dm = dm[arglist[0]: arglist[1]] elif cmd == u'sort': self._require_arglist(cmd, arglist) dm[colname] = operations.sort(col) elif cmd == u'sortby': self._require_arglist(cmd, arglist) dm = operations.sort(dm, by=col) elif cmd == u'reverse': if not arglist: dm = dm[::-1] else: dm[colname] = col[::-1] elif cmd == u'roll': self._require_arglist(cmd, arglist) steps = arglist[0] if not isinstance(steps, int): raise osexception(u'roll steps should be numeric') if len(arglist) == 1: dm = dm[-steps:] << dm[:-steps] else: dm[colname] = list(col[-steps:]) + list(col[:-steps]) elif cmd == u'weight': self._require_arglist(cmd, arglist) dm = operations.weight(col) return dm
def _create_live_datamatrix(self):
"""
desc:
Builds a live DataMatrix. That is, it takes the orignal DataMatrix
and applies all the operations as specified.
returns:
desc: A live DataMatrix.
type: DataMatrix
"""
if self.var.source == u'table':
src_dm = self.dm
else:
from datamatrix import io
src = self.experiment.pool[self.var.source_file]
if src.endswith(u'.xlsx'):
try:
src_dm = io.readxlsx(src)
except Exception as e:
raise osexception(u'Failed to read .xlsx file: %s' % src,
exception=e)
else:
try:
src_dm = io.readtxt(src)
except Exception as e:
raise osexception(u'Failed to read text file: %s' % src,
exception=e)
length = int(len(src_dm) * self.var.repeat)
dm = DataMatrix(length=0)
while len(dm) < length:
i = min(length - len(dm), len(src_dm))
if self.var.order == u'random':
dm <<= operations.shuffle(src_dm)[:i]
else:
dm <<= src_dm[:i]
if self.var.order == u'random':
dm = operations.shuffle(dm)
if self.ef is not None:
self.ef.dm = dm
dm = self.ef.enforce()
for cmd, arglist in self.operations:
# The column name is always specified last, or not at all
if arglist:
try:
colname = arglist[-1]
col = dm[colname]
except:
raise osexception(u'Column %s does not exist' %
arglist[-1])
if cmd == u'fullfactorial':
dm = operations.fullfactorial(dm)
elif cmd == u'shuffle':
if not arglist:
dm = operations.shuffle(dm)
else:
dm[colname] = operations.shuffle(col)
elif cmd == u'shuffle_horiz':
if not arglist:
dm = operations.shuffle_horiz(dm)
else:
dm = operations.shuffle_horiz(
*[dm[_colname] for _colname in arglist])
elif cmd == u'slice':
self._require_arglist(cmd, arglist, minlen=2)
dm = dm[arglist[0]:arglist[1]]
elif cmd == u'sort':
self._require_arglist(cmd, arglist)
dm[colname] = operations.sort(col)
elif cmd == u'sortby':
self._require_arglist(cmd, arglist)
dm = operations.sort(dm, by=col)
elif cmd == u'reverse':
if not arglist:
dm = dm[::-1]
else:
dm[colname] = col[::-1]
elif cmd == u'roll':
self._require_arglist(cmd, arglist)
steps = arglist[0]
if not isinstance(steps, int):
raise osexception(u'roll steps should be numeric')
if len(arglist) == 1:
dm = dm[-steps:] << dm[:-steps]
else:
dm[colname] = list(col[-steps:]) + list(col[:-steps])
elif cmd == u'weight':
self._require_arglist(cmd, arglist)
dm = operations.weight(col)
return dm
