def test_seriescolumn(): dm = DataMatrix(length=2) dm.col = SeriesColumn(depth=2) dm.col[0] = 1, 2 dm.col[1] = 3, 4 dm.col += 1 check_series(dm.col, [[2,3], [4,5]]) dm.col += 1, 2 check_series(dm.col, [[3,4], [6,7]]) dm.col -= 1 check_series(dm.col, [[2,3], [5,6]]) dm.col -= 1, 2 check_series(dm.col, [[1,2], [3,4]]) dm.col *= 2 check_series(dm.col, [[2,4], [6,8]]) dm.col *= 1.5, 3 check_series(dm.col, [[3,6], [18,24]]) dm.col /= 3 check_series(dm.col, [[1,2], [6,8]]) dm.col /= 1, 2 check_series(dm.col, [[1,2], [3,4]]) dm.col //= 1.5, 2.5 check_series(dm.col, [[0,1], [1,1]]) dm.col += np.array([ [0,0], [10, 10] ]) check_series(dm.col, [[0,1], [11,11]])
def check_select(col_type): dm = DataMatrix(length=2, default_col_type=col_type) dm.col = 1, 2 dm_ = dm.col < 2 check_col(dm_.col, [1]) dm_ = dm.col == 2 check_col(dm_.col, [2]) dm_ = (dm.col == 1) | (dm.col == 2) # or check_col(dm_.col, [1, 2]) dm_ = (dm.col == 1) & (dm.col == 2) # and check_col(dm_.col, []) dm_ = (dm.col == 1) ^ (dm.col == 2) # xor check_col(dm_.col, [1, 2]) # Pair-wise select by matching-length sequence dm_ = dm.col == (1, 3) check_col(dm_.col, [1]) # Check by set multimatching dm_ = dm.col == {2, 3, 4} check_col(dm_.col, [2]) dm_ = dm.col != {1, 3, 4} check_col(dm_.col, [2]) # Check by lambda comparison dm_ = dm.col == (lambda x: x == 2) check_col(dm_.col, [2]) dm_ = dm.col != (lambda x: x == 2) check_col(dm_.col, [1]) check_integrity(dm)
def _update(self, name, workspace_func): if (not hasattr(self, '_dock_widget') or not self._dock_widget.isVisible()): return self._dock_widget.setWindowTitle(_(u'Workspace ({})').format(name)) workspace = workspace_func() # If the workspace didn't reply, we try again in a second if workspace is None or workspace.get(u'no reply', False) is None: QTimer.singleShot(1000, lambda: self._update(name, workspace_func)) return # If the current kernel doesn't expose its workspace, indicate this if workspace.get(u'not supported', False) is None: dm = DataMatrix(length=0) dm.kernel_not_supported = -1 # Create a DataMatrix that exposes the workspace else: dm = DataMatrix(length=len(workspace)) dm.sorted = False dm.name = '' dm.value = '' dm.shape = '' dm.type = '' for row, (var, data) in zip(dm, workspace.items()): if data is None: oslogger.warning(u'invalid workspace data: {}'.format(var)) continue value, type_, shape = data row.value = value row.name = var if shape is not None: row.shape = repr(shape) row.type = type_ self._qdm.dm = dm self._qdm.refresh()
def test_io(): refdm = DataMatrix(length=3) refdm[u'tést'] = 1, 2, u'' refdm.B = u'mathôt', u'b', u'x' refdm.C = u'a,\\b"\'c', 8, u'' testdm = io.readtxt('testcases/data/data.csv') check_dm(refdm, testdm) io.writetxt(testdm, 'tmp.csv') testdm = io.readtxt('tmp.csv') check_dm(refdm, testdm) refdm = io.readtxt('testcases/data/line-ending-cr.csv') check_dm(refdm, testdm) refdm = io.readtxt('testcases/data/line-ending-crlf.csv') check_dm(refdm, testdm) io.writepickle(testdm, 'tmp.pickle') testdm = io.readpickle('tmp.pickle') check_dm(refdm, testdm) io.writexlsx(testdm, 'tmp.xlsx') testdm = io.readxlsx('tmp.xlsx') check_dm(refdm, testdm)
def test_z(): dm = DataMatrix(length=5) dm.a = range(-2,3) dm.z = ops.z(dm.a) for x, y in zip(dm.z, [-1.26, -0.63, 0, .63, 1.26]): assert(abs(x-y) < .1)
def test_z(): dm = DataMatrix(length=5) dm.a = range(-2, 3) dm.z = ops.z(dm.a) for x, y in zip(dm.z, [-1.26, -0.63, 0, .63, 1.26]): assert (abs(x - y) < .1)
def test_intcolumn(): _test_numericcolumn(IntColumn) _test_copying(IntColumn) # Test automatic conversion to int dm = DataMatrix(length=2) dm.col = IntColumn dm.col = 1.9, '2.9' check_col(dm.col, [1, 2]) # Test setting invalid values @raises(TypeError) def _(): dm.col[0] = 'x' _() @raises(TypeError) def _(): dm.col = 'x' _() @raises(TypeError) def _(): dm.col[:-1] = 'x' _() # Check dtype ok_(dm.col._seq.dtype == np.int64) check_integrity(dm)
def check_intcolumn_typing(): dm = DataMatrix(length=4, default_col_type=IntColumn) dm.f = 1.1, '1.8', 2, '2' ok_(all(isinstance(v, int) for v in dm.f)) @raises(TypeError) def _(): dm.inf = INF, -INF, 'inf', '-inf' _() @raises(TypeError) def _(): dm.nan = NAN, NAN, 'nan', 'nan' _() @raises(TypeError) def _(): dm.none = None, None, None, None _() @raises(TypeError) def _(): dm.s = 'alpha', 'beta', 'None', ' ' _() @raises(TypeError) def _(): dm.err = Exception, tuple, str, map _()
def combine(dm, l_neg_dist1, l_neg_dist2, l_neu_dist1, l_neu_dist2): """ Make a potential combination of distractors Arguments: dm --- a DataMatrix instance l_neg_dist1 --- list of potential first negative distractors l_neg_dist2 l_neg_dist1 l_neu_dist2 Returns: new_dm_exp """ # Create an empty data file which will eventually contain all experimental trial info new_dm_exp = DataMatrix() # Walk through the rows of the experimental dm for row in dm: # Retrieve trial info: target_scene = row.Scene target_emotion = row.Emotion trial_id = row.Trial_ID target_object = row.Object print("valence = ", target_emotion) #sys.exit() # Create an empty mini dm: trial_dm = DataMatrix(1) # Randomly select two possible distractors (from the same category as the target scene) if target_emotion == "neg": dist1 = random.choice(l_neg_dist1) dist2 = random.choice(l_neg_dist2) l_neg_dist1.remove(dist1) l_neg_dist2.remove(dist2) elif target_emotion == "neu": dist1 = random.choice(l_neu_dist1) dist2 = random.choice(l_neu_dist2) l_neu_dist1.remove(dist1) l_neu_dist2.remove(dist2) else: raise Exception("Unknown valence category") # Add info to the trial_dm: trial_dm["distractor_scene_1"] = dist1 trial_dm["distractor_scene_2"] = dist2 trial_dm["Scene"] = target_scene trial_dm["Emotion"] = target_emotion trial_dm["Trial_ID"] = trial_id trial_dm["Object"] = target_object # Merge the current trial to the big dm: new_dm_exp = new_dm_exp << trial_dm # After having looped through all rows, return the full dm: return new_dm_exp
def _test_numeric_properties(coltype, nan): dm = DataMatrix(length=4, default_col_type=coltype) dm.c = 1, 1, nan, 4 dm.d = [nan] * 4 assert dm.c.mean == 2 assert dm.c.median == 1 assert dm.c.std == np.std([1, 1, 4], ddof=1) assert dm.c.max == 4 assert dm.c.min == 1 assert dm.c.sum == 6 if coltype in (IntColumn, FloatColumn): with pytest.warns(RuntimeWarning): all_nan(dm.d.mean, nan) all_nan(dm.d.median, nan) all_nan(dm.d.std, nan) all_nan(dm.d.max, nan) all_nan(dm.d.min, nan) all_nan(dm.d.sum, nan) else: all_nan(dm.d.mean, nan) all_nan(dm.d.median, nan) all_nan(dm.d.std, nan) all_nan(dm.d.max, nan) all_nan(dm.d.min, nan) all_nan(dm.d.sum, nan)
def test_io(): refdm = DataMatrix(length=3) refdm[u'tést'] = 1, 2, u'' refdm.B = u'mathôt', u'b', u'x' refdm.C = u'a,\\b"\'c', 8, u'' testdm = io.readtxt('testcases/data/data.csv') check_dm(refdm, testdm) io.writetxt(testdm, 'tmp.csv') testdm = io.readtxt('tmp.csv') check_dm(refdm, testdm) refdm = io.readtxt('testcases/data/line-ending-cr.csv') check_dm(refdm, testdm) refdm = io.readtxt('testcases/data/line-ending-crlf.csv') check_dm(refdm, testdm) refdm = io.readtxt('testcases/data/data-with-bom.csv') check_dm(refdm, testdm) io.writepickle(testdm, 'tmp.pickle') testdm = io.readpickle('tmp.pickle') check_dm(refdm, testdm) io.writexlsx(testdm, 'tmp.xlsx') with pytest.warns(UserWarning): # Not all rows have column C testdm = io.readxlsx('tmp.xlsx') check_dm(refdm, testdm) io.writexlsx(testdm, 'tmp.xlsx') with pytest.warns(UserWarning): # Not all rows have column C testdm = io.readxlsx('tmp.xlsx') check_dm(refdm, testdm)
def check_mixedcolumn_sorting(): dm = DataMatrix(length=24) dm.c = [ 1, '1', 2, '2', 1.1, '1.1', 2.1, '2.1', INF, -INF, 'inf', '-inf', NAN, NAN, 'nan', 'nan', None, None, None, None, 'alpha', 'beta', 'None', '' ] dm.c = ops.shuffle(dm.c) dm = ops.sort(dm, by=dm.c) check_col(dm.c, [ -INF, -INF, 1, 1, 1.1, 1.1, 2, 2, 2.1, 2.1, INF, INF, '', 'None', 'alpha', 'beta', None, None, None, None, NAN, NAN, NAN, NAN, ])
def check_iteration(col_type): dm = DataMatrix(length=2, default_col_type=col_type) dm.col1 = 1, 2 dm.col2 = 3, 4 # Row iteration ref = [ [('col1', 1), ('col2', 3)], [('col1', 2), ('col2', 4)] ] for row, rowref in zip(dm, ref): assert(list(row) == rowref) # Column iteration ref = [ ('col1', [1,2]), ('col2', [3,4]) ] for (name, col), (ref_name, ref_col) in zip(dm.columns, ref): assert(name == ref_name) assert(list(col) == ref_col) # Cells within column iteration ref = [1,2] for val, ref_val in zip(dm.col1, ref): assert(val == ref_val) # Cells within row iteration ref = [ ('col1', 1), ('col2', 3) ] for (name, val), (ref_name, ref_val) in zip(dm[0], ref): assert(val == ref_val) assert(name == ref_name)
def test_fullfactorial(): dm = DataMatrix(length=3) dm.a = 'a', 'b', '' dm.b = 0, 1, 2 dm = ops.fullfactorial(dm) check_col(dm.a, ['a', 'b', 'a', 'b', 'a', 'b']) check_col(dm.b, [0, 0, 1, 1, 2, 2])
def test_weight(): dm = DataMatrix(length=3) dm.a = 'a', 'b', 'c' dm.b = 1, 0, 2 dm = ops.weight(dm.b) check_col(dm.a, ['a', 'c', 'c']) check_col(dm.b, [1, 2, 2])
def test_keep_only(): dm = DataMatrix(length=2) dm.a = 'a', 'b' dm.b = 0, 1 ops.keep_only(dm, ['b']) ok_('a' not in dm.column_names) ok_('b' in dm.column_names)
def check_str_operations(): dm = DataMatrix(length=2, default_col_type=MixedColumn) dm.col = 'a', 'b' check_col(dm.col, ['a', 'b']) dm.col += 'c', 'd' check_col(dm.col, ['ac', 'bd']) check_integrity(dm)
def check_int_operations(): dm = DataMatrix(length=2, default_col_type=IntColumn) dm.col = 1.5, 2.5 check_col(dm.col, [1, 2]) dm.col *= 1.5 check_col(dm.col, [1, 3]) check_integrity(dm)
def test_mixedcolumn(): _test_numericcolumn(MixedColumn) _test_copying(MixedColumn) dm = DataMatrix(length=4) dm.col = '1.1', '1', 'x', None check_col(dm.col, [1.1, 1, 'x', None]) dm.col[dm.col == {1, None}] = 'a', 'b' check_col(dm.col, [1.1, 'a', 'x', 'b'])
def test_group(): dm = DataMatrix(length=4) dm.a = 'b', 'b', 'a', 'a' dm.b = 'x', 'x', 'x', 'y' dm.c = IntColumn dm.c = 0, 1, 2, 3 dm = ops.group(dm, [dm.a, dm.b]) check_series(dm.c, [[2, np.nan], [3, np.nan], [0, 1]])
def test_auto_type(): dm = DataMatrix(length=2) dm.a = 'a', 1 dm.b = 0.1, 1 dm.c = 0, 1 dm = ops.auto_type(dm) assert isinstance(dm.a, MixedColumn) assert isinstance(dm.b, FloatColumn) assert isinstance(dm.c, IntColumn)
def test_nifti(): dm = DataMatrix(length=2) dm.n = NiftiColumn dm.n[0] = nib.Nifti2Image(np.array([[[0, 0], [1, 1]], [[-1, -1], [1, 1]]]), None) dm.n[1] = nib.Nifti2Image( np.array([[[1, 1], [0, 0]], [[np.nan, 1], [0, 0]]]), None) m = dm.n.mean.get_data() assert np.all(m == np.array([[[.5, .5], [.5, .5]], [[-1, 0], [.5, .5]]]))
def test_tuple_split(): dm = DataMatrix(length=4) dm.a = 'a', 'a', 'b', 'b' dm.b = 0, 1, 2, 3 dma, dmb = ops.tuple_split(dm.a, 'a', 'b') check_col(dma.a, ['a', 'a']) check_col(dma.b, [0, 1]) check_col(dmb.a, ['b', 'b']) check_col(dmb.b, [2, 3])
def test_auto_type(): dm = DataMatrix(length=2) dm.a = 'a', 1 dm.b = 0.1, 1 dm.c = 0, 1 ops.auto_type(dm) ok_(isinstance(dm.a, MixedColumn)) ok_(isinstance(dm.b, FloatColumn)) ok_(isinstance(dm.c, IntColumn))
def check_float_operations(): dm = DataMatrix(length=2, default_col_type=FloatColumn) dm.col = 1, 2 check_col(dm.col, [1, 2]) dm.col *= 2.5 check_col(dm.col, [2.5, 5]) dm.col *= np.inf, np.nan check_col(dm.col, [np.inf, np.nan]) check_integrity(dm)
def test_filter_(): dm = DataMatrix(length=4) dm.a = range(4) odd = fnc.filter_(lambda x: x % 2, dm.a) ok_(all([x % 2 for x in odd])) print(type(dm._rowid)) dm = fnc.filter_(lambda **d: d['a'] % 2, dm) print(type(dm._rowid)) eq_(dm.a, [1, 3])
def process_frame(run, frame, lm, cm): dm = DataMatrix(length=len(SUBJECTS)) dm.frame = frame dm.sub = IntColumn dm.x = FloatColumn dm.y = FloatColumn dm.pupil = FloatColumn dm.luminance = FloatColumn dm.change = FloatColumn print('Run {}, frame {}'.format(run, frame)) for row, sub in zip(dm, SUBJECTS): _dm = _get_subject_data(sub, run) _dm.pupil = ops.z(_dm.pupil) try: _row = (_dm.frame == frame)[0] except IndexError: continue row.sub = sub x = min(1279, max(0, _row.x)) y = min(546, max(0, _row.y)) if not x and not y: row.x = np.nan row.y = np.nan row.pupil = np.nan row.luminance = np.nan row.change = np.nan else: row.x = x row.y = y row.pupil = _row.pupil row.luminance = lm[int(y), int(x)] row.change = cm[int(y), int(x)] return dm
def test_map_(): for coltype in (MixedColumn, FloatColumn, IntColumn): dm = DataMatrix(length=2, default_col_type=coltype) dm.a = 1, 2 dm.a = fnc.map_(lambda x: x * 2, dm.a) eq_(dm.a, [2, 4]) ok_(isinstance(dm.a, coltype)) dm = fnc.map_(lambda **d: {'a': 0}, dm) eq_(dm.a, [0, 0]) ok_(isinstance(dm.a, coltype))
def _test_basic_properties(coltype): dm = DataMatrix(length=4, default_col_type=coltype) dm.c = 3, 1, 2, 3 dm.d = dm.c dm.e = 3, 1, 2, 3 assert dm.c.name == ['c', 'd'] assert dm.d.name == ['c', 'd'] assert dm.e.name == 'e' assert list(dm.c.unique) == [1, 2, 3] assert dm.c.count == 3
def test_sort(): dm = DataMatrix(length=2) dm.a = 'b', 'a' dm.b = 1, 0 dm.a = ops.sort(dm.a) check_col(dm.a, ['a', 'b']) check_col(dm.b, [1, 0]) dm = ops.sort(dm, by=dm.b) check_col(dm.a, ['b', 'a']) check_col(dm.b, [0, 1])
def test_keep_only(): dm = DataMatrix(length=2) dm.a = 'a', 'b' dm.b = 0, 1 dm.c = 'y', 'z' for cols in (['b', 'c'], [dm.b, dm.c]): dm = ops.keep_only(dm, *cols) assert 'a' not in dm.column_names assert 'b' in dm.column_names assert 'c' in dm.column_names
def check_shuffle(col_type): dm = DataMatrix(length=3, default_col_type=col_type) dm.col1 = 11,12,13 dm.col2 = 1,2,3 dm = operations.shuffle(dm) for row in dm: ok_(row.col1 == row.col2+10) dm.col1 = operations.shuffle(dm.col1) dm.col2 = operations.shuffle(dm.col2) check_integrity(dm)
def _test_basic_properties(coltype): dm = DataMatrix(length=4, default_col_type=coltype) dm.c = 3, 1, 2, 3 dm.d = dm.c dm.e = 3, 1, 2, 3 eq_(dm.c.name, ['c', 'd']) eq_(dm.d.name, ['c', 'd']) eq_(dm.e.name, 'e') eq_(list(dm.c.unique), [1, 2, 3]) eq_(dm.c.count, 3)
def _prf_dm(passx, data): prf_dm = DataMatrix(length=1) prf_dm.prf_x = NiftiColumn prf_dm.prf_y = NiftiColumn prf_dm.prf_sd = NiftiColumn prf_dm.prf_err = NiftiColumn prf_dm.prf_x = nib.Nifti2Image(passx[:, :, :, 0], affine=data.affine) prf_dm.prf_y = nib.Nifti2Image(passx[:, :, :, 1], affine=data.affine) prf_dm.prf_sd = nib.Nifti2Image(passx[:, :, :, 2], affine=data.affine) prf_dm.prf_err = nib.Nifti2Image(passx[:, :, :, 3], affine=data.affine) return prf_dm
def check_int_operations(): dm = DataMatrix(length=2, default_col_type=IntColumn) dm.col = 1.5, 2.5 check_col(dm.col, [1, 2]) dm.col *= 2.5 check_col(dm.col, [2, 4]) def _(): with pytest.raises(TypeError): dm.col *= 'x' _() check_integrity(dm)
def check_intcolumn_sorting(): dm = DataMatrix(length=8, default_col_type=IntColumn) dm.c = [ 1, '1', 2, '2', 1.1, '1.1', 2.1, '2.8', ] dm.c = ops.shuffle(dm.c) dm = ops.sort(dm, by=dm.c) check_col(dm.c, [ 1, 1, 1, 1, 2, 2, 2, 2 ])
def test_mixedcolumn(): check_getrow(MixedColumn) check_select(MixedColumn) check_concat(MixedColumn, invalid=u'') # Check type selectors dm = DataMatrix(length=6) dm.col = 1, 2, 3, 1.1, 2.1, 'a' eq_(len(dm.col == float), 2) eq_(len(dm.col != float), 4) eq_(len(dm.col == str), 1) eq_(len(dm.col != str), 5) eq_(len(dm.col == int), 3) eq_(len(dm.col != int), 3)
def test_split(): dm = DataMatrix(length=4) dm.a = 'a', 'a', 'b', 'b' dm.b = 0, 1, 2, 3 g = ops.split(dm.a) val, dm = g.next() eq_(val, 'a') check_col(dm.a, ['a', 'a']) check_col(dm.b, [0, 1]) val, dm = g.next() eq_(val, 'b') check_col(dm.a, ['b', 'b']) check_col(dm.b, [2, 3])
def check_select(col_type): dm = DataMatrix(length=2, default_col_type=col_type) dm.col = 1, 2 dm_ = dm.col < 2 check_col(dm_.col, [1]) dm_ = dm.col == 2 check_col(dm_.col, [2]) dm_ = (dm.col == 1) | (dm.col == 2) check_col(dm_.col, [1,2]) dm_ = (dm.col == 1) & (dm.col == 2) check_col(dm_.col, []) dm_ = (dm.col == 1) ^ (dm.col == 2) check_col(dm_.col, [1,2]) check_integrity(dm)
def check_nan_sort(): dm = DataMatrix(length=3, default_col_type=FloatColumn) dm.col1 = 2,np.nan,1 dm.col2 = 1,2,np.nan dm = operations.sort(dm, by=dm.col1) check_col(dm.col1, [1, 2, np.nan]) check_col(dm.col2, [np.nan, 1, 2]) dm = operations.sort(dm, by=dm.col2) check_col(dm.col1, [2, np.nan, 1]) check_col(dm.col2, [1, 2, np.nan]) dm.col1 = operations.sort(dm.col1) dm.col2 = operations.sort(dm.col2) check_col(dm.col1, [1, 2, np.nan]) check_col(dm.col2, [1, 2, np.nan]) check_integrity(dm)
def test_seriescolumn(): dm = DataMatrix(length=3) dm.col = SeriesColumn(depth=3) dm.col[0] = [1,2,3] dm.col[1] = [3,3,3] dm.col[2] = [4,4,4] ok_(all(dm.col.mean == [8./3, 9./3, 10/3.])) ok_(all(dm.col.median == [3,3,3])) ok_(all(dm.col.max == [4,4,4])) ok_(all(dm.col.min == [1,2,3])) ok_(all(dm.col.std == [ np.std([4,3,1], ddof=1), np.std([4,3,2], ddof=1), np.std([4,3,3], ddof=1) ]))
def test_bin_split(): dm = DataMatrix(length=4) dm.a = range(4) dm = ops.shuffle(dm) dm1, dm2 = ops.bin_split(dm.a, 2) check_col(dm1.a, [0,1]) check_col(dm2.a, [2,3]) dm1, dm2, dm3 = ops.bin_split(dm.a, 3) check_col(dm1.a, [0]) check_col(dm2.a, [1]) check_col(dm3.a, [2,3]) dm1, = ops.bin_split(dm.a, 1) check_col(dm1.a, [0,1,2,3]) @raises(ValueError) def _(): x, = ops.bin_split(dm.a, 5) _()
def test_io(): refdm = DataMatrix(length=3) refdm[u'tést'] = 1, 2, u'' refdm.B = u'mathôt', u'b', u'x' refdm.C = u'a,\\b"\'c', 8, u'' testdm = io.readtxt('testcases/data/data.csv') check_dm(refdm, testdm) io.writetxt(testdm, 'tmp.csv') testdm = io.readtxt('tmp.csv') check_dm(refdm, testdm) io.writepickle(testdm, 'tmp.pickle') testdm = io.readpickle('tmp.pickle') check_dm(refdm, testdm) io.writexlsx(testdm, 'tmp.xlsx') testdm = io.readxlsx('tmp.xlsx') check_dm(refdm, testdm)
def lmer_series(dm, formula, winlen=1): col = formula.split()[0] depth = dm[col].depth rm = None for i in range(0, depth, winlen): wm = dm[:] wm[col] = series.reduce_( series.window(wm[col], start=i, end=i+winlen)) lm = lmer(wm, formula) print('Sample %d' % i) print(lm) if rm is None: rm = DataMatrix(length=len(lm)) rm.effect = list(lm.effect) rm.p = SeriesColumn(depth=depth) rm.t = SeriesColumn(depth=depth) rm.est = SeriesColumn(depth=depth) rm.se = SeriesColumn(depth=depth) for lmrow, rmrow in zip(lm, rm): rmrow.p[i:i+winlen] = lmrow.p rmrow.t[i:i+winlen] = lmrow.t rmrow.est[i:i+winlen] = lmrow.est rmrow.se[i:i+winlen] = lmrow.se return rm
def test_intcolumn(): dm = DataMatrix(length=2) # Test assignment dm.col = IntColumn dm.col = 1 check_col(dm.col, [1, 1]) dm.col = 2, 3 check_col(dm.col, [2, 3]) dm.col[:-1] = 4 check_col(dm.col, [4, 3]) @raises(TypeError) def _(): dm.col[0] = "test" _() @raises(TypeError) def _(): dm.col[:] = "test" _() # Test shortening and lengthening dm.length = 0 dm.length = 4 # Check uniqueness dm.col = 1, 2, 1, 2 ok_(sorted(dm.col.unique) == [1, 2]) # Check dtype ok_(dm.col._seq.dtype == np.int64) check_integrity(dm)
def test_shuffle_horiz(): dm = DataMatrix(length=2) dm.a = 'a', 'b' dm.b = 0, 1 dm.c = '-', '-' while True: dm2 = ops.shuffle_horiz(dm) try: check_row(dm2[0], [0, '-', 'a']) break except: pass while True: dm2 = ops.shuffle_horiz(dm.a, dm.b) try: check_row(dm2[0], [0, 'a', '-']) break except: pass for i in range(1000): dm2 = ops.shuffle_horiz(dm.a, dm.b) check_col(dm.c, ['-', '-'])
def check_operations(col_type): dm = DataMatrix(length=2, default_col_type=col_type) dm.col = 1, 2 dm.col += 1 check_col(dm.col, [2, 3]) dm.col += 1, 2 check_col(dm.col, [3, 5]) dm.col -= 1 check_col(dm.col, [2, 4]) dm.col -= 1, 2 check_col(dm.col, [1, 2]) dm.col *= 2 check_col(dm.col, [2, 4]) dm.col *= 1.5, 3 check_col(dm.col, [3, 12]) dm.col /= 3 check_col(dm.col, [1, 4]) dm.col /= 1, 2 check_col(dm.col, [1, 2]) dm.col //= 1.5, 2.5 check_col(dm.col, [0, 0]) check_integrity(dm)
def test_seriescolumn(): dm1 = DataMatrix(length=2) dm1.col1 = SeriesColumn(2) dm1.col1 = 1, 2 dm1.col_shared = SeriesColumn(2) dm1.col_shared = 3, 4 dm2 = DataMatrix(length=2) dm2.col2 = SeriesColumn(2) dm2.col2 = 5, 6 dm2.col_shared = SeriesColumn(2) dm2.col_shared = 7, 8 dm3 = dm1 << dm2 check_series(dm3.col1, [[1,1],[2,2],[0,0],[0,0]]) check_series(dm3.col_shared, [[3,3],[4,4],[7,7],[8,8]]) check_series(dm3.col2, [[0,0],[0,0],[5,5],[6,6]]) dm3.i = [4,0,2,1] dm4 = dm3.i <= 2 dm5 = (dm3.i <= 2) | (dm3.i >= 3) check_integrity(dm1) check_integrity(dm2) check_integrity(dm3) check_integrity(dm4) check_integrity(dm5)
def parsefile(path): """ desc: Parse a single .txt data file, and return it as a DataMatrix. arguments: path: desc: The path to the data file. type: str returns: type: DataMatrix """ print('Reading %s' % path) src = io.readtxt(path, delimiter='\t') dm = DataMatrix(length=0) trialdm = None for row in src: if isinstance(row.state, str): if 'start_trial' in row.state: if trialdm is not None: print('%d: %d samples (%d invalid) %s' \ % (trialdm.trialid[0], sample, invalid, \ trialdm.snelheid[0])) dm <<= trialdm trialdm = DataMatrix(length=1) trialdm.pupil = SeriesColumn(1000) trialdm.trialid = int(row.state.split()[1]) trialdm.path = path sample = 0 invalid = 0 continue if 'var' in row.state: l = row.state[4:].split(maxsplit=1) if len(l) == 1: continue var, val = tuple(l) # Length is a special property for datamatrix if var == 'length': var = '_length' trialdm[var] = val continue continue if trialdm is not None: if row.Rpsize != '' and row.Rpsize > 0: trialdm.pupil[0, sample] = row.Rpsize else: trialdm.pupil[0, sample] = np.nan invalid += 1 sample += 1 dm <<= trialdm return dm
def check_concat(col_type, invalid): dm1 = DataMatrix(length=2, default_col_type=col_type) dm1.col1 = 1, 2 dm1.col_shared = 3, 4 dm2 = DataMatrix(length=2, default_col_type=col_type) dm2.col2 = 5, 6 dm2.col_shared = 7, 8 dm3 = dm1 << dm2 check_col(dm3.col1, [1,2,invalid,invalid]) check_col(dm3.col_shared, [3,4,7,8]) check_col(dm3.col2, [invalid,invalid,5,6])
def subject_summary(dm): """ desc: Plots the mean difference in pupil size between dark and bright trials for each participant as a bar plot. The time window is indicated by the PEAKWIN constant. This data is also written to a .csv file. arguments: dm: type: DataMatrix """ x = np.arange(len(dm.subject_nr.unique)) sm = DataMatrix(length=len(dm.subject_nr.unique)) sm.subject_nr = 0 sm.effect_win = FloatColumn sm.effect_win_se = FloatColumn sm.effect_full = FloatColumn sm.effect_full_se = FloatColumn for i, s in enumerate(dm.subject_nr.unique): _dm = dm.subject_nr == s sm.subject_nr[i] = s sm.effect_win[i], sm.effect_win_se[i] = effect_se(_dm, PEAKWIN[0], PEAKWIN[1]) sm.effect_full[i], sm.effect_full_se[i] = effect_se(_dm) sm = operations.sort(sm, by=sm.effect_win) plot.new(size=(4, 3)) plt.axhline(0, color="black") plt.plot(sm.effect_win, "o-", color=green[-1]) plt.errorbar(x, sm.effect_win, yerr=sm.effect_win_se, linestyle="", color=green[-1], capsize=0) plt.xlim(-1, 30) plt.ylabel("Pupil-size difference (normalized)") plt.xlabel("Participant") plt.xticks([]) plot.save("subject_summary") io.writetxt(sm, "%s/subject_summary.csv" % OUTPUT_FOLDER)
def test_mixedcolumn(): dm = DataMatrix(length=2) # Test assignment dm.col = 1 check_col(dm.col, [1, 1]) dm.col = 2, 3 check_col(dm.col, [2, 3]) dm.col[:-1] = 4 check_col(dm.col, [4, 3]) dm.col[:] = "test" check_col(dm.col, ["test", "test"]) # Test shortening and lengthening dm.length = 0 dm.length = 4 # Check uniqueness dm.col = 1, 2, 1, 2 ok_(sorted(dm.col.unique) == [1, 2]) check_integrity(dm)
def check_sort(col_type): dm = DataMatrix(length=3, default_col_type=col_type) dm.col1 = 3,2,1 dm.col2 = 1,2,3 dm = operations.sort(dm, by=dm.col1) check_col(dm.col1, [1, 2, 3]) check_col(dm.col2, [3, 2, 1]) dm = operations.sort(dm, by=dm.col2) check_col(dm.col1, [3, 2, 1]) check_col(dm.col2, [1, 2, 3]) dm.col2 = operations.sort(dm.col2, by=dm.col1) check_col(dm.col2, [3, 2, 1]) dm.col1 = operations.sort(dm.col1) dm.col2 = operations.sort(dm.col2) check_col(dm.col1, [1, 2, 3]) check_col(dm.col2, [1, 2, 3]) check_integrity(dm)
def test_floatcolumn(): dm = DataMatrix(length=2) # Test assignment dm.col = FloatColumn dm.col = 1 check_col(dm.col, [1, 1]) dm.col = 2, 3 check_col(dm.col, [2, 3]) dm.col[:-1] = 4 check_col(dm.col, [4, 3]) dm.col[:] = "test" for value in dm.col: ok_(np.isnan(value)) # Test shortening and lengthening dm.length = 0 dm.length = 4 # Check uniqueness dm.col = 1, 2, 1, 2 ok_(sorted(dm.col.unique) == [1, 2]) # Check dtype ok_(dm.col._seq.dtype == np.float64) check_integrity(dm)