def test_errors(self): with self.assertRaises(ValueError): trans = n.CatTransform(norm_mode='whatever', name='cat') with self.assertRaises(ValueError): trans = n.CatTransform(norm_mode='min_max', name='cat') trans = n.CatTransform( name='cat', norm_mode='mean_std', ) with self.assertRaises(AssertionError): trans.pour(np.array([1]))
def test_df_iter(self): trans = n.CatTransform(name='cat', index_to_cat_val=np.unique( self.array[:, 0:2].astype(np.str)), input_dtype=np.dtype('S')) array_iter = [ self.array[0:2, 0:2].astype(str), self.array[2:4, 0:2].astype(str) ] df_iter = [] for array in array_iter: df = pd.DataFrame(data=array, index=[0, 1], columns=['a', 'b']) df_iter.append(df) trans.calc_global_values(data_iter=df_iter) target = np.array([[[0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0]], [[0.0, 0.0, 1.0, 0.0], [1.0, 0.0, 0.0, 0.0]], [[0.0, 0.0, 0.0, 1.0], [0.0, 0.0, 1.0, 0.0]], [[0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 0.0, 1.0]]]) indices = np.array([[1, 2], [2, 0], [3, 2], [1, 3]]) for i in xrange(2): self.pour_pump( trans, self.array[:, 0:2].astype(np.str), { 'cat/missing_vals': np.array([['', ''], ['', ''], ['', ''], ['', '']], dtype='|S4'), 'cat/one_hots': target, 'cat/indices': indices }) trans = self.write_read(trans, self.temp_dir)
def test_two_cols(self): trans = n.CatTransform(name='cat', index_to_cat_val=np.unique( self.array[:, 0:2].astype(np.str))) array_iter = [ self.array[0:2, 0:2].astype(str), self.array[2:4, 0:2].astype(str) ] trans.calc_global_values(data_iter=array_iter) target = np.array([[[0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0]], [[0.0, 0.0, 1.0, 0.0], [1.0, 0.0, 0.0, 0.0]], [[0.0, 0.0, 0.0, 1.0], [0.0, 0.0, 1.0, 0.0]], [[0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 0.0, 1.0]]]) indices = np.array([[1, 2], [2, 0], [3, 2], [1, 3]]) for i in xrange(2): self.pour_pump( trans, self.array[:, 0:2].astype(np.str), { 'cat/missing_vals': np.array([['', ''], ['', ''], ['', ''], ['', '']], dtype='|S4'), 'cat/one_hots': target, 'cat/indices': indices }) trans = self.write_read(trans, self.temp_dir)
def test_mean_std(self): trans = n.CatTransform(name='cat', norm_mode='mean_std', index_to_cat_val=sorted( np.unique(self.array[:, 1:2].astype(np.str)))) array_iter = [ self.array[0:2, 1:2].astype(str), self.array[2:4, 1:2].astype(str) ] trans.calc_global_values(data_iter=array_iter) target = np.array([ [[0., 1, 0]], [[1, 0, 0]], [[0, 1, 0]], [[0, 0, 1]], ]).astype(float) # print trans.mean target = (target - trans.mean) / trans.std indices = np.array([[1], [0], [1], [2]]) for i in xrange(2): self.pour_pump( trans, self.array[:, 1:2].astype(np.str), { 'cat/missing_vals': np.array([[''], [''], [''], ['']], dtype='|S4'), 'cat/one_hots': target, 'cat/indices': indices }) trans = self.write_read(trans, self.temp_dir)
def test_read_write(self): for i in xrange(3): if i in (0, 1): array = self.array[:, i:i + 1].astype(np.str) else: array = self.array[:, i:i + 1].astype(np.float) trans = n.CatTransform(name='cat', norm_mode='mean_std') trans.calc_global_values(array) self.write_read_example(trans, array, self.temp_dir)
def test_index_to_cat_val(self): trans = n.CatTransform(name='cat', index_to_cat_val=['a', 'b']) trans.calc_global_values(self.array[:, 0:1].astype(np.str)) target = np.array([[[1.0, 0.0]], [[0.0, 1.0]], [[0.0, 0.0]], [[1.0, 0.0]]]).astype(float) indices = np.array([[0], [1], [-1], [0]]) for i in xrange(2): self.pour_pump( trans, self.array[:, 0:1].astype(np.str), { 'cat/missing_vals': [[''], [''], ['c'], ['']], 'cat/one_hots': target, 'cat/indices': indices }) trans = self.write_read(trans, self.temp_dir)
def test_no_norm(self): trans = n.CatTransform(name='cat') trans.calc_global_values(self.array[:, 0:1].astype(np.str)) target = np.array([[[1., 0., 0.]], [[0., 1., 0.]], [[0., 0., 1.]], [[1., 0., 0.]]]) indices = np.array([[0], [1], [2], [0]]) for i in xrange(2): self.pour_pump( trans, self.array[:, 0:1].astype(np.str), { 'cat/missing_vals': [[''], [''], [''], ['']], 'cat/one_hots': target, 'cat/indices': indices }) trans = self.write_read(trans, self.temp_dir)
def test_null(self): trans = n.CatTransform(name='cat', ) trans.calc_global_values(self.array[:, 2:3].astype(np.float64)) target = np.array([[[0, 1, 0]], [[0, 0, 1]], [[1, 0, 0]], [[0, 1, 0]]]).astype(float) indices = np.array([[1], [2], [0], [1]]) for i in xrange(2): self.pour_pump( trans, self.array[:, 2:3].astype(np.float64), { 'cat/missing_vals': np.array([[0.0], [0.0], [0.0], [0.0]], dtype=float), 'cat/one_hots': target, 'cat/indices': indices }) trans = self.write_read(trans, self.temp_dir)
def test_mean_std(self): trans = n.CatTransform(name='cat', norm_mode='mean_std') trans.calc_global_values(self.array[:, 1:2].astype(np.str)) target = np.array([ [[0., 1, 0]], [[1, 0, 0]], [[0, 1, 0]], [[0, 0, 1]], ]).astype(float) target = (target - trans.mean) / trans.std indices = np.array([[1], [0], [1], [2]]) for i in xrange(2): self.pour_pump( trans, self.array[:, 1:2].astype(np.str), { 'cat/missing_vals': np.array([[''], [''], [''], ['']], dtype='|S4'), 'cat/one_hots': target, 'cat/indices': indices }) trans = self.write_read(trans, self.temp_dir)
def test_array(self): array = self._get_array() dataset_transform = tr.DatasetTransform(name='DT') dataset_transform.add_transform( cols=[0], transform=ct.CatTransform( name='CAT', norm_mode='mean_std', index_to_cat_val=sorted(np.unique(array[:, 0: 1])), input_dtype=np.dtype('U') ) ) dataset_transform.add_transform( cols=[1, 2, 3], transform=dt.DateTimeTransform( name='DATE', norm_mode='min_max', fill_nat_func=lambda a: np.array(datetime.datetime(1950, 1, 1)), input_dtype=np.datetime64, ) ) dataset_transform.add_transform( cols=[4, 5, 6], transform=nt.NumTransform( name='NUM', norm_mode='min_max', input_dtype=np.float64, fill_nan_func=lambda a: np.array(0), ) ) dataset_transform.add_transform( cols=[7, 8], transform=st.StringTransform( name='STRING', word_tokenizer=en_tokenizer, index_to_word=['__UNK__'] + self._get_index_to_word(array[:, 7:9], en_tokenizer), input_dtype=np.dtype('U'), max_sent_len=20 ) ) dataset_transform.calc_global_values(data=array) for i in xrange(2): self.pour_pump( dataset_transform, array, { 'DT/CAT/indices': [[0], [1], [2], [0]], 'DT/CAT/missing_vals': np.array([[u''], [u''], [u''], [u'']]), 'DT/CAT/one_hots': [[[1.0, -0.5773502691896258, -0.5773502691896258]], [[-1.0, 1.7320508075688774, -0.5773502691896258]], [[-1.0, -0.5773502691896258, 1.7320508075688774]], [[1.0, -0.5773502691896258, -0.5773502691896258]]], 'DT/DATE/nats': [[False, False, True], [False, True, True], [False, False, True], [False, False, True]], 'DT/DATE/diff': np.array([[datetime.timedelta(0), datetime.timedelta(0), datetime.timedelta(0)], [datetime.timedelta(0), datetime.timedelta(0), datetime.timedelta(0)], [datetime.timedelta(0), datetime.timedelta(0), datetime.timedelta(0)], [datetime.timedelta(0), datetime.timedelta(0), datetime.timedelta(0)]], dtype='timedelta64[us]'), 'DT/DATE/nums': np.array([[0.9976643838327857, 0.9976643838327857, 0.0], [0.9977039705474843, 0.0, 0.0], [0.9977435572621828, 0.9988915719884407, 0.0], [0.9976643838327857, 1.0, 0.0]]), 'DT/NUM/nans': [[False, False, True], [False, True, True], [False, False, True], [False, False, True]], 'DT/NUM/nums': [[0.09090909090909091, 0.18181818181818182, 0.0], [0.36363636363636365, 0.0, 0.0], [0.6363636363636364, 0.7272727272727273, 0.0], [0.9090909090909091, 1.0, 0.0]], 'DT/STRING/indices': [[[9, 29, 50, 30, 29, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [6, 38, 2, 23, 49, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]], [[7, 16, 43, 28, 49, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [5, 26, 53, 31, 22, 50, 8, 46, 42, 15, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1]], [[12, 41, 29, 34, 54, 2, 30, 1, 18, 3, 10, 3, -1, -1, -1, -1, -1, -1, -1, -1], [13, 21, 45, 39, 27, 14, 20, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]], [[4, 32, 33, 14, 48, 44, 31, 51, 47, 43, 52, 17, 3, -1, -1, -1, -1, -1, -1, -1], [11, 1, 24, 19, 36, 43, 40, 35, 25, 37, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1]]], 'DT/STRING/missing_vals': np.array([[[u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u''], [u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'']], [[u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u''], [u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'']], [[u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u''], [u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'']], [[u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u''], [u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'', u'']]], dtype='|U10'), 'DT/STRING/tokenize_diff': np.array([['[["d", 16, 17, ""], ["d", 18, 32, ""]]', '[["d", 8, 9, ""], ["d", 19, 20, ""], ["d", 21, 35, ""]]'], ['[["d", 24, 25, ""], ["d", 26, 40, ""]]', '[["d", 58, 59, ""], ["d", 60, 69, ""]]'], ['[["d", 21, 22, ""], ["d", 26, 27, ""], ["d", 37, 38, ""], ["d", 42, 43, ""], ["d", 44, 52, ""]]', '[["d", 35, 36, ""], ["d", 37, 49, ""]]'], ['[["d", 2, 3, ""], ["d", 57, 58, ""], ["d", 59, 66, ""]]', '[["d", 3, 4, ""], ["d", 45, 46, ""], ["d", 47, 56, ""]]']], dtype='|U95'), # 'DT/Partition_0/tubes/missing_cols': np.array([1, 2]), # 'DT/Partition_0/tubes/missing_array': np.array([['b', 'None', 'b', 'c'], [1.0, 2.0, np.nan, 1.0]], dtype=np.object), } ) # self.write_read_example(dataset_transform, array, self.temp_dir) dataset_transform = self.write_read(dataset_transform, self.temp_dir)