def __init__(self, data_dir): data_cols = [ 'order_id', 'product_id', 'features', 'label' ] data = [np.load(os.path.join(data_dir, '{}.npy'.format(i)), mmap_mode='r') for i in data_cols] df = DataFrame(columns=data_cols, data=data) self.data_dim = df['features'].shape[1] print df.shapes() print 'loaded data' self.test_df = df.mask(df['label'] == -1) self.train_df = df.mask(df['label'] != -1) self.train_df, self.val_df = self.train_df.train_test_split(train_size=0.9) print 'train size', len(self.train_df) print 'val size', len(self.val_df) print 'test size', len(self.test_df) self.feature_means = np.load(os.path.join(data_dir, 'feature_means.npy')) self.feature_maxs = np.load(os.path.join(data_dir, 'feature_maxs.npy')) self.feature_mins = np.load(os.path.join(data_dir, 'feature_mins.npy'))
def __init__(self, data_dir): data_cols = ['order_id', 'product_id', 'features', 'label'] data = [ np.load(os.path.join(data_dir, '{}.npy'.format(i)), mmap_mode='r') for i in data_cols ] df = DataFrame(columns=data_cols, data=data) self.data_dim = df['features'].shape[1] print df.shapes() print 'loaded data' self.test_df = df.mask(df['label'] == -1) self.train_df = df.mask(df['label'] != -1) self.train_df, self.val_df = self.train_df.train_test_split( train_size=0.9) print 'train size', len(self.train_df) print 'val size', len(self.val_df) print 'test size', len(self.test_df) self.feature_means = np.load( os.path.join(data_dir, 'feature_means.npy')) self.feature_maxs = np.load(os.path.join(data_dir, 'feature_maxs.npy')) self.feature_mins = np.load(os.path.join(data_dir, 'feature_mins.npy'))
class DataReader(object): def __init__(self, data_dir): data_cols = [ 'user_id', 'history_length', 'order_size_history', 'reorder_size_history', 'order_number_history', 'order_dow_history', 'order_hour_history', 'days_since_prior_order_history', ] data = [np.load(os.path.join(data_dir, '{}.npy'.format(i)), mmap_mode='r') for i in data_cols] self.test_df = DataFrame(columns=data_cols, data=data) print self.test_df.shapes() print 'loaded data' self.train_df, self.val_df = self.test_df.train_test_split(train_size=0.9) print 'train size', len(self.train_df) print 'val size', len(self.val_df) print 'test size', len(self.test_df) def train_batch_generator(self, batch_size): return self.batch_generator( batch_size=batch_size, df=self.train_df, shuffle=True, num_epochs=10000, is_test=False ) def val_batch_generator(self, batch_size): return self.batch_generator( batch_size=batch_size, df=self.val_df, shuffle=True, num_epochs=10000, is_test=False ) def test_batch_generator(self, batch_size): return self.batch_generator( batch_size=batch_size, df=self.test_df, shuffle=False, num_epochs=1, is_test=True ) def batch_generator(self, batch_size, df, shuffle=True, num_epochs=10000, is_test=False): batch_gen = df.batch_generator(batch_size, shuffle=shuffle, num_epochs=num_epochs, allow_smaller_final_batch=is_test) for batch in batch_gen: batch['order_dow_history'] = np.roll(batch['order_dow_history'], -1, axis=1) batch['order_hour_history'] = np.roll(batch['order_hour_history'], -1, axis=1) batch['days_since_prior_order_history'] = np.roll(batch['days_since_prior_order_history'], -1, axis=1) batch['order_number_history'] = np.roll(batch['order_number_history'], -1, axis=1) batch['next_reorder_size'] = np.roll(batch['reorder_size_history'], -1, axis=1) if not is_test: batch['history_length'] = batch['history_length'] - 1 yield batch
class DataReader(object): def __init__(self, data_dir): data_cols = [ 'user_id', 'history_length', 'order_size_history', 'reorder_size_history', 'order_number_history', 'order_dow_history', 'order_hour_history', 'days_since_prior_order_history', ] data = [ np.load(os.path.join(data_dir, '{}.npy'.format(i)), mmap_mode='r') for i in data_cols ] self.test_df = DataFrame(columns=data_cols, data=data) print self.test_df.shapes() print 'loaded data' self.train_df, self.val_df = self.test_df.train_test_split( train_size=0.9) print 'train size', len(self.train_df) print 'val size', len(self.val_df) print 'test size', len(self.test_df) def train_batch_generator(self, batch_size): return self.batch_generator(batch_size=batch_size, df=self.train_df, shuffle=True, num_epochs=10000, is_test=False) def val_batch_generator(self, batch_size): return self.batch_generator(batch_size=batch_size, df=self.val_df, shuffle=True, num_epochs=10000, is_test=False) def test_batch_generator(self, batch_size): return self.batch_generator(batch_size=batch_size, df=self.test_df, shuffle=False, num_epochs=1, is_test=True) def batch_generator(self, batch_size, df, shuffle=True, num_epochs=10000, is_test=False): batch_gen = df.batch_generator(batch_size, shuffle=shuffle, num_epochs=num_epochs, allow_smaller_final_batch=is_test) for batch in batch_gen: batch['order_dow_history'] = np.roll(batch['order_dow_history'], -1, axis=1) batch['order_hour_history'] = np.roll(batch['order_hour_history'], -1, axis=1) batch['days_since_prior_order_history'] = np.roll( batch['days_since_prior_order_history'], -1, axis=1) batch['order_number_history'] = np.roll( batch['order_number_history'], -1, axis=1) batch['next_reorder_size'] = np.roll(batch['reorder_size_history'], -1, axis=1) if not is_test: batch['history_length'] = batch['history_length'] - 1 yield batch
class DataReader(object): def __init__(self, data_dir): data_cols = [ 'user_id', 'product_id', 'aisle_id', 'department_id', 'is_ordered_history', 'index_in_order_history', 'order_dow_history', 'order_hour_history', 'days_since_prior_order_history', 'order_size_history', 'reorder_size_history', 'order_is_weekend_history', 'order_part_of_day_history', 'order_number_history', 'history_length', 'product_name', 'product_name_length', 'eval_set', 'label' ] data = [ np.load(os.path.join(data_dir, '{}.npy'.format(i)), mmap_mode='r') for i in data_cols ] self.test_df = DataFrame(columns=data_cols, data=data) print(self.test_df.shapes()) print('loaded data') self.train_df, self.val_df = self.test_df.train_test_split( train_size=0.9) print('train size', len(self.train_df)) print('val size', len(self.val_df)) print('test size', len(self.test_df)) def train_batch_generator(self, batch_size): return self.batch_generator(batch_size=batch_size, df=self.train_df, shuffle=True, num_epochs=10000, is_test=False) def val_batch_generator(self, batch_size): return self.batch_generator(batch_size=batch_size, df=self.val_df, shuffle=True, num_epochs=10000, is_test=False) def test_batch_generator(self, batch_size): return self.batch_generator(batch_size=batch_size, df=self.test_df, shuffle=False, num_epochs=1, is_test=True) def batch_generator(self, batch_size, df, shuffle=True, num_epochs=10000, is_test=False): batch_gen = df.batch_generator(batch_size, shuffle=shuffle, num_epochs=num_epochs, allow_smaller_final_batch=is_test) for batch in batch_gen: batch['order_dow_history'] = np.roll(batch['order_dow_history'], -1, axis=1) batch['order_hour_history'] = np.roll(batch['order_hour_history'], -1, axis=1) batch['days_since_prior_order_history'] = np.roll( batch['days_since_prior_order_history'], -1, axis=1) batch['order_is_weekend_history'] = np.roll( batch['order_is_weekend_history'], -1, axis=1) batch['order_part_of_day_history'] = np.roll( batch['order_part_of_day_history'], -1, axis=1) batch['order_number_history'] = np.roll( batch['order_number_history'], -1, axis=1) batch['next_is_ordered'] = np.roll(batch['is_ordered_history'], -1, axis=1) batch['is_none'] = batch['product_id'] == 0 if not is_test: batch['history_length'] = batch['history_length'] - 1 yield batch