def generate_feature(train_file, test_file, train_feature_file, test_feature_file): trn = pd.read_csv(train_file) tst = pd.read_csv(test_file) n_trn = trn.shape[0] trn.time = pd.to_datetime(trn.time) tst.time = pd.to_datetime(tst.time) df = pd.concat([trn, tst], axis=0) last_date = df[['course_id', 'time']].groupby('course_id', as_index=False).max() last_date.columns = ['course_id', 'last_date'] df = pd.merge(df, last_date, on='course_id', how='left') df['days_before_last_date'] = (df.last_date - df.time).apply(lambda x: pd.Timedelta(x).days) df['weeks_before_last_date'] = df.days_before_last_date // 7 df.ix[df.weeks_before_last_date == 4, 'weeks_before_last_date'] = 3 df['last_month'] = df.last_date.apply(lambda x: x.month) df.drop(['time', 'last_date'], axis=1, inplace=True) df.set_index('enrollment_id', inplace=True) X = encode_categorical_features(df, n=n_trn, min_obs=10) X = X.tocsr() dump_svmlight_file(X[:n_trn], trn.enrollment_id.values, train_feature_file, zero_based=False) dump_svmlight_file(X[n_trn:], tst.enrollment_id.values, test_feature_file, zero_based=False)
def generate_feature(train_file, label_file, test_file, feature_dir, feature_name): # Load data files logging.info('Loading training and test data') trn = pd.read_csv(train_file, index_col=0) tst = pd.read_csv(test_file, index_col=0) label = pd.read_csv(label_file, index_col=0) n_trn = trn.shape[0] n_tst = tst.shape[0] logging.info('Combining training and test data') df = pd.concat([trn, tst], ignore_index=True) cols = list(df.columns) num_cols = [x for x in cols if x[0] == 'n'] cnt_cols = [x for x in cols if x[0] == 'o'] cat_cols = [x for x in cols if x[0] == 'c' or x[0] == 'r'] # no transformation for numerical variables logging.info('Imputing missing values in numerical columns by 0') X_num = df.ix[:, num_cols].values X_num[np.isnan(X_num)] = 0. # log(1 + x) transformation for ordinal variables X_cnt = df.ix[:, cnt_cols].values X_cnt[np.isnan(X_cnt)] = 0. X_cnt = np.log1p(X_cnt) # One-Hot-Encoding for categorical variables logging.info('One-hot-encoding categorical columns') X_col = encode_categorical_features(df[cat_cols], min_obs=3, n=n_trn, nan_as_var=False) X = sparse.hstack((X_num, X_cnt, X_col)) X = X.tocsr() logging.info('Saving features into {}'.format(feature_dir)) for i in range(label.shape[1]): train_feature_file = os.path.join( feature_dir, '{}.trn{:02d}.sps'.format(feature_name, i)) test_feature_file = os.path.join( feature_dir, '{}.tst{:02d}.sps'.format(feature_name, i)) dump_svmlight_file(X[:n_trn], label.ix[:, i], train_feature_file, zero_based=False) dump_svmlight_file(X[n_trn:], np.zeros((n_tst, )), test_feature_file, zero_based=False)
def generate_feature(train_file, label_file, test_file, feature_dir, feature_name): # Load data files logging.info('Loading training and test data') trn = pd.read_csv(train_file, index_col=0) tst = pd.read_csv(test_file, index_col=0) label = pd.read_csv(label_file, index_col=0) n_trn = trn.shape[0] n_tst = tst.shape[0] logging.info('Combining training and test data') df = pd.concat([trn, tst], ignore_index=True) cols = list(df.columns) num_cols = [x for x in cols if x[0] == 'n'] cnt_cols = [x for x in cols if x[0] == 'o'] cat_cols = [x for x in cols if x[0] == 'c' or x[0] == 'r'] # no transformation for numerical variables logging.info('Imputing missing values in numerical columns by 0') X_num = df.ix[:, num_cols].values X_num[np.isnan(X_num)] = 0. # log(1 + x) transformation for ordinal variables X_cnt = df.ix[:, cnt_cols].values X_cnt[np.isnan(X_cnt)] = 0. X_cnt = np.log1p(X_cnt) # One-Hot-Encoding for categorical variables logging.info('One-hot-encoding categorical columns') X_col = encode_categorical_features(df[cat_cols], min_obs=10, n=n_trn, nan_as_var=False) X = sparse.hstack((X_num, X_cnt, X_col)) X = X.tocsr() logging.info('Saving features into {}'.format(feature_dir)) for i in range(label.shape[1]): train_feature_file = os.path.join(feature_dir, '{}.trn{:02d}.sps'.format(feature_name, i)) test_feature_file = os.path.join(feature_dir, '{}.tst{:02d}.sps'.format(feature_name, i)) dump_svmlight_file(X[:n_trn], label.ix[:, i], train_feature_file, zero_based=False) dump_svmlight_file(X[n_trn:], np.zeros((n_tst,)), test_feature_file, zero_based=False)
def generate_feature(train_file, test_file, object_file, train_feature_file, test_feature_file): logging.info('loading input data') trn = pd.read_csv(train_file) tst = pd.read_csv(test_file) obj = pd.read_csv(object_file, header=None) obj.columns = ['course_id', 'object', 'category', 'children', 'start'] n_trn = trn.shape[0] trn.time = pd.to_datetime(trn.time) tst.time = pd.to_datetime(tst.time) df = pd.concat([trn, tst], axis=0) # get last dates of courses last_date = df[['course_id', 'time']].groupby('course_id', as_index=False).max() last_date.columns = ['course_id', 'last_date'] # extract object information obj.children.fillna('', inplace=True) obj['n_children'] = obj.children.apply( lambda x: int(np.log2(1 + len(x.split())))) obj.start.replace('null', '1999-01-01 00:00:00', inplace=True) obj.start = pd.to_datetime(obj.start) obj = pd.merge(obj, last_date, on='course_id', how='left') obj['obj_days_before_last_date'] = ( obj.last_date - obj.start).apply(lambda x: pd.Timedelta(x).days) obj.ix[obj.obj_days_before_last_date > 30, 'obj_days_before_last_date'] = 30 # merge log data with last coursedate and object information df = pd.merge(df, last_date, on='course_id', how='left') df = pd.merge( df, obj[['object', 'category', 'n_children', 'obj_days_before_last_date']], on='object', how='left') df['days_before_last_date'] = ( df.last_date - df.time).apply(lambda x: pd.Timedelta(x).days) df['weeks_before_last_date'] = df.days_before_last_date // 7 df.ix[df.weeks_before_last_date == 4, 'weeks_before_last_date'] = 3 df['last_month'] = df.last_date.apply(lambda x: x.month) df['obj_10_days_after_last_date'] = df.obj_days_before_last_date.apply( lambda x: 1 if x < 0 and x >= -10 else 0) df.obj_days_before_last_date = df.obj_days_before_last_date.apply( lambda x: np.sign(x) * int(np.log2(1 + np.sign(x) * x)) \ if ~pd.isnull(x) else x ) df.drop(['time', 'last_date'], axis=1, inplace=True) df.set_index('enrollment_id', inplace=True) X = encode_categorical_features(df, n=n_trn, min_obs=100, nan_as_var=True) X = X.tocsr() dump_svmlight_file(X[:n_trn], trn.enrollment_id.values, train_feature_file, zero_based=False) dump_svmlight_file(X[n_trn:], tst.enrollment_id.values, test_feature_file, zero_based=False)
def generate_feature(train_file, test_file, object_file, user_feature_file, course_feature_file, train_feature_file, test_feature_file): logging.info('loading input data') trn = pd.read_csv(train_file) tst = pd.read_csv(test_file) obj = pd.read_csv(object_file, header=None) obj.columns = ['course_id', 'object', 'category', 'children', 'start'] n_trn = trn.shape[0] trn.time = pd.to_datetime(trn.time) tst.time = pd.to_datetime(tst.time) df = pd.concat([trn, tst], axis=0) df['count'] = 1 # get last dates of courses last_date = df[['course_id', 'time']].groupby('course_id', as_index=False).max() last_date.columns = ['course_id', 'last_date'] # extract object information obj.children.fillna('', inplace=True) obj['n_children'] = obj.children.apply( lambda x: int(np.log2(1 + len(x.split())))) obj.start.replace('null', '2013-10-27 00:00:00', inplace=True) obj.start = pd.to_datetime(obj.start) obj = pd.merge(obj, last_date, on='course_id', how='left') obj['obj_days_before_last_date'] = ( obj.last_date - obj.start).apply(lambda x: pd.Timedelta(x).days) obj.ix[obj.obj_days_before_last_date > 30, 'obj_days_before_last_date'] = 32 obj.ix[obj.obj_days_before_last_date < -10, 'obj_days_before_last_date'] = -16 # merge log data with last coursedate and object information df = pd.merge(df, last_date, on='course_id', how='left') df = pd.merge( df, obj[['object', 'category', 'n_children', 'obj_days_before_last_date']], on='object', how='left') df['days_before_last_date'] = ( df.last_date - df.time).apply(lambda x: pd.Timedelta(x).days) df['weeks_before_last_date'] = df.days_before_last_date // 7 df.ix[df.weeks_before_last_date == 4, 'weeks_before_last_date'] = 3 df['last_month'] = df.last_date.apply(lambda x: x.month) df['days_after_obj_date'] = df.obj_days_before_last_date - df.days_before_last_date df.ix[df.days_after_obj_date < 0, 'days_after_obj_date'] = -1 df.days_after_obj_date = df.days_after_obj_date.apply(lambda x: np.sign( x) * int(np.log2(1 + np.abs(x))) if ~pd.isnull(x) else x) df['obj_10_days_after_last_date'] = df.obj_days_before_last_date.apply( lambda x: 1 if x < 0 and x >= -10 else 0) df.obj_days_before_last_date = df.obj_days_before_last_date.apply( lambda x: np.sign(x) * int(np.log2(1 + np.abs(x))) if ~pd.isnull(x) else x) eid = df.drop(['time', 'last_date'], axis=1) eid.set_index('enrollment_id', inplace=True) X = encode_categorical_features(eid, n=n_trn, min_obs=10, nan_as_var=True) X = X.tocsr() dump_svmlight_file(X[:n_trn], trn.enrollment_id.values, train_feature_file, zero_based=False) dump_svmlight_file(X[n_trn:], tst.enrollment_id.values, test_feature_file, zero_based=False) cid = df.drop(['enrollment_id', 'time', 'last_date'], axis=1) cid.set_index('course_id', inplace=True) X = encode_categorical_features(cid, n=n_trn, min_obs=10, nan_as_var=True) X = X.tocsr() with open(course_feature_file, 'w') as f: for i in range(X.shape[0]): x = X[i].toarray().flatten() idx = np.where(x != 0)[0] features = ' '.join(['{}:{}'.format(j + 1, x[j]) for j in idx]) f.write('{} {}\n'.format(cid.index.values[i], features)) uid = df.drop(['enrollment_id', 'time', 'last_date'], axis=1) uid.set_index('username', inplace=True) X = encode_categorical_features(uid, n=n_trn, min_obs=10, nan_as_var=True) X = X.tocsr() with open(user_feature_file, 'w') as f: for i in range(X.shape[0]): x = X[i].toarray().flatten() idx = np.where(x != 0)[0] features = ' '.join(['{}:{}'.format(j + 1, x[j]) for j in idx]) f.write('{} {}\n'.format(uid.index.values[i], features))