def get_item(root): print('load') csv_file = os.path.join(root, 'test', 'test_data', 'test_data') item = pd.read_csv(csv_file, dtype={ 'article_id': str, 'hh': int, 'gender': str, 'age_range': str, 'read_article_ids': str }, sep='\t') print('loaded!!') sparse_features = ['article_id', 'hh','gender','age_range','len_bin'] dense_features = ['image_feature'] target = ['label'] len_lis = [] read_article_ids_all = item['read_article_ids'].tolist() for i in range(len(item)): li = read_article_ids_all[i] if type(li) == float: len_lis.append(0) continue len_li = len(li.split(',')) len_lis.append(len_li) item['len'] = len_lis item['len_bin'] = pd.qcut(item['len'],6,duplicates='drop') id_to_artic = dict() artics = item['article_id'].tolist() with open(os.path.join(DATASET_PATH, 'test', 'test_data', 'test_image_features.pkl'), 'rb') as handle: image_feature_dict = pickle.load(handle) print('image_feaeture_dict loaded..') for feat in sparse_features: lbe = LabelEncoder() item[feat] = lbe.fit_transform(item[feat]) # test set으로 구성해도 되고 item 을.. fixlen_feature_columns = [] for feat in sparse_features: if feat == 'article_id': fixlen_feature_columns.append(SparseFeat(feat,1896)) else: fixlen_feature_columns.append(SparseFeat(feat,item[feat].nunique())) #fixlen_feature_columns = [SparseFeat(feat, item[feat].nunique()) for feat in sparse_features] fixlen_feature_columns += [DenseFeat(feat,len(image_feature_dict[artics[0]])) for feat in dense_features] print(fixlen_feature_columns) idx_artics_all = item['article_id'].tolist() for i in range(len(artics)): idx_artic = idx_artics_all[i] if idx_artic not in id_to_artic.keys(): id_to_artic[idx_artic] = artics[i] linear_feature_columns = fixlen_feature_columns dnn_feature_columns = fixlen_feature_columns fixlen_feature_names = get_fixlen_feature_names(linear_feature_columns + dnn_feature_columns) fixlen_feature_names_global = fixlen_feature_names model = xDeepFM(linear_feature_columns, dnn_feature_columns, task= 'binary') #bind_nsml(model, list(), args.task) return model, fixlen_feature_names_global, item,image_feature_dict, id_to_artic
def main(args, local): if args.arch == 'xDeepFM' and args.mode == 'train': s = time.time() csv_file = os.path.join(DATASET_PATH, 'train', 'train_data', 'train_data') item = pd.read_csv(csv_file, dtype={ 'article_id': str, 'hh': int, 'gender': str, 'age_range': str, 'read_article_ids': str }, sep='\t') label_data_path = os.path.join(DATASET_PATH, 'train', os.path.basename(os.path.normpath(csv_file)).split('_')[0] + '_label') label = pd.read_csv(label_data_path, dtype={'label': int}, sep='\t') item['label'] = label sparse_features = ['article_id', 'hh','gender','age_range','len_bin'] dense_features = ['image_feature'] target = ['label'] len_lis = [] read_article_ids_all = item['read_article_ids'].tolist() for i in range(len(item)): li = read_article_ids_all[i] if type(li) == float: len_lis.append(0) continue len_li = len(li.split(',')) len_lis.append(len_li) item['len'] = len_lis item['len_bin'] = pd.qcut(item['len'],6,duplicates='drop') id_to_artic = dict() artics = item['article_id'].tolist() with open(os.path.join(DATASET_PATH, 'train', 'train_data', 'train_image_features.pkl'), 'rb') as handle: image_feature_dict = pickle.load(handle) for feat in sparse_features: lbe = LabelEncoder() item[feat] = lbe.fit_transform(item[feat]) fixlen_feature_columns = [SparseFeat(feat, item[feat].nunique()) for feat in sparse_features] fixlen_feature_columns += [DenseFeat(feat,len(image_feature_dict[artics[0]])) for feat in dense_features] idx_artics_all = item['article_id'].tolist() for i in range(len(artics)): idx_artic = idx_artics_all[i] if idx_artic not in id_to_artic.keys(): id_to_artic[idx_artic] = artics[i] #image_feature_dict[article_id] 로 가져오면 되니까 일단 패스 linear_feature_columns = fixlen_feature_columns dnn_feature_columns = fixlen_feature_columns fixlen_feature_names = get_fixlen_feature_names(linear_feature_columns + dnn_feature_columns) print(fixlen_feature_names) global fixlen_feature_names_global fixlen_feature_names_global = fixlen_feature_names model = xDeepFM(linear_feature_columns, dnn_feature_columns, task= 'binary') print('---model defined---') # 만들었던 파일들 저장하는 것도 하나 짜기, 매번 돌릴 수 없으니까 print(time.time() - s ,'seconds') if use_nsml and args.mode == 'train': bind_nsml(model,[], args.task) if args.mode == 'test': print('_infer root - : ', DATASET_PATH) print('test') model, fixlen_feature_names_global, item, image_feature_dict, id_to_artic = get_item(DATASET_PATH) bind_nsml(model, [], args.task) checkpoint_session = ['401','team_62/airush2/176'] nsml.load(checkpoint = str(checkpoint_session[0]), session = str(checkpoint_session[1])) print('successfully loaded') if (args.mode == 'train'): if args.dry_run: print('start dry-running...!') args.num_epochs = 1 else: print('start training...!') # 미리 전체를 다 만들어놓자 굳이 generator 안써도 되겠네 nsml.save('infer') print('end') print('end_main') if args.pause: nsml.paused(scope=local)
def main(args): if args.arch == 'MLP': model = get_mlp(num_classes=args.num_classes) elif args.arch == 'Resnet': model = get_resnet18(num_classes=args.num_classes) elif args.arch == 'xDeepFM': s = time.time() csv_file = os.path.join(DATASET_PATH, 'train', 'train_data', 'train_data') item = pd.read_csv(csv_file, dtype={ 'article_id': str, 'hh': int, 'gender': str, 'age_range': str, 'read_article_ids': str }, sep='\t') label_data_path = os.path.join( DATASET_PATH, 'train', os.path.basename(os.path.normpath(csv_file)).split('_')[0] + '_label') label = pd.read_csv(label_data_path, dtype={'label': int}, sep='\t') item['label'] = label print(len(item)) sparse_features = [ 'article_id', 'hh', 'gender', 'age_range', 'len_bin' ] dense_features = ['image_feature'] target = ['label'] print(time.time() - s, 'seconds') s = time.time() len_lis = [] read_article_ids_all = item['read_article_ids'].tolist() for i in range(len(item)): li = read_article_ids_all[i] if type(li) == float: len_lis.append(0) continue len_li = len(li.split(',')) len_lis.append(len_li) print(f'read_article_ids_all len : {len(read_article_ids_all)}') """ def extract_len_read_article(read_article_ids): if type(read_article_ids) == float: return 0 else : return len(read_article_ids.split(',')) read_article_ids_all = item['read_article_ids'].tolist() with Pool(processes=6) as p: len_lis = list(tqdm(p.imap(extract_len_read_article, read_article_ids_all), total=len(read_article_ids_all))) """ item['len'] = len_lis item['len_bin'] = pd.qcut(item['len'], 6, duplicates='drop') print('len_bin finished ', time.time() - s, 'seconds') id_to_artic = dict() artics = item['article_id'].tolist() with open( os.path.join(DATASET_PATH, 'train', 'train_data', 'train_image_features.pkl'), 'rb') as handle: image_feature_dict = pickle.load(handle) for feat in sparse_features: lbe = LabelEncoder() item[feat] = lbe.fit_transform(item[feat]) fixlen_feature_columns = [ SparseFeat(feat, item[feat].nunique()) for feat in sparse_features ] fixlen_feature_columns += [ DenseFeat(feat, len(image_feature_dict[artics[0]])) for feat in dense_features ] print(artics[0]) print(fixlen_feature_columns) """ [SparseFeat(name='article_id', dimension=1896, use_hash=False, dtype='int32', embedding_name='article_id', embedding=True), SparseFeat(name='hh', dimension=24, use_hash=False, dtype='int32', embedding_name='hh', embedding=True), SparseFeat(name='gender', dimension=2, use_hash=False, dtype='int32', embedding_name='gender', embedding=True), SparseFeat(name='age_range', dimension=9, use_hash=False, dtype='int32', embedding_name='age_range', embedding=True), SparseFeat(name='len_bin', dimension=5, use_hash=False, dtype='int32', embedding_name='len_bin', embedding=True), DenseFeat(name='image_feature', dimension=2048, dtype='float32')] """ print('---fixlen_feature_columns finished---') s = time.time() idx_artics_all = item['article_id'].tolist() print(f'idx_artics_all len : {len(idx_artics_all)}') print(f'artics len : {len(artics)}') for i in range(len(artics)): idx_artic = idx_artics_all[i] if idx_artic not in id_to_artic.keys(): id_to_artic[idx_artic] = artics[i] print(f'id_to_artic len : {len(id_to_artic)}') print(time.time() - s, 'seconds') #image_feature_dict[article_id] 로 가져오면 되니까 일단 패스 linear_feature_columns = fixlen_feature_columns dnn_feature_columns = fixlen_feature_columns fixlen_feature_names = get_fixlen_feature_names( linear_feature_columns + dnn_feature_columns) print(fixlen_feature_names) global fixlen_feature_names_global fixlen_feature_names_global = fixlen_feature_names model = xDeepFM(linear_feature_columns, dnn_feature_columns, task='binary') print('---model defined---') # 만들었던 파일들 저장하는 것도 하나 짜기, 매번 돌릴 수 없으니까 """ if args.use_gpu: model = model.cuda() else: model = model.cpu() """ optimizer = tf.keras.optimizers.Adam(args.lr) # negative sampling item_pos = item[item['label'] == 1] item_neg = item[item['label'] == 0] print(f'len item_pos : {len(item_pos)}') print(f'len item_neg : {len(item_neg)}') dn_1 = item_neg.sample(n=2 * len(item_pos), random_state=42) dn_1.reset_index() print(f'len dn_1 : {len(dn_1)}') data_1 = pd.concat([dn_1, item_pos]).sample(frac=1, random_state=42).reset_index() print(f'len data_1 : {len(data_1)}') print('--- negative sampling completed ---') s = time.time() data_1_article_idxs = data_1['article_id'].tolist() li = [] for i in range(len(data_1_article_idxs)): image_feature = image_feature_dict[id_to_artic[data_1_article_idxs[i]]] li.append(image_feature) print(f'len image_feature : {len(li)}') data_1['image_feature'] = li li = [] print(f'finished data_1_image_feature : {time.time() - s} sec') print(f'generate all x_train') if use_nsml: bind_nsml(model, optimizer, args.task) if args.pause: nsml.paused(scope=locals()) if (args.mode == 'train') or args.dry_run: best_loss = 1000 if args.dry_run: print('start dry-running...!') args.num_epochs = 1 else: print('start training...!') # 미리 전체를 다 만들어놓자 굳이 generator 안써도 되겠네 model.compile( tf.keras.optimizers.Adam(args.lr), 'mse', metrics=['accuracy'], ) train_generator = data_generator(data_1) lr_scheduler = tf.keras.callbacks.LearningRateScheduler(scheduler) save_cbk = CustomModelCheckpoint() history = model.fit_generator(train_generator, epochs=200, verbose=2, workers=8, steps_per_epoch=np.ceil( len(data_1) / 2048), callbacks=[lr_scheduler, save_cbk]) print('again') """
def main(args, local): if args.arch == 'xDeepFM' and args.mode == 'train': s = time.time() csv_file = os.path.join(DATASET_PATH, 'train', 'train_data', 'train_data') item = pd.read_csv(csv_file, dtype={ 'article_id': str, 'hh': int, 'gender': str, 'age_range': str, 'read_article_ids': str }, sep='\t') label_data_path = os.path.join(DATASET_PATH, 'train', os.path.basename(os.path.normpath(csv_file)).split('_')[0] + '_label') label = pd.read_csv(label_data_path, dtype={'label': int}, sep='\t') item['label'] = label s = time.time() #print(f'before test article preprocess : {len(item)}') #print(f'after test article preprocess : {len(item)}') #print(f'time : {time.time() - s}') sparse_features = ['article_id', 'hh','gender','age_range','len_bin'] dense_features = ['image_feature', 'read_cnt_prob'] target = ['label'] ############################ make more feature !!!!!!! ################################# ############## 1. read_article_ids len cnt -- user feature ################################################# len_lis = [] read_article_ids_all = item['read_article_ids'].tolist() for i in range(len(item)): li = read_article_ids_all[i] if type(li) == float: len_lis.append(0) continue len_li = len(li.split(',')) len_lis.append(len_li) item['len'] = len_lis item['len_bin'] = pd.qcut(item['len'],6,duplicates='drop') id_to_artic = dict() artics = item['article_id'].tolist() #print(item.head(3)) #print('columns name : ',item.columns) sparse_features = ['article_id', 'hh','gender','age_range','len_bin'] dense_features = ['image_feature', 'read_cnt_prob'] fixlen_feature_columns = [SparseFeat(feat, item[feat].nunique()) for feat in sparse_features] fixlen_feature_columns += [DenseFeat('image_feature',2048)] fixlen_feature_columns += [DenseFeat('read_cnt_prob',1)] #print(f'fixlen_feature_columns : {fixlen_feature_columns}') linear_feature_columns = fixlen_feature_columns dnn_feature_columns = fixlen_feature_columns fixlen_feature_names = get_fixlen_feature_names(linear_feature_columns + dnn_feature_columns) print(fixlen_feature_names) global fixlen_feature_names_global fixlen_feature_names_global = fixlen_feature_names model = xDeepFM(linear_feature_columns, dnn_feature_columns, task= 'regression') print('---model defined---') #print(time.time() - s ,'seconds') if use_nsml and args.mode == 'train': bind_nsml(model,[], args.task) if args.mode == 'test': #print('_infer root - : ', DATASET_PATH) #print('test') #print('DATASET_PATH: ', DATASET_PATH) file_list= glob.glob(f'{DATASET_PATH}/test/test_data/*') #print('file_list: ',file_list) model, fixlen_feature_names_global, item, image_feature_dict,lit,lit_cnt_prob = get_item(DATASET_PATH,args.mode) bind_nsml(model, [], args.task) checkpoint_session = ['3','team_62/airush2/361'] nsml.load(checkpoint = str(checkpoint_session[0]), session = str(checkpoint_session[1])) #print('successfully loaded') if (args.mode == 'train'): #print('DATASET_PATH: ', DATASET_PATH) #file_list= glob.glob(f'{DATASET_PATH}/train/train_data/*') #print('file_list :',file_list) if args.dry_run: print('start dry-running...!') args.num_epochs = 1 else: print('start training...!') # 미리 전체를 다 만들어놓자 굳이 generator 안써도 되겠네 nsml.save('infer') print('end') #print('end_main') if args.pause: nsml.paused(scope=local)
def get_item(root, phase): #print('load') csv_file = os.path.join(root, 'test', 'test_data', 'test_data') item = pd.read_csv(csv_file, dtype={ 'article_id': str, 'hh': int, 'gender': str, 'age_range': str, 'read_article_ids': str }, sep='\t') #print('loaded!!') sparse_features = ['article_id', 'hh','gender','age_range','len_bin'] dense_features = ['image_feature', 'read_cnt_prob'] global lit_cnt_prob_list lit_cnt_prob_list = lit_cnt_prob_list.replace(' ','') lit_cnt_prob_list = lit_cnt_prob_list.replace('\n','') lit_cnt_prob = lit_cnt_prob_list.split(',') len_lis = [] read_article_ids_all = item['read_article_ids'].tolist() for i in range(len(item)): li = read_article_ids_all[i] if type(li) == float: len_lis.append(0) continue len_li = len(li.split(',')) len_lis.append(len_li) item['len'] = len_lis item['len_bin'] = pd.qcut(item['len'],6,duplicates='drop') artics = item['article_id'].tolist() lit = list(set(artics)) lit.sort() print(f'len lit : {len(lit)}') #### fea #print('feature dict generate') #resnet_feature_extractor('test') with open(os.path.join('/data/airush2/test/test_data/test_image_features.pkl'), 'rb') as handle: image_feature_dict = pickle.load(handle) print('image_feaeture_dict loaded..') print('check artic feature') print(f"757518f4a3da : {image_feature_dict['757518f4a3da']}") lbe = LabelEncoder() lbe.fit(lit) item['article_id' + '_onehot'] = lbe.transform(item['article_id']) for feat in sparse_features[1:]: lbe = LabelEncoder() item[feat + '_onehot'] = lbe.fit_transform(item[feat]) #print('----- after onehot encoding -----') #print(item.head(10)) # test set으로 구성해도 되고 item 을.. fixlen_feature_columns = [SparseFeat('article_id',1896)] fixlen_feature_columns += [SparseFeat(feat, item[feat +'_onehot'].nunique()) for feat in sparse_features[1:]] fixlen_feature_columns += [DenseFeat('image_feature',len(image_feature_dict[artics[0]]))] fixlen_feature_columns += [DenseFeat('read_cnt_prob',1)] #print(fixlen_feature_columns) idx_artics_all = item['article_id'].tolist() linear_feature_columns = fixlen_feature_columns dnn_feature_columns = fixlen_feature_columns fixlen_feature_names = get_fixlen_feature_names(linear_feature_columns + dnn_feature_columns) fixlen_feature_names_global = fixlen_feature_names model = xDeepFM(linear_feature_columns, dnn_feature_columns, task= 'binary') #bind_nsml(model, list(), args.task) return model, fixlen_feature_names_global, item,image_feature_dict, lit, lit_cnt_prob
def main(args): if args.arch == 'xDeepFM': s = time.time() csv_file = os.path.join(DATASET_PATH, 'train', 'train_data', 'train_data') item = pd.read_csv(csv_file, dtype={ 'article_id': str, 'hh': int, 'gender': str, 'age_range': str, 'read_article_ids': str }, sep='\t') label_data_path = os.path.join( DATASET_PATH, 'train', os.path.basename(os.path.normpath(csv_file)).split('_')[0] + '_label') label = pd.read_csv(label_data_path, dtype={'label': int}, sep='\t') item['label'] = label sparse_features = [ 'article_id', 'hh', 'gender', 'age_range', 'len_bin' ] dense_features = ['image_feature'] target = ['label'] len_lis = [] read_article_ids_all = item['read_article_ids'].tolist() for i in range(len(item)): li = read_article_ids_all[i] if type(li) == float: len_lis.append(0) continue len_li = len(li.split(',')) len_lis.append(len_li) item['len'] = len_lis item['len_bin'] = pd.qcut(item['len'], 6, duplicates='drop') id_to_artic = dict() artics = item['article_id'].tolist() with open( os.path.join(DATASET_PATH, 'train', 'train_data', 'train_image_features.pkl'), 'rb') as handle: image_feature_dict = pickle.load(handle) for feat in sparse_features: lbe = LabelEncoder() item[feat] = lbe.fit_transform(item[feat]) fixlen_feature_columns = [ SparseFeat(feat, item[feat].nunique()) for feat in sparse_features ] fixlen_feature_columns += [ DenseFeat(feat, len(image_feature_dict[artics[0]])) for feat in dense_features ] idx_artics_all = item['article_id'].tolist() for i in range(len(artics)): idx_artic = idx_artics_all[i] if idx_artic not in id_to_artic.keys(): id_to_artic[idx_artic] = artics[i] #image_feature_dict[article_id] 로 가져오면 되니까 일단 패스 linear_feature_columns = fixlen_feature_columns dnn_feature_columns = fixlen_feature_columns fixlen_feature_names = get_fixlen_feature_names( linear_feature_columns + dnn_feature_columns) print(fixlen_feature_names) global fixlen_feature_names_global fixlen_feature_names_global = fixlen_feature_names model = xDeepFM(linear_feature_columns, dnn_feature_columns, task='regression') print('---model defined---') # 만들었던 파일들 저장하는 것도 하나 짜기, 매번 돌릴 수 없으니까 print(time.time() - s, 'seconds') optimizer = tf.keras.optimizers.Adam(args.lr) s = time.time() # negative sampling item_pos = item[item['label'] == 1] item_neg = item[item['label'] == 0] dn_1 = item_neg.sample(n=3 * len(item_pos), random_state=42) dn_1.reset_index() data_1 = pd.concat([dn_1, item_pos]).sample(frac=1, random_state=42).reset_index() data_1_article_idxs = data_1['article_id'].tolist() li = [] for i in range(len(data_1_article_idxs)): image_feature = image_feature_dict[id_to_artic[data_1_article_idxs[i]]] li.append(image_feature) data_1['image_feature'] = li li = [] print(f'finished data_1_image_feature : {time.time() - s} sec') if use_nsml: bind_nsml(model, optimizer, args.task) if args.pause: nsml.paused(scope=locals()) if (args.mode == 'train') or args.dry_run: best_loss = 1000 if args.dry_run: print('start dry-running...!') args.num_epochs = 1 else: print('start training...!') # 미리 전체를 다 만들어놓자 굳이 generator 안써도 되겠네 model.compile( tf.keras.optimizers.Adam(args.lr), 'mse', metrics=['accuracy'], ) train_generator = data_generator(data_1) lr_scheduler = tf.keras.callbacks.LearningRateScheduler(scheduler) save_cbk = CustomModelCheckpoint() history = model.fit_generator(train_generator, epochs=100, verbose=2, workers=8, steps_per_epoch=np.ceil( len(data_1) / 2048), callbacks=[lr_scheduler, save_cbk]) print('again')
def main(args): if args.arch == 'xDeepFM': s = time.time() csv_file = os.path.join(DATASET_PATH, 'train', 'train_data', 'train_data') item = pd.read_csv(csv_file, dtype={ 'article_id': str, 'hh': int, 'gender': str, 'age_range': str, 'read_article_ids': str }, sep='\t') label_data_path = os.path.join( DATASET_PATH, 'train', os.path.basename(os.path.normpath(csv_file)).split('_')[0] + '_label') label = pd.read_csv(label_data_path, dtype={'label': int}, sep='\t') item['label'] = label s = time.time() print(f'before test article preprocess : {len(item)}') sparse_features = [ 'article_id', 'hh', 'gender', 'age_range', 'len_bin' ] dense_features = ['image_feature', 'read_cnt_prob'] target = ['label'] ############################ make more feature !!!!!!! ################################# ############## 1. read_article_ids len cnt -- user feature ################################################# len_lis = [] read_article_ids_all = item['read_article_ids'].tolist() for i in range(len(item)): li = read_article_ids_all[i] if type(li) == float: len_lis.append(0) continue len_li = len(li.split(',')) len_lis.append(len_li) item['len'] = len_lis item['len_bin'] = pd.qcut(item['len'], 6, duplicates='drop') id_to_artic = dict() artics = item['article_id'].tolist() ################ 2. read_cnt, total_cnt, prob_read_cnt --- article feature #################################### read_cnt = item[item['label'] == 1].groupby('article_id').agg( {'hh': 'count'}) read_cnt = read_cnt.reset_index() read_cnt = read_cnt.rename(columns={'hh': 'read_cnt'}) read_cnt_list = read_cnt['read_cnt'].tolist() read_cnt_artic_list = read_cnt['article_id'].tolist() print(f'len read_cnt : {len(read_cnt)}') print(read_cnt.head(3)) total_cnt = item.groupby('article_id').agg({'hh': 'count'}) total_cnt = total_cnt.reset_index() total_cnt = total_cnt.rename(columns={'hh': 'read_cnt'}) total_cnt_list = total_cnt['read_cnt'].tolist() total_cnt_artic_list = total_cnt['article_id'].tolist() print(f'len read_cnt : {len(total_cnt)}') print(total_cnt.head(3)) # lit # test_article_ids list lit_cnt = [] lit_total_cnt = [] lit_cnt_prob = [] lit = list(set(artics)) lit.sort() print(lit[:10]) print(f'len(lit):{len(lit)}') for i in range(len(lit)): # lit_cnt cur_artic = lit[i] if cur_artic not in read_cnt_artic_list: lit_cnt.append(0) else: for j in range(len(read_cnt_artic_list)): if cur_artic == read_cnt_artic_list[j]: lit_cnt.append(read_cnt_list[j]) break # lit_total_cnt if cur_artic not in total_cnt_artic_list: lit_total_cnt.append(0) else: for j in range(len(total_cnt_artic_list)): if cur_artic == total_cnt_artic_list[j]: lit_total_cnt.append(total_cnt_list[j]) break # lit_cnt_prob if lit_total_cnt[i] == 0: lit_cnt_prob.append(0) else: lit_cnt_prob.append(lit_cnt[i] / lit_total_cnt[i]) print('--- read_cnt article feature completed ---') print(f'lit_cnt {len(lit_cnt)}') print(f'lit_total_cnt {len(lit_total_cnt)}') print(f'lit_cnt_prob {len(lit_cnt_prob)}') #### fea print('feature dict generate') file_list1 = os.listdir(DATASET_PATH) file_list2 = os.listdir(DATASET_PATH + '/train') file_list3 = os.listdir(DATASET_PATH + '/train/train_data') print(file_list1) print(file_list2) print(file_list3) resnet_feature_extractor(args.mode) print(file_list1) print(file_list2) print(file_list3) # One hot Encoding with open(os.path.join('train_image_features_50.pkl'), 'rb') as handle: image_feature_dict = pickle.load(handle) print('check artic feature') print(f"757518f4a3da : {image_feature_dict['757518f4a3da']}") lbe = LabelEncoder() lbe.fit(lit) item['article_id' + '_onehot'] = lbe.transform(item['article_id']) print(lbe.classes_) for feat in sparse_features[1:]: lbe = LabelEncoder() item[feat + '_onehot'] = lbe.fit_transform( item[feat]) # 이때 고친 라벨이 같은 라벨인지도 필수로 확인해야함 print(item.head(10)) print('columns name : ', item.columns) fixlen_feature_columns = [SparseFeat('article_id', len(lit))] fixlen_feature_columns += [ SparseFeat(feat, item[feat + '_onehot'].nunique()) for feat in sparse_features[1:] ] fixlen_feature_columns += [ DenseFeat('image_feature', len(image_feature_dict[artics[0]])) ] fixlen_feature_columns += [DenseFeat('read_cnt_prob', 1)] print(f'fixlen_feature_columns : {fixlen_feature_columns}') idx_artics_all = item['article_id' + '_onehot'].tolist() for i in range(len(artics)): idx_artic = idx_artics_all[i] if idx_artic not in id_to_artic.keys(): id_to_artic[idx_artic] = artics[i] linear_feature_columns = fixlen_feature_columns dnn_feature_columns = fixlen_feature_columns fixlen_feature_names = get_fixlen_feature_names( linear_feature_columns + dnn_feature_columns) print(fixlen_feature_names) global fixlen_feature_names_global fixlen_feature_names_global = fixlen_feature_names model = xDeepFM(linear_feature_columns, dnn_feature_columns, task='binary') print('---model defined---') print(time.time() - s, 'seconds') ##### print need for artic in lit: print(artic, end=',') print() print('new') print() print(len(lit_cnt_prob)) for prob in lit_cnt_prob: prob = round(prob, 4) print(prob, end=',') print() print('end') print('--------------') optimizer = tf.keras.optimizers.Adam(args.lr) s = time.time() # negative sampling item_pos = item[item['label'] == 1] item_neg = item[item['label'] == 0] dn_1 = item_neg.sample(n=3 * len(item_pos), random_state=42) dn_2 = item_neg.sample(n=3 * len(item_pos), random_state=20) dn_3 = item_neg.sample(n=3 * len(item_pos), random_state=7) dn_4 = item_neg.sample(n=3 * len(item_pos), random_state=33) dn_5 = item_neg.sample(n=3 * len(item_pos), random_state=41) dn_1.reset_index() data_1 = pd.concat([dn_1, item_pos]).sample(frac=1, random_state=42).reset_index() data_1_article_idxs = data_1['article_id_onehot'].tolist() data_1_article = data_1['article_id'].tolist() print(f'len data_1 : {len(data_1)}') print(data_1.head(5)) li1 = [] li2 = [] li3 = [] for i in range(len(data_1_article)): for j in range(len(lit_cnt_prob)): if data_1_article[i] == lit[j]: li3.append(lit_cnt_prob[j]) break data_1['read_cnt_prob'] = li3 print('---read_cnt_prob end---') ## preprocess append data_2 = pd.concat([dn_2, item_pos]).sample(frac=1, random_state=42).reset_index() data_3 = pd.concat([dn_3, item_pos]).sample(frac=1, random_state=42).reset_index() data_4 = pd.concat([dn_4, item_pos]).sample(frac=1, random_state=42).reset_index() data_5 = pd.concat([dn_5, item_pos]).sample(frac=1, random_state=42).reset_index() li = [] for i in range(len(data_1_article_idxs)): image_feature = image_feature_dict[id_to_artic[data_1_article_idxs[i]]] li.append(image_feature) print(f'article_id : {data_1_article[0]}') print(f'article_image_feature : {image_feature_dict[data_1_article[0]]}') data_1['image_feature'] = li li = [] print(f'finished data_1_image_feature : {time.time() - s} sec') if use_nsml: bind_nsml(model, optimizer, args.task) if args.pause: nsml.paused(scope=locals()) if (args.mode == 'train') or args.dry_run: best_loss = 1000 if args.dry_run: print('start dry-running...!') args.num_epochs = 1 else: print('start training...!') model.compile( tf.keras.optimizers.Adam(args.lr), 'mse', metrics=['accuracy'], ) train_generator = data_generator(data_1) lr_scheduler = tf.keras.callbacks.LearningRateScheduler(scheduler) #k_fold 할때는 check point 빼자 save_cbk = CustomModelCheckpoint() history = model.fit_generator(train_generator, epochs=100, verbose=2, workers=8, steps_per_epoch=np.ceil( len(data_1) / 2048), callbacks=[lr_scheduler, save_cbk]) print('again')