# remove dir if os.path.isdir(model_saving_addr): shutil.rmtree(model_saving_addr) # for DNN idx_1 = n_one_hot_slot idx_2 = idx_1 + n_mul_hot_slot * max_len_per_slot ########################################################### ########################################################### print('Loading data start!') tf.set_random_seed(123) # load training data train_ft, train_label = func.tf_input_pipeline(train_file_name, batch_size, n_epoch, label_col_idx, record_defaults) n_val_inst = func.count_lines(val_file_name[0]) val_ft, val_label = func.tf_input_pipeline(val_file_name, n_val_inst, 1, label_col_idx, record_defaults) n_val_batch = n_val_inst // batch_size # load test data test_ft, test_label = func.tf_input_pipeline_test(test_file_name, batch_size, 1, label_col_idx, record_defaults) print('Loading data set 1 done!') # load training data train_ft_corr = func.tf_input_pipeline_wo_label(train_file_name_corr,
if not os.path.exists(base_path): os.mkdir(base_path) # remove dir if os.path.isdir(model_saving_addr): shutil.rmtree(model_saving_addr) ########################################################### ########################################################### # if input is tfrecord format print('Loading data start!') tf.set_random_seed(rnd_seed) if input_format == 'csv': # load data set 1 train_ft_1, train_label_1 = func.tf_input_pipeline(train_file_name_1, batch_size_1, n_epoch, \ label_col_idx_1, record_defaults_1) test_ft_1, test_label_1 = func.tf_input_pipeline_test(test_file_name_1, batch_size_1, 1, \ label_col_idx_1, record_defaults_1) # load data set 2 train_ft_2, train_label_2 = func.tf_input_pipeline(train_file_name_2, batch_size_2, n_epoch, \ label_col_idx_2, record_defaults_2) test_ft_2, test_label_2 = func.tf_input_pipeline_test(test_file_name_2, batch_size_2, 1, \ label_col_idx_2, record_defaults_2) elif input_format == 'tfrecord': train_ft_1, train_label_1 = func.tfrecord_input_pipeline(train_file_name_1, num_csv_col_1, \ batch_size_1, n_epoch) test_ft_1, test_label_1 = func.tfrecord_input_pipeline_test(test_file_name_1, num_csv_col_1, \ batch_size_1, 1) train_ft_2, train_label_2 = func.tfrecord_input_pipeline(train_file_name_2, num_csv_col_2, \ batch_size_2, n_epoch)