print('Loading data start!') tf.set_random_seed(123) # load training data train_ft, train_label = func.tf_input_pipeline(train_file_name, batch_size, n_epoch, label_col_idx, record_defaults) n_val_inst = func.count_lines(val_file_name[0]) val_ft, val_label = func.tf_input_pipeline(val_file_name, n_val_inst, 1, label_col_idx, record_defaults) n_val_batch = n_val_inst // batch_size # load test data test_ft, test_label = func.tf_input_pipeline_test(test_file_name, batch_size, 1, label_col_idx, record_defaults) print('Loading data set 1 done!') # load training data train_ft_corr = func.tf_input_pipeline_wo_label(train_file_name_corr, batch_size_corr, n_epoch_corr, record_defaults_corr) print('Loading data set 2 done!') ######################################################################## # partition input for correlation loss def partition_input_corr(x_input_corr): # generate idx_list len_list = []
# remove dir if os.path.isdir(model_saving_addr): shutil.rmtree(model_saving_addr) ########################################################### ########################################################### # if input is tfrecord format print('Loading data start!') tf.set_random_seed(rnd_seed) if input_format == 'csv': # load data set 1 train_ft_1, train_label_1 = func.tf_input_pipeline(train_file_name_1, batch_size_1, n_epoch, \ label_col_idx_1, record_defaults_1) test_ft_1, test_label_1 = func.tf_input_pipeline_test(test_file_name_1, batch_size_1, 1, \ label_col_idx_1, record_defaults_1) # load data set 2 train_ft_2, train_label_2 = func.tf_input_pipeline(train_file_name_2, batch_size_2, n_epoch, \ label_col_idx_2, record_defaults_2) test_ft_2, test_label_2 = func.tf_input_pipeline_test(test_file_name_2, batch_size_2, 1, \ label_col_idx_2, record_defaults_2) elif input_format == 'tfrecord': train_ft_1, train_label_1 = func.tfrecord_input_pipeline(train_file_name_1, num_csv_col_1, \ batch_size_1, n_epoch) test_ft_1, test_label_1 = func.tfrecord_input_pipeline_test(test_file_name_1, num_csv_col_1, \ batch_size_1, 1) train_ft_2, train_label_2 = func.tfrecord_input_pipeline(train_file_name_2, num_csv_col_2, \ batch_size_2, n_epoch) test_ft_2, test_label_2 = func.tfrecord_input_pipeline_test(test_file_name_2, num_csv_col_2, \ batch_size_2, 1)