# remove dir
if os.path.isdir(model_saving_addr):
shutil.rmtree(model_saving_addr)
# for DNN
idx_1 = n_one_hot_slot
idx_2 = idx_1 + n_mul_hot_slot * max_len_per_slot
###########################################################
###########################################################
print('Loading data start!')
tf.set_random_seed(123)
# load training data
train_ft, train_label = func.tf_input_pipeline(train_file_name, batch_size,
n_epoch, label_col_idx,
record_defaults)
n_val_inst = func.count_lines(val_file_name[0])
val_ft, val_label = func.tf_input_pipeline(val_file_name, n_val_inst, 1,
label_col_idx, record_defaults)
n_val_batch = n_val_inst // batch_size
# load test data
test_ft, test_label = func.tf_input_pipeline_test(test_file_name, batch_size,
1, label_col_idx,
record_defaults)
print('Loading data set 1 done!')
# load training data
train_ft_corr = func.tf_input_pipeline_wo_label(train_file_name_corr,
if not os.path.exists(base_path):
os.mkdir(base_path)
# remove dir
if os.path.isdir(model_saving_addr):
shutil.rmtree(model_saving_addr)
###########################################################
###########################################################
# if input is tfrecord format
print('Loading data start!')
tf.set_random_seed(rnd_seed)
if input_format == 'csv':
# load data set 1
train_ft_1, train_label_1 = func.tf_input_pipeline(train_file_name_1, batch_size_1, n_epoch, \
label_col_idx_1, record_defaults_1)
test_ft_1, test_label_1 = func.tf_input_pipeline_test(test_file_name_1, batch_size_1, 1, \
label_col_idx_1, record_defaults_1)
# load data set 2
train_ft_2, train_label_2 = func.tf_input_pipeline(train_file_name_2, batch_size_2, n_epoch, \
label_col_idx_2, record_defaults_2)
test_ft_2, test_label_2 = func.tf_input_pipeline_test(test_file_name_2, batch_size_2, 1, \
label_col_idx_2, record_defaults_2)
elif input_format == 'tfrecord':
train_ft_1, train_label_1 = func.tfrecord_input_pipeline(train_file_name_1, num_csv_col_1, \
batch_size_1, n_epoch)
test_ft_1, test_label_1 = func.tfrecord_input_pipeline_test(test_file_name_1, num_csv_col_1, \
batch_size_1, 1)
train_ft_2, train_label_2 = func.tfrecord_input_pipeline(train_file_name_2, num_csv_col_2, \
batch_size_2, n_epoch)
