def __init__(self, output_path, es_patience, callbacks):
if output_path is None:
callbacks = None
final_model = None
cb = None
else:
# Make directory for outputs
check_mkdir(join_path(output_path, 'checkpoints'))
check_mkdir(join_path(output_path, 'models'))
checkpoint_model = join_path(output_path,
['checkpoints', 'epoch{epoch}.h5'])
best_model = join_path(output_path, ['models', 'best_model.h5'])
final_model = join_path(output_path, ['models', 'final_model.h5'])
cb = []
if 'es' in callbacks:
cb.append(
EarlyStopping(patience=es_patience,
restore_best_weights=True))
if 'checkpoint_model' in callbacks:
cb.append(
ModelCheckpoint(filepath=checkpoint_model,
monitor='val_loss'))
if 'best_model' in callbacks:
cb.append(
ModelCheckpoint(filepath=best_model,
monitor='val_loss',
save_best_only=True))
if len(cb) == 0:
cb = None
self.params = {
'batch_size': 10000,
'epochs': 1000,
'output_path': output_path,
'callbacks': cb,
'final_model': final_model
}
else:
extracted_pkl = join_path(args.data_folder,
['extracted_txt', 'extracted.pkl'])
params_path = join_path(args.data_folder, 'params.pkl')
try:
params = pickle_load(params_path)
except Exception:
params = {}
params['encode_data_test_ratio'] = args.test_ratio
params['encode_data_random_seed'] = args.random_seed
pickle_save(params, params_path)
dataset = pickle_load(extracted_pkl)
random.Random(1).shuffle(dataset)
test_size = int(len(dataset) * args.test_ratio)
test_set = dataset[:test_size]
train_set = dataset[test_size:]
train_path = join_path(args.data_folder, 'train_data')
check_mkdir(train_path)
print(f'Encoding training data: {len(train_set)}')
train_dict_word2idx = encode_knn_dataset(train_set, train_path)
test_path = join_path(args.data_folder, 'test_data')
check_mkdir(test_path)
print(f'Encoding test data: {len(test_set)}')
train_dict_word2idx = encode_knn_dataset(test_set, test_path)
print('Completed')
"--layers_act",
default=['tanh', 'linear', 'tanh', 'linear', 'tanh', 'linear'])
parser.add_argument("--epochs", default=10)
parser.add_argument("--es_patience", default=2)
args = parser.parse_args()
if args.data_folder is None:
model_output = args.model_output
tensor_folder = args.tensor_folder
else:
date = datetime.datetime.today().strftime('%Y_%m_%d')
if args.remark is not None:
date = date + '_' + args.remark
model_output = join_path(args.data_folder, ['models', date])
check_mkdir(model_output)
tensor_folder = join_path(args.data_folder, 'tensor')
tensor_list = listdir(tensor_folder)
model = DNN(model_output, es_patience=args.es_patience, callbacks=['es'])
if args.data_folder is not None:
params_path = join_path(args.data_folder, 'params.pkl')
try:
params = pickle_load(params_path)
except Exception:
params = {}
model_params = model.get_params()
model_params['epochs'] = args.epochs
model_params['es_patience'] = args.es_patience
parser.add_argument("--random_seed", type=int, default=1)
parser.add_argument("--chunk_size", type=int, default=300)
parser.add_argument("--mode", default='attention')
args = parser.parse_args()
if args.data_folder is None:
extracted_pkl = args.extracted_pkl
tensor_folder = args.tensor_folder
else:
extracted_pkl = join_path(args.data_folder,
['extracted_txt', 'extracted.pkl'])
tensor_folder = join_path(args.data_folder, ['tensor', args.mode])
params_path = join_path(args.data_folder, 'params.pkl')
check_mkdir(tensor_folder)
try:
params = pickle_load(params_path)
except Exception:
params = {}
params['encode_data_test_ratio'] = args.test_ratio
params['encode_data_random_seed'] = args.random_seed
params['encode_data_chunk_size'] = args.chunk_size
pickle_save(params, params_path)
dataset = pickle_load(extracted_pkl)
random.Random(1).shuffle(dataset)
test_size = int(len(dataset) * args.test_ratio)
test_set = dataset[:test_size]
train_set = dataset[test_size:]