def main(args):
if 'sourcedir.tar.gz' in args.tensorboard_dir:
tensorboard_dir = re.sub('source/sourcedir.tar.gz', 'model',
args.tensorboard_dir)
else:
tensorboard_dir = args.tensorboard_dir
logging.info("Writing TensorBoard logs to {}".format(tensorboard_dir))
logging.info("getting data")
train_dataset = process_input(args.epochs, args.batch_size, args.train,
'train', args.data_config)
eval_dataset = process_input(args.epochs, args.batch_size, args.eval,
'eval', args.data_config)
validation_dataset = process_input(args.epochs, args.batch_size,
args.validation, 'validation',
args.data_config)
logging.info("configuring model")
logging.info("Hosts: " + os.environ.get('SM_HOSTS'))
size = len(args.hosts)
#Deal with this
model = get_model(args.learning_rate, args.weight_decay, args.optimizer,
args.momentum, size)
callbacks = []
if args.current_host == args.hosts[0]:
callbacks.append(
ModelCheckpoint(args.output_data_dir + '/checkpoint-{epoch}.h5'))
callbacks.append(CustomTensorBoardCallback(log_dir=tensorboard_dir))
logging.info("Starting training")
history = model.fit(
x=train_dataset[0],
y=train_dataset[1],
steps_per_epoch=(num_examples_per_epoch('train') // args.batch_size) //
size,
epochs=args.epochs,
validation_data=validation_dataset,
validation_steps=(num_examples_per_epoch('validation') //
args.batch_size) // size,
callbacks=callbacks)
score = model.evaluate(eval_dataset[0],
eval_dataset[1],
steps=num_examples_per_epoch('eval') //
args.batch_size,
verbose=0)
logging.info('Test loss:{}'.format(score[0]))
logging.info('Test accuracy:{}'.format(score[1]))
# PS: Save model and history only on worker 0
if args.current_host == args.hosts[0]:
save_history(args.model_dir + "/ps_history.p", history)
save_model(model, args.model_dir)
def main(args):
mpi = False
if 'sourcedir.tar.gz' in args.tensorboard_dir:
tensorboard_dir = re.sub('source/sourcedir.tar.gz', 'model', args.tensorboard_dir)
else:
tensorboard_dir = args.tensorboard_dir
logging.info("Writing TensorBoard logs to {}".format(tensorboard_dir))
if 'sagemaker_mpi_enabled' in args.fw_params:
if args.fw_params['sagemaker_mpi_enabled']:
import horovod.tensorflow.keras as hvd
mpi = True
# Horovod: initialize Horovod.
hvd.init()
# Horovod: pin GPU to be used to process local rank (one GPU per process)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.gpu_options.visible_device_list = str(hvd.local_rank())
K.set_session(tf.Session(config=config))
else:
hvd = None
logging.info("Running with MPI={}".format(mpi))
logging.info("getting data")
train_dataset = process_input(args.epochs, args.batch_size, args.train, 'train', args.data_config)
eval_dataset = process_input(args.epochs, args.batch_size, args.eval, 'eval', args.data_config)
validation_dataset = process_input(args.epochs, args.batch_size, args.validation, 'validation', args.data_config)
logging.info("configuring model")
model = get_model(args.learning_rate, args.weight_decay, args.optimizer, args.momentum, 1, mpi, hvd)
callbacks = []
if mpi:
callbacks.append(hvd.callbacks.BroadcastGlobalVariablesCallback(0))
callbacks.append(hvd.callbacks.MetricAverageCallback())
callbacks.append(hvd.callbacks.LearningRateWarmupCallback(warmup_epochs=5, verbose=1))
callbacks.append(tf.keras.callbacks.ReduceLROnPlateau(patience=10, verbose=1))
if hvd.rank() == 0:
callbacks.append(ModelCheckpoint(args.output_data_dir + '/checkpoint-{epoch}.h5'))
callbacks.append(CustomTensorBoardCallback(log_dir=tensorboard_dir))
else:
callbacks.append(ModelCheckpoint(args.output_data_dir + '/checkpoint-{epoch}.h5'))
callbacks.append(CustomTensorBoardCallback(log_dir=tensorboard_dir))
logging.info("Starting training")
size = 1
if mpi:
size = hvd.size()
history = model.fit(x=train_dataset[0],
y=train_dataset[1],
steps_per_epoch=(num_examples_per_epoch('train') // args.batch_size) // size,
epochs=args.epochs,
validation_data=validation_dataset,
validation_steps=(num_examples_per_epoch('validation') // args.batch_size) // size,
callbacks=callbacks)
score = model.evaluate(eval_dataset[0],
eval_dataset[1],
steps=num_examples_per_epoch('eval') // args.batch_size,
verbose=0)
logging.info('Test loss:{}'.format(score[0]))
logging.info('Test accuracy:{}'.format(score[1]))
# Horovod: Save model and history only on worker 0 (i.e. master)
if mpi:
if hvd.rank() == 0:
save_history(args.model_dir + "/hvd_history.p", history)
return save_model(model, args.model_output_dir)
else:
save_history(args.model_dir + "/hvd_history.p", history)
return save_model(model, args.model_output_dir)
for i in range(1, n+1):
if i == 1: res = a % p
else:
res = pow(res, i, p)
return res
def a_N_frac(a, n, p):
return large_a_n_frac(a, n, p)
assert reminder(10, 1) == 0
assert reminder(10, 2) == 0
assert reminder(10, 5) == 0
assert reminder(10, 3) == 1
assert frac(3) == 6
assert frac(2) == 2
assert a_N_frac(2, 1, 2) == 0
assert a_N_frac(3, 3, 2) == 1
if __name__ == '__main__':
# file_n = 'data/A-small-practice.in'
file_n = 'data/A-large-practice.in'
# result = 'small_output.txt'
process_input(main_f=large_a_n_frac, input_file=file_n)
# small_input_validation(a_N_frac, input_file=file_n, result_file=result)
def main(args):
if 'sourcedir.tar.gz' in args.tensorboard_dir:
tensorboard_dir = re.sub('source/sourcedir.tar.gz', 'model',
args.tensorboard_dir)
else:
tensorboard_dir = args.tensorboard_dir
logging.info("Writing TensorBoard logs to {}".format(tensorboard_dir))
logging.info("getting data")
train_dataset = process_input(args.epochs, args.batch_size, args.train,
'train', args.data_config)
eval_dataset = process_input(args.epochs, args.batch_size, args.eval,
'eval', args.data_config)
validation_dataset = process_input(args.epochs, args.batch_size,
args.validation, 'validation',
args.data_config)
crime_lr = 1.1999607522739098e-06
diversity_lr = 1.599953611730598e-06
optimizer = tf.keras.optimizers.Adam(learning_rate=crime_lr)
### build optimizer
#steps_per_epoch = tf.data.experimental.cardinality(train_dataset).numpy()
#num_train_steps = steps_per_epoch * args.epochs
#num_warmup_steps = int(0.1*num_train_steps)
#init_lr = 5e-5
#optimizer = optimization.create_optimizer(
# init_lr=init_lr,
# num_train_steps=num_train_steps,
# num_warmup_steps=num_warmup_steps,
# optimizer_type='adamw')
###
#print('train_dataset type:', type(train_dataset))
#print('train_dataset text type:', type(train_dataset['text']))
#print('train_dataset label type:', type(train_dataset['label']))
#print('eval_dataset type:', type(eval_dataset))
#print('eval_dataset text type:', type(eval_dataset['text']))
#print('eval_dataset label type:', type(eval_dataset['label']))
#print('validation_dataset type:', type(validation_dataset))
#print('validation_dataset text type:', type(validation_dataset['text']))
#print('validation_dataset label type:', type(validation_dataset['label']))
logging.info("configuring model")
logging.info("Hosts: " + os.environ.get('SM_HOSTS'))
size = len(args.hosts)
model = get_model(args.learning_rate, args.weight_decay, optimizer,
args.momentum, size)
callbacks = []
if args.current_host == args.hosts[0]:
callbacks.append(
ModelCheckpoint(args.output_data_dir + '/checkpoint-{epoch}.h5'))
#callbacks.append(CustomTensorBoardCallback(log_dir=tensorboard_dir))
logging.info("Starting training")
#history = model.fit(
# x=train_dataset,
# steps_per_epoch=(num_examples_per_epoch('train') // args.batch_size) // size,
# epochs=args.epochs,
# validation_data=validation_dataset,
# validation_steps=(num_examples_per_epoch('validation') // args.batch_size) // size,
# callbacks=callbacks)
history = model.fit(x=train_dataset,
epochs=args.epochs,
validation_data=validation_dataset,
callbacks=callbacks)
#score = model.evaluate(
# eval_dataset,
# steps=num_examples_per_epoch('eval') // args.batch_size,
# verbose=0)
score = model.evaluate(eval_dataset)
logging.info('Test loss:{}'.format(score[0]))
logging.info('Test accuracy:{}'.format(score[1]))
# PS: Save model and history only on worker 0
if args.current_host == args.hosts[0]:
save_history(args.model_dir + "/ps_history.p", history)
save_model(model, args.model_dir)