def main():
config = load_config()
if config['random_seed']:
random.seed(config['random_seed'])
torch.manual_seed(config['random_seed'])
disable_cometml = not config['use_cometml']
experiment = Experiment(api_key=config['comet_key'], project_name=config['project_name'], disabled=disable_cometml)
model_config = config['model']
experiment.log_multiple_params(model_config)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
train_dataset = StockDataset(config, device=device, mode='train')
train_data_loader = DataLoader(train_dataset, batch_size=config['model']['batch_size'], shuffle=True)
dev_dataset = StockDataset(config, device=device, mode='dev')
dev_data_loader = DataLoader(dev_dataset, batch_size=config['model']['batch_size'], shuffle=False)
model = Model(input_size=model_config['input_size'],
hidden_size=model_config['hidden_size'],
output_size=model_config['output_size'],
num_layers=model_config['num_layers'],
dropout=model_config['dropout'],
device=device)
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=model_config['learning_rate'])
train_and_evaluate(model=model,
criterion=criterion,
optimizer=optimizer,
train_data_loader=train_data_loader,
dev_data_loader=dev_data_loader,
epochs_count=model_config['epochs_count'],
experiment=experiment)
def main():
# unused args provided by service
arguments = parse_args()
arguments.pop('job_dir', None)
arguments.pop('job-dir', None)
output_dir = arguments.pop('output_dir')
model.BUCKET = arguments.pop('bucket')
model.TRAIN_STEPS = arguments.pop('train_steps')
model.EVAL_INTERVAL = arguments.pop('eval_int')
model.PATTERN = arguments.pop('pattern')
model.BATCH_SIZE = arguments.pop('batch_size')
model.NUM_LAYERS = arguments.pop('num_layers')
model.FIRST_LAYER_SIZE = arguments.pop('first_layer_size')
model.N_EMBEDS = arguments.pop('n_embeds')
model.LEARNING_RATE = arguments.pop('learning_rate')
model.DROPOUT_RATE = arguments.pop('dropout_rate')
# Append trial_id to path if we are doing hptuning
# This code can be removed if you are not using hyperparameter tuning
config_json = json.loads(os.environ.get('TF_CONFIG', '{}'))
output_dir = os.path.join(output_dir,
config_json.get('task', {}).get('trial', ''))
# Run the training job
# learn_runner.run(model.experiment_fn, output_dir)
model.train_and_evaluate(output_dir)
def main():
"""Parses command line arguments and kicks off model training."""
args = _parse_arguments(sys.argv[1:])[0]
# Configure path for hyperparameter tuning.
trial_id = json.loads(
os.environ.get('TF_CONFIG', '{}')).get('task', {}).get('trial', '')
output_path = args.job_dir if not trial_id else args.job_dir + '/'
model_layers = model.get_layers(args.model_type)
image_model = model.build_model(model_layers, args.job_dir)
model_history = model.train_and_evaluate(
image_model, args.epochs, args.steps_per_epoch, args.job_dir)
# Created by Luis Alejandro ([email protected]). # Copyright © Do not distribute or use without authorization from author import argparse import model if __name__ == '__main__': # Creates argument parser parser = argparse.ArgumentParser() parser.add_argument('--state', help='US state to work with (Default: MI)', type=str) parser.add_argument('--version', help='Version of the datasets to load (Default: None)', type=int) parser.add_argument('--epochs', help='Number of epochs to train (Default: 100)', type=int) parser.add_argument('--model_type', help='Type of model use (Options: linear, dnn, rnn', type=str) args = parser.parse_args() model.train_and_evaluate(args)
run_config = tf.estimator.RunConfig(
tf_random_seed=19830610,
log_step_count_steps=1000,
save_checkpoints_secs=EVAL_EVERY_SEC,
keep_checkpoint_max=1,
model_dir=model_dir
)
model.BUCKET = arguments.pop('bucket')
model.BATCH_SIZE = arguments.pop('batch_size')
model.TRAIN_STEPS = (arguments.pop('train_examples') * 1000) / model.BATCH_SIZE
model.EVAL_STEPS = arguments.pop('eval_steps')
print ("Will train for {} steps using batch_size={}".format(model.TRAIN_STEPS, model.BATCH_SIZE))
model.PATTERN = arguments.pop('pattern')
model.NEMBEDS= arguments.pop('nembeds')
model.NNSIZE = arguments.pop('nnsize')
print ("Will use DNN size of {}".format(model.NNSIZE))
# Append trial_id to path if we are doing hptuning
# This code can be removed if you are not using hyperparameter tuning
output_dir = os.path.join(
output_dir,
json.loads(
os.environ.get('TF_CONFIG', '{}')
).get('task', {}).get('trial', '')
)
# Run the training job
model.train_and_evaluate(hparams, run_config)
parser.add_argument("--target_update_frequency", type=int, default=10000)
parser.add_argument("--gradient_update_frequency", type=int, default=4)
parser.add_argument("--dueling", type=bool, default=True)
parser.add_argument("--double_dqn", type=bool, default=False)
parser.add_argument("--record_video_every", type=int, default=1000)
parser.add_argument("--checkpoint_frequency", type=int, default=100000)
parser.add_argument("--strarting_fire_steps", type=int, default=10)
parser.add_argument("--random_seed", type=int, default=123)
args = parser.parse_args()
args = args.__dict__
output_dir = os.path.join(args["output_dir"], args["env_name"],
args["run_id"])
checkpoint_path = os.path.join(output_dir, "checkpoint")
monitor_path = os.path.join(output_dir, "monitor")
os.makedirs(output_dir, exist_ok=True)
os.makedirs(checkpoint_path, exist_ok=True)
os.makedirs(monitor_path, exist_ok=True)
checkpoint_weights_base = os.path.join(checkpoint_path,
'checkpoint_weights')
checkpoint_weights_filename = checkpoint_weights_base + ".h5f"
checkpoint_step_filename = checkpoint_weights_base + "_step.txt"
log_filename = os.path.join(output_dir, "log.txt")
weights_filename = os.path.join(checkpoint_path, 'final_weights.h5f')
model.train_and_evaluate(args, monitor_path, checkpoint_step_filename,
checkpoint_weights_filename, weights_filename,
log_filename)
# unused args provided by service
arguments.pop('job_dir', None)
arguments.pop('job-dir', None)
# assign the arguments to the model variables
model.outdir = arguments.pop('outdir')
model.BUCKET = arguments.pop('BUCKET')
model.NUM_EPOCH = arguments.pop('NUM_EPOCH')
model.BATCH_SIZE = arguments.pop('BATCH_SIZE')
model.SAVE_CKPT_STEPS = arguments.pop('SAVE_CKPT_STEPS')
model.EVAL_INTERVAL_SEC = arguments.pop('EVAL_INTERVAL_SEC')
model.HIDDEN_UNITS = arguments.pop('HIDDEN_UNITS')
model.MAX_STEPS = arguments.pop('MAX_STEPS')
model.LEARNING_RATE_LINEAR = arguments.pop('LEARNING_RATE_LINEAR')
model.LEARNING_RATE_DNN = arguments.pop('LEARNING_RATE_DNN')
model.model = arguments.pop('model')
# model.TRAIN_STEPS = (arguments.pop('train_examples') * 1000) / model.BATCH_SIZE
# model.EVAL_STEPS = arguments.pop('eval_steps')
print("Will train for {} steps using batch_size={}".format(
model.MAX_STEPS, model.BATCH_SIZE))
if model.model == "linear":
print("Will use linear model")
else:
print("Will use DNN size of {}".format(model.HIDDEN_UNITS))
# Append trial_id to path if we are doing hptuning
# Run the training job
model.train_and_evaluate(model.outdir, model.model)
args = parser.parse_args()
arguments = args.__dict__
# unused args provided by service
arguments.pop('job_dir', None)
arguments.pop('job-dir', None)
output_dir = arguments.pop('output_dir')
model.BUCKET = arguments.pop('bucket')
model.BATCH_SIZE = arguments.pop('batch_size')
model.TRAIN_STEPS = (arguments.pop('train_examples') *
1000) / model.BATCH_SIZE
model.EVAL_STEPS = arguments.pop('eval_steps')
print("Will train for {} steps using batch_size={}".format(
model.TRAIN_STEPS, model.BATCH_SIZE))
model.PATTERN = arguments.pop('pattern')
model.NEMBEDS = arguments.pop('nembeds')
model.NNSIZE = arguments.pop('nnsize')
print("Will use DNN size of {}".format(model.NNSIZE))
# Append trial_id to path if we are doing hptuning
# This code can be removed if you are not using hyperparameter tuning
output_dir = os.path.join(
output_dir,
json.loads(os.environ.get('TF_CONFIG', '{}')).get('task',
{}).get('trial', ''))
# Run the training job
model.train_and_evaluate(output_dir)
default=10)
parser.add_argument('--nfil2',
help='number of filters in second layer for CNN',
type=int,
default=20)
parser.add_argument('--dprob',
help='dropout probability for CNN',
type=float,
default=0.25)
parser.add_argument('--batch_norm',
help='if specified, do batch_norm for CNN',
dest='batch_norm',
action='store_true')
# boolean flags
parser.set_defaults(use_tpu=False, batch_norm=False)
args = parser.parse_args()
hparams = args.__dict__
print(hparams)
output_dir = hparams.pop('output_dir')
# Append trial_id to path for hptuning
output_dir = os.path.join(
output_dir,
json.loads(os.environ.get('TF_CONFIG', '{}')).get('task',
{}).get('trial', ''))
# Run the training job
model.train_and_evaluate(output_dir, hparams)
default=5)
parser.add_argument('--numTrees',
help='Number of trees',
type=int,
default=100)
parser.add_argument('--projectId',
help='ID (not name) of your project',
required=True)
parser.add_argument(
'--job-dir',
help='output directory for model, automatically provided by gcloud',
required=True)
args = parser.parse_args()
arguments = args.__dict__
model.PROJECT = arguments['projectId']
model.KEYDIR = 'trainer'
estimator, rmse = model.train_and_evaluate(arguments['frac'],
arguments['maxDepth'],
arguments['numTrees'])
loc = model.save_model(estimator, arguments['job_dir'], 'babyweight')
print("Saved model to {}".format(loc))
# this is for hyperparameter tuning
hpt = hypertune.HyperTune()
hpt.report_hyperparameter_tuning_metric(hyperparameter_metric_tag='rmse',
metric_value=rmse,
global_step=0)
# done
parser.add_argument(
'--job-dir',
help='this model ignores this field, but it is required by gcloud',
default='junk'
)
args = parser.parse_args()
arguments = args.__dict__
# unused args provided by service
arguments.pop('job_dir', None)
arguments.pop('job-dir', None)
output_dir = arguments.pop('output_dir')
model.BUCKET = arguments.pop('bucket')
model.TRAIN_STEPS = arguments.pop('train_steps')
model.PATTERN = arguments.pop('pattern')
# Append trial_id to path if we are doing hptuning
# This code can be removed if you are not using hyperparameter tuning
output_dir = os.path.join(
output_dir,
json.loads(
os.environ.get('TF_CONFIG', '{}')
).get('task', {}).get('trial', '')
)
# Run the training job
#learn_runner.run(model.experiment_fn, output_dir)
model.train_and_evaluate(output_dir)
"--num_evals",
help="Number of times to evaluate model on eval data training.",
type=int,
default=5)
parser.add_argument("--num_examples_to_train_on",
help="Number of examples to train on.",
type=int,
default=100)
parser.add_argument(
"--output_dir",
help="GCS location to write checkpoints and export models",
required=True)
parser.add_argument(
"--train_data_path",
help="GCS location pattern of train files containing eval URLs",
required=True)
parser.add_argument(
"--job-dir",
help="this model ignores this field, but it is required by gcloud",
default="junk")
args = parser.parse_args()
hparams = args.__dict__
hparams.pop("job-dir", None)
model.train_and_evaluate(hparams)
"""MNIST classifier.
Usage:
trainer.task --outdir <outdir> [--batch_size <batch_size>]
[--num_steps <num_steps>] [--hidden_units <hidden_units>]
[--use_batch_normalization] [--job-dir <job-dir>]
Options:
-h --help Show this screen.
--batch_size <batch_size> Batch size [default: 550]
--num_steps <num_steps> # training iterations [default: 100]
--hidden_units <hidden_units> Hidden units [default: 100]
--use_batch_normalization Use batch normalization
--job-dir <job-dir> This model ignores this field, but it's required for gcloud [default: blank]
"""
from docopt import docopt
#from . import model
import model
if __name__ == '__main__':
arguments = docopt(__doc__)
outdir = arguments['<outdir>']
model.NUM_STEPS = int(arguments['--num_steps'])
model.BATCH_SIZE = int(arguments['--batch_size'])
model.HIDDEN_UNITS = [
int(h) for h in arguments['--hidden_units'].split(',')
]
model.USE_BATCH_NORMALIZATION = arguments['--use_batch_normalization']
model.train_and_evaluate(outdir)
default=10,
type=int)
parser.add_argument(
'--min_eval_frequency',
help='Minimum number of training steps between evaluations',
default=1,
type=int)
args = parser.parse_args()
arguments = args.__dict__
# Unused args provided by service
arguments.pop('job_dir', None)
arguments.pop('job-dir', None)
output_dir = arguments.pop('output_dir')
# Append trial_id to path if we are doing hptuning
# This code can be removed if you are not using hyperparameter tuning
output_dir = os.path.join(
output_dir,
json.loads(os.environ.get('TF_CONFIG', '{}')).get('task',
{}).get('trial', ''))
# Run the training job
try:
shutil.rmtree(output_dir, ignore_errors=True) # start fresh each time
model.train_and_evaluate(output_dir, arguments['keras'])
except:
traceback.print_exc()
import model
model.train_and_evaluate("alex_net", optimizer="momentum")
"""Run a training job on Cloud ML Engine for a given use case.
Usage:
trainer.task --train_data_paths <train_data_paths> --output_dir <outdir>
[--batch_size <batch_size>] [--hidden_units <hidden_units>]
Options:
-h --help Show this screen.
--batch_size <batch_size> Integer value indiciating batch size [default: 150]
--hidden_units <hidden_units> CSV seperated integers indicating hidden layer
sizes. For a fully connected model.', [default: 100]
"""
from docopt import docopt
import model # Your model.py file.
if __name__ == '__main__':
arguments = docopt(__doc__)
# Assign model variables to commandline arguments
model.TRAIN_PATHS = arguments['<train_data_paths>']
model.BATCH_SIZE = int(arguments['--batch_size'])
model.HIDDEN_UNITS = [
int(h) for h in arguments['--hidden_units'].split(',')
]
model.OUTPUT_DIR = arguments['<outdir>']
# Run the training job
model.train_and_evaluate()
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import os
import model
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--bucket', help='GCS path to output.', required=True)
parser.add_argument('--frac',
help='Fraction of input to process',
type=float,
required=True)
parser.add_argument(
'--job-dir',
help='this model ignores this field, but it is required by gcloud',
default='junk')
args = parser.parse_args()
arguments = args.__dict__
estimator = model.train_and_evaluate(arguments['frac'])
loc = model.save_model(
estimator, 'gs://{}/babyweight/sklearn'.format(arguments['bucket']),
'babyweight')
print("Saved model to {}".format(loc))
# done
'--projectId',
help = 'ID (not name) of your project',
required = True
)
parser.add_argument(
'--job-dir',
help = 'output directory for model, automatically provided by gcloud',
required = True
)
args = parser.parse_args()
arguments = args.__dict__
model.PROJECT = arguments['projectId']
model.KEYDIR = 'trainer'
estimator, rmse = model.train_and_evaluate(arguments['frac'],
arguments['maxDepth'],
arguments['numTrees']
)
loc = model.save_model(estimator,
arguments['job_dir'], 'babyweight')
print("Saved model to {}".format(loc))
# this is for hyperparameter tuning
hpt = hypertune.HyperTune()
hpt.report_hyperparameter_tuning_metric(
hyperparameter_metric_tag='rmse',
metric_value=rmse,
global_step=0)
# done
type=int)
args = parser.parse_args()
arguments = args.__dict__
# Unused args provided by service
arguments.pop('job_dir', None)
arguments.pop('job-dir', None)
# Append trial_id to path if we are doing hptuning
# This code can be removed if you are not using hyperparameter tuning
arguments['output_dir'] = os.path.join(
arguments['output_dir'],
json.loads(os.environ.get('TF_CONFIG', '{}')).get('task',
{}).get('trial', ''))
arguments['video_hidden_units'] = [
int(x) for x in arguments['video_hidden_units'].split(' ')
]
arguments['frame_hidden_units'] = [
int(x) for x in arguments['frame_hidden_units'].split(' ')
]
# Run the training job
try:
shutil.rmtree(arguments['output_dir'],
ignore_errors=True) # start fresh each time
model.train_and_evaluate(arguments)
except:
traceback.print_exc()
import model
model.train_and_evaluate("vgg_16",
epochs=200,
train_batch_size=32,
learning_rate=1e-5,
optimizer="rmsprop")
import model
model.train_and_evaluate("aein_net")
int(x) for x in arguments["dec_dnn_hidden_units"].split(" ")
]
## LSTM Encoder-Decoder Autoencoder
arguments["enc_lstm_hidden_units"] = [
int(x) for x in arguments["enc_lstm_hidden_units"].split(" ")
]
arguments["dec_lstm_hidden_units"] = [
int(x) for x in arguments["dec_lstm_hidden_units"].split(" ")
]
arguments["lstm_dropout_output_keep_probs"] = [
float(x)
for x in arguments["lstm_dropout_output_keep_probs"].split(" ")
]
arguments["dnn_hidden_units"] = [
int(x) for x in arguments["dnn_hidden_units"].split(" ")
]
# Fix eps argument
arguments["eps"] = float(arguments["eps"])
# Append trial_id to path if we are doing hptuning
# This code can be removed if you are not using hyperparameter tuning
arguments["output_dir"] = os.path.join(
arguments["output_dir"],
json.loads(os.environ.get("TF_CONFIG", "{}")).get("task",
{}).get("trial", ""))
# Run the training job
train_and_evaluate(arguments)
'--topk',
help='In batch prediction, how many top items should we emit for each user?',
type=int,
default=3
)
parser.add_argument(
'--job-dir',
help='this model ignores this field, but it is required by gcloud',
default='junk'
)
args = parser.parse_args()
arguments = args.__dict__
# unused args provided by service
arguments.pop('job_dir', None)
arguments.pop('job-dir', None)
# Append trial_id to path if we are doing hptuning
# This code can be removed if you are not using hyperparameter tuning
output_dir = arguments['output_dir']
output_dir = os.path.join(
output_dir,
json.loads(
os.environ.get('TF_CONFIG', '{}')
).get('task', {}).get('trial', '')
)
# Run the training job
model.train_and_evaluate(arguments)
dest='batch_norm',
action='store_true')
parser.set_defaults(batch_norm=False)
args = parser.parse_args()
hparams = args.__dict__
# unused args provided by service
hparams.pop('job_dir', None)
hparams.pop('job-dir', None)
output_dir = hparams.pop('output_dir')
# Append trial_id to path so hptuning jobs don't overwrite eachother
output_dir = os.path.join(
output_dir,
json.loads(
os.environ.get('TF_CONFIG', '{}')
).get('task', {}).get('trial', '')
)
# calculate train_steps if not provided
if hparams['train_steps'] < 1:
# 10,000 steps at batch_size of 512
hparams['train_steps'] = (10000 * 512) // hparams['train_batch_size']
print "Training for {} steps".format(hparams['train_steps'])
# Run the training job
model.train_and_evaluate(output_dir, hparams)
