def main(): config = load_config() if config['random_seed']: random.seed(config['random_seed']) torch.manual_seed(config['random_seed']) disable_cometml = not config['use_cometml'] experiment = Experiment(api_key=config['comet_key'], project_name=config['project_name'], disabled=disable_cometml) model_config = config['model'] experiment.log_multiple_params(model_config) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') train_dataset = StockDataset(config, device=device, mode='train') train_data_loader = DataLoader(train_dataset, batch_size=config['model']['batch_size'], shuffle=True) dev_dataset = StockDataset(config, device=device, mode='dev') dev_data_loader = DataLoader(dev_dataset, batch_size=config['model']['batch_size'], shuffle=False) model = Model(input_size=model_config['input_size'], hidden_size=model_config['hidden_size'], output_size=model_config['output_size'], num_layers=model_config['num_layers'], dropout=model_config['dropout'], device=device) criterion = torch.nn.CrossEntropyLoss() optimizer = torch.optim.Adam(model.parameters(), lr=model_config['learning_rate']) train_and_evaluate(model=model, criterion=criterion, optimizer=optimizer, train_data_loader=train_data_loader, dev_data_loader=dev_data_loader, epochs_count=model_config['epochs_count'], experiment=experiment)
def main(): # unused args provided by service arguments = parse_args() arguments.pop('job_dir', None) arguments.pop('job-dir', None) output_dir = arguments.pop('output_dir') model.BUCKET = arguments.pop('bucket') model.TRAIN_STEPS = arguments.pop('train_steps') model.EVAL_INTERVAL = arguments.pop('eval_int') model.PATTERN = arguments.pop('pattern') model.BATCH_SIZE = arguments.pop('batch_size') model.NUM_LAYERS = arguments.pop('num_layers') model.FIRST_LAYER_SIZE = arguments.pop('first_layer_size') model.N_EMBEDS = arguments.pop('n_embeds') model.LEARNING_RATE = arguments.pop('learning_rate') model.DROPOUT_RATE = arguments.pop('dropout_rate') # Append trial_id to path if we are doing hptuning # This code can be removed if you are not using hyperparameter tuning config_json = json.loads(os.environ.get('TF_CONFIG', '{}')) output_dir = os.path.join(output_dir, config_json.get('task', {}).get('trial', '')) # Run the training job # learn_runner.run(model.experiment_fn, output_dir) model.train_and_evaluate(output_dir)
def main(): """Parses command line arguments and kicks off model training.""" args = _parse_arguments(sys.argv[1:])[0] # Configure path for hyperparameter tuning. trial_id = json.loads( os.environ.get('TF_CONFIG', '{}')).get('task', {}).get('trial', '') output_path = args.job_dir if not trial_id else args.job_dir + '/' model_layers = model.get_layers(args.model_type) image_model = model.build_model(model_layers, args.job_dir) model_history = model.train_and_evaluate( image_model, args.epochs, args.steps_per_epoch, args.job_dir)
# Created by Luis Alejandro ([email protected]). # Copyright © Do not distribute or use without authorization from author import argparse import model if __name__ == '__main__': # Creates argument parser parser = argparse.ArgumentParser() parser.add_argument('--state', help='US state to work with (Default: MI)', type=str) parser.add_argument('--version', help='Version of the datasets to load (Default: None)', type=int) parser.add_argument('--epochs', help='Number of epochs to train (Default: 100)', type=int) parser.add_argument('--model_type', help='Type of model use (Options: linear, dnn, rnn', type=str) args = parser.parse_args() model.train_and_evaluate(args)
run_config = tf.estimator.RunConfig( tf_random_seed=19830610, log_step_count_steps=1000, save_checkpoints_secs=EVAL_EVERY_SEC, keep_checkpoint_max=1, model_dir=model_dir ) model.BUCKET = arguments.pop('bucket') model.BATCH_SIZE = arguments.pop('batch_size') model.TRAIN_STEPS = (arguments.pop('train_examples') * 1000) / model.BATCH_SIZE model.EVAL_STEPS = arguments.pop('eval_steps') print ("Will train for {} steps using batch_size={}".format(model.TRAIN_STEPS, model.BATCH_SIZE)) model.PATTERN = arguments.pop('pattern') model.NEMBEDS= arguments.pop('nembeds') model.NNSIZE = arguments.pop('nnsize') print ("Will use DNN size of {}".format(model.NNSIZE)) # Append trial_id to path if we are doing hptuning # This code can be removed if you are not using hyperparameter tuning output_dir = os.path.join( output_dir, json.loads( os.environ.get('TF_CONFIG', '{}') ).get('task', {}).get('trial', '') ) # Run the training job model.train_and_evaluate(hparams, run_config)
parser.add_argument("--target_update_frequency", type=int, default=10000) parser.add_argument("--gradient_update_frequency", type=int, default=4) parser.add_argument("--dueling", type=bool, default=True) parser.add_argument("--double_dqn", type=bool, default=False) parser.add_argument("--record_video_every", type=int, default=1000) parser.add_argument("--checkpoint_frequency", type=int, default=100000) parser.add_argument("--strarting_fire_steps", type=int, default=10) parser.add_argument("--random_seed", type=int, default=123) args = parser.parse_args() args = args.__dict__ output_dir = os.path.join(args["output_dir"], args["env_name"], args["run_id"]) checkpoint_path = os.path.join(output_dir, "checkpoint") monitor_path = os.path.join(output_dir, "monitor") os.makedirs(output_dir, exist_ok=True) os.makedirs(checkpoint_path, exist_ok=True) os.makedirs(monitor_path, exist_ok=True) checkpoint_weights_base = os.path.join(checkpoint_path, 'checkpoint_weights') checkpoint_weights_filename = checkpoint_weights_base + ".h5f" checkpoint_step_filename = checkpoint_weights_base + "_step.txt" log_filename = os.path.join(output_dir, "log.txt") weights_filename = os.path.join(checkpoint_path, 'final_weights.h5f') model.train_and_evaluate(args, monitor_path, checkpoint_step_filename, checkpoint_weights_filename, weights_filename, log_filename)
# unused args provided by service arguments.pop('job_dir', None) arguments.pop('job-dir', None) # assign the arguments to the model variables model.outdir = arguments.pop('outdir') model.BUCKET = arguments.pop('BUCKET') model.NUM_EPOCH = arguments.pop('NUM_EPOCH') model.BATCH_SIZE = arguments.pop('BATCH_SIZE') model.SAVE_CKPT_STEPS = arguments.pop('SAVE_CKPT_STEPS') model.EVAL_INTERVAL_SEC = arguments.pop('EVAL_INTERVAL_SEC') model.HIDDEN_UNITS = arguments.pop('HIDDEN_UNITS') model.MAX_STEPS = arguments.pop('MAX_STEPS') model.LEARNING_RATE_LINEAR = arguments.pop('LEARNING_RATE_LINEAR') model.LEARNING_RATE_DNN = arguments.pop('LEARNING_RATE_DNN') model.model = arguments.pop('model') # model.TRAIN_STEPS = (arguments.pop('train_examples') * 1000) / model.BATCH_SIZE # model.EVAL_STEPS = arguments.pop('eval_steps') print("Will train for {} steps using batch_size={}".format( model.MAX_STEPS, model.BATCH_SIZE)) if model.model == "linear": print("Will use linear model") else: print("Will use DNN size of {}".format(model.HIDDEN_UNITS)) # Append trial_id to path if we are doing hptuning # Run the training job model.train_and_evaluate(model.outdir, model.model)
args = parser.parse_args() arguments = args.__dict__ # unused args provided by service arguments.pop('job_dir', None) arguments.pop('job-dir', None) output_dir = arguments.pop('output_dir') model.BUCKET = arguments.pop('bucket') model.BATCH_SIZE = arguments.pop('batch_size') model.TRAIN_STEPS = (arguments.pop('train_examples') * 1000) / model.BATCH_SIZE model.EVAL_STEPS = arguments.pop('eval_steps') print("Will train for {} steps using batch_size={}".format( model.TRAIN_STEPS, model.BATCH_SIZE)) model.PATTERN = arguments.pop('pattern') model.NEMBEDS = arguments.pop('nembeds') model.NNSIZE = arguments.pop('nnsize') print("Will use DNN size of {}".format(model.NNSIZE)) # Append trial_id to path if we are doing hptuning # This code can be removed if you are not using hyperparameter tuning output_dir = os.path.join( output_dir, json.loads(os.environ.get('TF_CONFIG', '{}')).get('task', {}).get('trial', '')) # Run the training job model.train_and_evaluate(output_dir)
default=10) parser.add_argument('--nfil2', help='number of filters in second layer for CNN', type=int, default=20) parser.add_argument('--dprob', help='dropout probability for CNN', type=float, default=0.25) parser.add_argument('--batch_norm', help='if specified, do batch_norm for CNN', dest='batch_norm', action='store_true') # boolean flags parser.set_defaults(use_tpu=False, batch_norm=False) args = parser.parse_args() hparams = args.__dict__ print(hparams) output_dir = hparams.pop('output_dir') # Append trial_id to path for hptuning output_dir = os.path.join( output_dir, json.loads(os.environ.get('TF_CONFIG', '{}')).get('task', {}).get('trial', '')) # Run the training job model.train_and_evaluate(output_dir, hparams)
default=5) parser.add_argument('--numTrees', help='Number of trees', type=int, default=100) parser.add_argument('--projectId', help='ID (not name) of your project', required=True) parser.add_argument( '--job-dir', help='output directory for model, automatically provided by gcloud', required=True) args = parser.parse_args() arguments = args.__dict__ model.PROJECT = arguments['projectId'] model.KEYDIR = 'trainer' estimator, rmse = model.train_and_evaluate(arguments['frac'], arguments['maxDepth'], arguments['numTrees']) loc = model.save_model(estimator, arguments['job_dir'], 'babyweight') print("Saved model to {}".format(loc)) # this is for hyperparameter tuning hpt = hypertune.HyperTune() hpt.report_hyperparameter_tuning_metric(hyperparameter_metric_tag='rmse', metric_value=rmse, global_step=0) # done
parser.add_argument( '--job-dir', help='this model ignores this field, but it is required by gcloud', default='junk' ) args = parser.parse_args() arguments = args.__dict__ # unused args provided by service arguments.pop('job_dir', None) arguments.pop('job-dir', None) output_dir = arguments.pop('output_dir') model.BUCKET = arguments.pop('bucket') model.TRAIN_STEPS = arguments.pop('train_steps') model.PATTERN = arguments.pop('pattern') # Append trial_id to path if we are doing hptuning # This code can be removed if you are not using hyperparameter tuning output_dir = os.path.join( output_dir, json.loads( os.environ.get('TF_CONFIG', '{}') ).get('task', {}).get('trial', '') ) # Run the training job #learn_runner.run(model.experiment_fn, output_dir) model.train_and_evaluate(output_dir)
"--num_evals", help="Number of times to evaluate model on eval data training.", type=int, default=5) parser.add_argument("--num_examples_to_train_on", help="Number of examples to train on.", type=int, default=100) parser.add_argument( "--output_dir", help="GCS location to write checkpoints and export models", required=True) parser.add_argument( "--train_data_path", help="GCS location pattern of train files containing eval URLs", required=True) parser.add_argument( "--job-dir", help="this model ignores this field, but it is required by gcloud", default="junk") args = parser.parse_args() hparams = args.__dict__ hparams.pop("job-dir", None) model.train_and_evaluate(hparams)
"""MNIST classifier. Usage: trainer.task --outdir <outdir> [--batch_size <batch_size>] [--num_steps <num_steps>] [--hidden_units <hidden_units>] [--use_batch_normalization] [--job-dir <job-dir>] Options: -h --help Show this screen. --batch_size <batch_size> Batch size [default: 550] --num_steps <num_steps> # training iterations [default: 100] --hidden_units <hidden_units> Hidden units [default: 100] --use_batch_normalization Use batch normalization --job-dir <job-dir> This model ignores this field, but it's required for gcloud [default: blank] """ from docopt import docopt #from . import model import model if __name__ == '__main__': arguments = docopt(__doc__) outdir = arguments['<outdir>'] model.NUM_STEPS = int(arguments['--num_steps']) model.BATCH_SIZE = int(arguments['--batch_size']) model.HIDDEN_UNITS = [ int(h) for h in arguments['--hidden_units'].split(',') ] model.USE_BATCH_NORMALIZATION = arguments['--use_batch_normalization'] model.train_and_evaluate(outdir)
default=10, type=int) parser.add_argument( '--min_eval_frequency', help='Minimum number of training steps between evaluations', default=1, type=int) args = parser.parse_args() arguments = args.__dict__ # Unused args provided by service arguments.pop('job_dir', None) arguments.pop('job-dir', None) output_dir = arguments.pop('output_dir') # Append trial_id to path if we are doing hptuning # This code can be removed if you are not using hyperparameter tuning output_dir = os.path.join( output_dir, json.loads(os.environ.get('TF_CONFIG', '{}')).get('task', {}).get('trial', '')) # Run the training job try: shutil.rmtree(output_dir, ignore_errors=True) # start fresh each time model.train_and_evaluate(output_dir, arguments['keras']) except: traceback.print_exc()
import model model.train_and_evaluate("alex_net", optimizer="momentum")
"""Run a training job on Cloud ML Engine for a given use case. Usage: trainer.task --train_data_paths <train_data_paths> --output_dir <outdir> [--batch_size <batch_size>] [--hidden_units <hidden_units>] Options: -h --help Show this screen. --batch_size <batch_size> Integer value indiciating batch size [default: 150] --hidden_units <hidden_units> CSV seperated integers indicating hidden layer sizes. For a fully connected model.', [default: 100] """ from docopt import docopt import model # Your model.py file. if __name__ == '__main__': arguments = docopt(__doc__) # Assign model variables to commandline arguments model.TRAIN_PATHS = arguments['<train_data_paths>'] model.BATCH_SIZE = int(arguments['--batch_size']) model.HIDDEN_UNITS = [ int(h) for h in arguments['--hidden_units'].split(',') ] model.OUTPUT_DIR = arguments['<outdir>'] # Run the training job model.train_and_evaluate()
# Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import argparse import os import model if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--bucket', help='GCS path to output.', required=True) parser.add_argument('--frac', help='Fraction of input to process', type=float, required=True) parser.add_argument( '--job-dir', help='this model ignores this field, but it is required by gcloud', default='junk') args = parser.parse_args() arguments = args.__dict__ estimator = model.train_and_evaluate(arguments['frac']) loc = model.save_model( estimator, 'gs://{}/babyweight/sklearn'.format(arguments['bucket']), 'babyweight') print("Saved model to {}".format(loc)) # done
'--projectId', help = 'ID (not name) of your project', required = True ) parser.add_argument( '--job-dir', help = 'output directory for model, automatically provided by gcloud', required = True ) args = parser.parse_args() arguments = args.__dict__ model.PROJECT = arguments['projectId'] model.KEYDIR = 'trainer' estimator, rmse = model.train_and_evaluate(arguments['frac'], arguments['maxDepth'], arguments['numTrees'] ) loc = model.save_model(estimator, arguments['job_dir'], 'babyweight') print("Saved model to {}".format(loc)) # this is for hyperparameter tuning hpt = hypertune.HyperTune() hpt.report_hyperparameter_tuning_metric( hyperparameter_metric_tag='rmse', metric_value=rmse, global_step=0) # done
type=int) args = parser.parse_args() arguments = args.__dict__ # Unused args provided by service arguments.pop('job_dir', None) arguments.pop('job-dir', None) # Append trial_id to path if we are doing hptuning # This code can be removed if you are not using hyperparameter tuning arguments['output_dir'] = os.path.join( arguments['output_dir'], json.loads(os.environ.get('TF_CONFIG', '{}')).get('task', {}).get('trial', '')) arguments['video_hidden_units'] = [ int(x) for x in arguments['video_hidden_units'].split(' ') ] arguments['frame_hidden_units'] = [ int(x) for x in arguments['frame_hidden_units'].split(' ') ] # Run the training job try: shutil.rmtree(arguments['output_dir'], ignore_errors=True) # start fresh each time model.train_and_evaluate(arguments) except: traceback.print_exc()
import model model.train_and_evaluate("vgg_16", epochs=200, train_batch_size=32, learning_rate=1e-5, optimizer="rmsprop")
import model model.train_and_evaluate("aein_net")
int(x) for x in arguments["dec_dnn_hidden_units"].split(" ") ] ## LSTM Encoder-Decoder Autoencoder arguments["enc_lstm_hidden_units"] = [ int(x) for x in arguments["enc_lstm_hidden_units"].split(" ") ] arguments["dec_lstm_hidden_units"] = [ int(x) for x in arguments["dec_lstm_hidden_units"].split(" ") ] arguments["lstm_dropout_output_keep_probs"] = [ float(x) for x in arguments["lstm_dropout_output_keep_probs"].split(" ") ] arguments["dnn_hidden_units"] = [ int(x) for x in arguments["dnn_hidden_units"].split(" ") ] # Fix eps argument arguments["eps"] = float(arguments["eps"]) # Append trial_id to path if we are doing hptuning # This code can be removed if you are not using hyperparameter tuning arguments["output_dir"] = os.path.join( arguments["output_dir"], json.loads(os.environ.get("TF_CONFIG", "{}")).get("task", {}).get("trial", "")) # Run the training job train_and_evaluate(arguments)
'--topk', help='In batch prediction, how many top items should we emit for each user?', type=int, default=3 ) parser.add_argument( '--job-dir', help='this model ignores this field, but it is required by gcloud', default='junk' ) args = parser.parse_args() arguments = args.__dict__ # unused args provided by service arguments.pop('job_dir', None) arguments.pop('job-dir', None) # Append trial_id to path if we are doing hptuning # This code can be removed if you are not using hyperparameter tuning output_dir = arguments['output_dir'] output_dir = os.path.join( output_dir, json.loads( os.environ.get('TF_CONFIG', '{}') ).get('task', {}).get('trial', '') ) # Run the training job model.train_and_evaluate(arguments)
dest='batch_norm', action='store_true') parser.set_defaults(batch_norm=False) args = parser.parse_args() hparams = args.__dict__ # unused args provided by service hparams.pop('job_dir', None) hparams.pop('job-dir', None) output_dir = hparams.pop('output_dir') # Append trial_id to path so hptuning jobs don't overwrite eachother output_dir = os.path.join( output_dir, json.loads( os.environ.get('TF_CONFIG', '{}') ).get('task', {}).get('trial', '') ) # calculate train_steps if not provided if hparams['train_steps'] < 1: # 10,000 steps at batch_size of 512 hparams['train_steps'] = (10000 * 512) // hparams['train_batch_size'] print "Training for {} steps".format(hparams['train_steps']) # Run the training job model.train_and_evaluate(output_dir, hparams)