def init(args): """ Load data, build model, create optimizer, create vars to hold metrics, etc. """ #need to handle really large text fields csv.field_size_limit(sys.maxsize) #load vocab and other lookups desc_embed = args.lmbda > 0 print("loading lookups...") dicts = datasets.load_lookups(args, desc_embed=desc_embed) model = tools.pick_model(args, dicts) print(model) if not args.test_model: optimizer = optim.Adam(model.parameters(), weight_decay=args.weight_decay, lr=args.lr) else: optimizer = None params = tools.make_param_dict(args) return args, model, optimizer, params, dicts
def init(args): """ Load data, build model, create optimizer, create vars to hold metrics, etc. """ #need to handle really large text fields csv.field_size_limit( sys.maxsize) # Sets field size to max available for strings # freq_params = None # if args.samples or args.lmbda > 0: # print("loading code frequencies...") # code_freqs, n = datasets.load_code_freqs(args.data_path, version=args.version) # print("code_freqs:", sorted(code_freqs.items(), key=operator.itemgetter(1), reverse=True)[:10], "n:", n) # freq_params = (code_freqs, n) #load vocab and other lookups # desc_embed = args.lmbda > 0 # dicts = datasets.load_lookups(args.data_path, args.vocab, Y=args.Y, desc_embed=desc_embed, version=args.version) # LOAD VOCAB DICTS dicts = datasets.load_vocab_dict( args.vocab_path) # CHANGED args.vocab --> args.vocab_path model = tools.pick_model(args, dicts) print(model) optimizer = optim.Adam(model.params_to_optimize(), weight_decay=args.weight_decay, lr=args.lr) params = tools.make_param_dict(args) return args, model, optimizer, params, dicts
def train_epochs(args, model, optimizer, params, dicts): """ Main loop. does train and test """ metrics_hist = defaultdict(lambda: []) metrics_hist_te = defaultdict(lambda: []) metrics_hist_tr = defaultdict(lambda: []) test_only = args.test_model is not None evaluate = args.test_model is not None #train for n_epochs unless criterion metric does not improve for [patience] epochs for epoch in range(args.n_epochs): #only test on train/test set on very last epoch if epoch == 0 and not args.test_model: model_dir = os.path.join( MODEL_DIR, '_'.join([ args.model, time.strftime('%b_%d_%H:%M:%S', time.localtime()) ])) os.mkdir(model_dir) elif args.test_model: model_dir = os.path.dirname(os.path.abspath(args.test_model)) metrics_all = one_epoch(model, optimizer, args.Y, epoch, args.n_epochs, args.batch_size, args.data_path, args.version, test_only, dicts, model_dir, args.samples, args.gpu, args.quiet) for name in metrics_all[0].keys(): metrics_hist[name].append(metrics_all[0][name]) for name in metrics_all[1].keys(): metrics_hist_te[name].append(metrics_all[1][name]) for name in metrics_all[2].keys(): metrics_hist_tr[name].append(metrics_all[2][name]) metrics_hist_all = (metrics_hist, metrics_hist_te, metrics_hist_tr) #save metrics, model, params persistence.save_everything(args, metrics_hist_all, model, model_dir, params, args.criterion, evaluate) if test_only: #we're done break if args.criterion in metrics_hist.keys(): if early_stop(metrics_hist, args.criterion, args.patience): #stop training, do tests on test and train sets, and then stop the script print("%s hasn't improved in %d epochs, early stopping..." % (args.criterion, args.patience)) test_only = True args.test_model = '%s/model_best_%s.pth' % (model_dir, args.criterion) model = tools.pick_model(args, dicts) return epoch + 1
def init(args): """ Load data, build model, create optimizer, create vars to hold metrics, etc. """ #need to handle really large text fields csv.field_size_limit(sys.maxsize) #load vocab and other lookups desc_embed = args.lmbda > 0 print("loading lookups...") dicts = datasets.load_lookups(args, desc_embed=desc_embed) META_TEST = args.test_model is not None model, start_epoch, optimizer = tools.pick_model(args, dicts, META_TEST) print(model) params = tools.make_param_dict(args) return args, model, optimizer, params, dicts, start_epoch
def init(args): """ Load data, build model, create optimizer, create vars to hold metrics, etc. """ #need to handle really large text fields csv.field_size_limit( sys.maxsize) # Sets field size to max available for strings # LOAD VOCAB DICTS dicts = datasets.load_vocab_dict( args.vocab_path) # CHANGED args.vocab --> args.vocab_path model = tools.pick_model(args, dicts) print(model) optimizer = optim.Adam(model.params_to_optimize(), weight_decay=args.weight_decay, lr=args.lr) params = tools.make_param_dict(args) return args, model, optimizer, params, dicts
def train_epochs(args, model, optimizer, params, dicts, start_epoch): """ Main loop. does train and test """ metrics_hist = defaultdict(lambda: []) metrics_hist_te = defaultdict(lambda: []) metrics_hist_tr = defaultdict(lambda: []) if args.reload_model: #get existing metrics up to model checkpoint: with open(os.path.join(args.reload_model, 'metrics.json'), 'r') as f: metrics_hist_all = json.load(f) #this will be a little expensive-- iterate through and re-assign: for el in metrics_hist_all: if '_te' in el: metrics_hist_te[re.sub( '_te$', '', el)] = metrics_hist_all[el][:start_epoch] elif '_tr' in el: metrics_hist_tr[re.sub( '_tr$', '', el)] = metrics_hist_all[el][:start_epoch] else: #dev metrics_hist[el] = metrics_hist_all[el][:start_epoch] META_TEST = args.test_model is not None test_only = args.test_model is not None evaluate = args.test_model is not None #train for n_epochs unless criterion metric does not improve for [patience] epochs for epoch in range( start_epoch, args.n_epochs ): #only train for _x_ more epochs from best-saved model* #only test on train/test set on very last epoch if epoch == start_epoch and not args.test_model: model_dir = os.path.join( MODEL_DIR, '_'.join([ args.model, time.strftime('%b_%d_%H:%M', time.localtime()) ])) os.mkdir(model_dir) #save model versioning (git) info: repo = git.Repo(search_parent_directories=True) branch = repo.active_branch.name print("branch:", branch) sha = repo.head.object.hexsha print("SHA hash:", sha) persistence.save_git_versioning_info( model_dir, (branch, sha, args.description)) elif args.test_model: model_dir = os.path.dirname(os.path.abspath(args.test_model)) metrics_all = one_epoch(args, model, optimizer, epoch, test_only, dicts, model_dir, META_TEST) for name in metrics_all[0].keys(): metrics_hist[name].append(metrics_all[0][name]) for name in metrics_all[1].keys(): metrics_hist_te[name].append(metrics_all[1][name]) for name in metrics_all[2].keys(): metrics_hist_tr[name].append(metrics_all[2][name]) metrics_hist_all = (metrics_hist, metrics_hist_te, metrics_hist_tr) #save metrics, model, params assert '-' not in args.criterion persistence.save_everything(args, metrics_hist_all, model, model_dir, params, optimizer, evaluate) if test_only: #we're done break if args.criterion in metrics_hist.keys(): if early_stop(metrics_hist, args.criterion, args.patience): #stop training, do tests on test and train sets, and then stop the script print("%s hasn't improved in %d epochs, early stopping..." % (args.criterion, args.patience)) test_only = True test_m = [ o for o in os.listdir(model_dir) if 'model_best' in o ] assert (len(test_m) == 1) args.test_model = os.path.join(model_dir, test_m[0]) model, _, _ = tools.pick_model(args, dicts, META_TEST) return epoch - start_epoch + 1