def main(): models.setup() create_colors() create_rarities() create_layouts() create_formats() models.close()
def setup(): """Setup for upstream plugin""" models.setup() utils.setup() addbook.setup() covers.setup() merge_authors.setup() import data data.setup() # setup template globals from openlibrary.i18n import ugettext, ungettext, gettext_territory web.template.Template.globals.update({ "gettext": ugettext, "ugettext": ugettext, "_": ugettext, "ungettext": ungettext, "gettext_territory": gettext_territory, "random": random.Random(), "commify": web.commify, "group": web.group, "storage": web.storage, "all": all, "any": any, "locals": locals }); import jsdef web.template.STATEMENT_NODES["jsdef"] = jsdef.JSDefNode setup_jquery_urls()
def run(params): model = None if 'load_model_from_dir' in params.__dict__ and params.load_model_from_dir: print('Loading the model from an existing dir!') model_params = pickle.load( open(os.path.join(params.dir_name, 'config.pkl'), 'rb')) if 'lookup_table' in params.__dict__: model_params.lookup_table = params.lookup_table if 'sentiment_dic' in params.__dict__: model_params.sentiment_dic = params.sentiment_dic model = models.setup(model_params) model.load_state_dict( torch.load(os.path.join(params.dir_name, 'model'))) model = model.to(params.device) else: model = models.setup(params).to(params.device) if not ('fine_tune' in params.__dict__ and params.fine_tune == False): print('Training the model!') train(params, model) model = torch.load(params.best_model_file) os.remove(params.best_model_file) performance_dict = test(model, params) performance_str = print_performance(performance_dict, params) save_model(model, params, performance_str) return performance_dict
def setup(config='./cherryconfig.ini'): # create our app from root app = cherrypy.Application(c.Root(),config=config) # update our config cherrypy.config.update(config) # setup the db connection m.setup() # setup a tool to rset our db session cherrypy.tools.reset_db = cherrypy.Tool('on_end_resource', m.reset_session) # validates a user is logged in cherrypy.tools.check_active_login = cherrypy.Tool('before_handler', check_active_login, priority = 10) # setting our user from session data cherrypy.tools.set_user = cherrypy.Tool('before_handler', set_user) # set values on the request object for what section / subsection cherrypy.tools.set_section = cherrypy.Tool('before_handler', set_section) return app
def main(): """Command line interface for setup database""" save_cwd = os.getcwd() try: parser = ArgumentParser(description="Setup ORCM database") parser.add_argument("--alembic-ini", dest='alembic_ini', metavar='/path/to/alembic/config/file', type=str, default="schema_migration.ini", help="A path to the Alembic config file") args = parser.parse_args() ini_dir, ini_file = os.path.split(args.alembic_ini) os.chdir(os.path.join(save_cwd, ini_dir)) models.setup(alembic_ini=ini_file) except OperationalError as e: print("\nError: %s\n" % e) print("Make sure the database exists and the current user has " "proper permissions to create tables and issue DDL & DML " "statements.") except RuntimeError as e: print("\nError: %s\n" % e) finally: os.chdir(save_cwd)
def initialize(): global db, api db = Database(app) # Setup models import models models.setup() # Register REST api api = RestAPI(app) api.register(models.Flow) api.setup()
def setUp(self): self.app = create_app() self.client = self.app.test_client self.database_name = "planttest" self.database_path = "postgresql://{}@{}:{}/{}".format( "postgres", "localhost", "5432", self.database_name) setup(self.app, self.database_path) self.new_plant = { "name": "Watermelon", "sientific_name": "Citrullus lanatus", "is_poisonuons": False, "primary_color": "green" }
def create_app(): app = Flask(__name__) app.jinja_env.auto_reload = True app.config.from_object(config.Config()) CORS(app) # models setup models.setup(app) # security setup Security(app, models.user.USER_DATASTORE,login_form=models.user.ExtendedLoginForm) # register app register_blueprint(app) return app
def show(opt): opt.use_att = utils.if_use_att(opt.caption_model) loader = DataLoader(opt, is_show=True) opt.vocab_size = loader.vocab_size opt.seq_length = loader.seq_length infos = {} if opt.start_from is not None: # open old infos and check if models are compatible with open(os.path.join(opt.start_from, 'infos_'+opt.id+'.pkl')) as f: infos = cPickle.load(f) saved_model_opt = infos['opt'] need_be_same=["caption_model", "rnn_type", "rnn_size", "num_layers"] for checkme in need_be_same: assert vars(saved_model_opt)[checkme] == vars(opt)[checkme], "! Command line argument and saved model disagree on '%s' " % checkme model = models.setup(opt) model.cuda() model.load_state_dict(torch.load(os.path.join(opt.start_from, 'model-best.pth'))) crit = utils.LanguageModelCriterion() # eval model eval_kwargs = {} eval_kwargs.update(vars(opt)) eval_kwargs.update({'split': 'show', 'dataset': opt.input_json, 'language_eval': 0, 'beam_size': 5, 'print_all_beam': True}) val_loss, predictions, lang_stats = eval_utils.eval_split(model, crit, loader, eval_kwargs)
def run(self): if self.lstm_model_path != 'no_lstm_get': self.lstm_model = models.setup(self.opt) self.lstm_model.load_state_dict(torch.load(self.lstm_model_path)) self.lstm_model.cuda() self.lstm_model.eval() self.lstmSignal.emit(self.lstm_model)
def evaluate_model(opt): # Load infos with open(opt.infos_path) as f: infos = cPickle.load(f) # override and collect parameters if len(opt.input_fc_dir) == 0: opt.input_fc_dir = infos['opt'].input_fc_dir opt.input_att_dir = infos['opt'].input_att_dir opt.input_label_h5 = infos['opt'].input_label_h5 if len(opt.input_json) == 0: opt.input_json = infos['opt'].input_json if opt.batch_size == 0: opt.batch_size = infos['opt'].batch_size if len(opt.id) == 0: opt.id = infos['opt'].id ignore = ["id", "batch_size", "beam_size", "start_from", "language_eval"] for k in vars(infos['opt']).keys(): if k not in ignore: if k in vars(opt): assert vars(opt)[k] == vars( infos['opt'])[k], k + ' option not consistent' else: vars(opt).update({k: vars(infos['opt'])[k] }) # copy over options from model vocab = infos['vocab'] # ix -> word mapping # Setup the model model = models.setup(opt) model.load_state_dict(torch.load(opt.model)) model.cuda() model.eval() crit = utils.LanguageModelCriterion() # Create the Data Loader instance if len(opt.image_folder) == 0: loader = DataLoader(opt) else: loader = DataLoaderRaw({ 'folder_path': opt.image_folder, 'coco_json': opt.coco_json, 'batch_size': opt.batch_size, 'cnn_model': opt.cnn_model }) # When eval using provided pretrained model, the vocab may be different from what you have in your cocotalk.json # So make sure to use the vocab in infos file. loader.ix_to_word = infos['vocab'] # Set sample options loss, split_predictions, lang_stats = eval_utils.eval_split( model, crit, loader, vars(opt)) print('loss: ', loss) if lang_stats: print(lang_stats) if opt.dump_json == 1: # dump the json json.dump(split_predictions, open('vis/vis.json', 'w'))
def main(args): cfg.id = args.id cfg.caption_model = args.caption_model cfg.use_cuda = 0 cfg.max_seq_length = args.max_seq_length cfg.start_from = args.start_from cfg.eval_images = args.eval_images cfg_dict = vars(cfg) for k, v in cfg_dict.items(): print(k + ': \t' + str(v)) with open(cfg.vocab_path, 'rb') as f: vocab = pickle.load(f) cfg.vocab_size = len(vocab) model = models.setup(cfg) model.eval() model.cuda() with open(os.path.join(cfg.start_from, 'infos_' + cfg.id + '_best.pkl'), 'rb') as f: infos = pickle.load(f) eval_kwargs = { 'eval_split': 'test', 'eval_time': True, 'save_tree_shape': 1, 'eval_mode': 1 } eval_kwargs.update(vars(cfg)) # lang_stats = eval_utils.eval_split(model, vocab, eval_kwargs) lang_stats = eval_seqtree.eval_split(model, vocab, eval_kwargs) for k, v in lang_stats.items(): print("{}: {}".format(k, v))
def load_model(): opt = parse_args() # Load infos infos = load_infos(opt) ignore = ["id", "batch_size", "beam_size", "start_from_best"] for k in vars(infos['opt']).keys(): if k not in ignore: if k in vars(opt): assert vars(opt)[k] == vars( infos['opt'])[k], k + ' option not consistent' else: vars(opt).update({k: vars(infos['opt'])[k] }) # copy over options from model print(opt) # Setup the model model_cnn = models.setup_cnn(opt) model_cnn.cuda() model = models.setup(opt) model.cuda() # Make sure in the evaluation mode model_cnn.eval() model.eval() ix_to_word = infos['vocab'] return model_cnn, model, ix_to_word, opt
def predict_with_image_folder(self, image_folder): # Setup the model print("self.opt", self.opt) model = models.setup(self.opt) if self.opt.use_cpu: state_dict = torch.load(self.opt.model, map_location={'cuda:0': 'cpu'}) print('state_dict keys', state_dict.keys()) model.load_state_dict(state_dict) else: model.load_state_dict(torch.load(self.opt.model)) if not self.opt.use_cpu: model.cuda() model.eval() crit = utils.LanguageModelCriterion() # Create the Data Loader instance loader = DataLoaderRaw({ 'folder_path': image_folder, 'batch_size': 1, 'cnn_model': self.opt.cnn_model, 'use_cpu': self.opt.use_cpu, }) # When eval using provided pretrained model, the vocab may be different from what you have in your cocotalk.json # So make sure to use the vocab in infos file. loader.ix_to_word = self.infos['vocab'] print('vars opt', vars(self.opt)) # Set sample options predictions = eval_utils.predict(model, crit, loader, vars(self.opt)) print('len(predictions)', len(predictions)) for idx, p in enumerate(predictions): print('#%d\n' % idx) for k in p.keys(): print('\t%s: %s' % (k, p[k]))
def main(args): cfg.id = args.id cfg.dataset = args.dataset cfg.caption_model = args.caption_model cfg.use_cuda = args.use_cuda cfg.seed = args.seed cfg.batch_size = args.batch_size cfg.learning_rate = args.learning_rate cfg.max_seq_length = args.max_seq_length cfg.learning_rate_decay_start = args.learning_rate_decay_start cfg.save_checkpoint_every = args.save_checkpoint_every cfg.checkpoint_path = args.checkpoint_path cfg.start_from = args.start_from cfg.use_tiny = args.use_tiny cfg.eval_images = args.eval_images cfg_dict = vars(cfg) for k, v in cfg_dict.items(): print(k + ': \t' + str(v)) torch.manual_seed(cfg.seed) if cfg.use_cuda: torch.cuda.manual_seed(cfg.seed) with open(cfg.vocab_path, 'rb') as f: vocab = pickle.load(f) cfg.vocab_size = len(vocab) model = models.setup(cfg).cuda() # dp_model = torch.nn.DataParallel(model) # dp_model.train() model.train() train(model, vocab, cfg)
def build_i2t(self, infos, loader): self.i2t_model = I2T_Model_init(self.opt, models.setup(self.opt)) self.dp_i2t_model = torch.nn.DataParallel( self.i2t_model) if len(self.opt.gpus) > 1 else self.i2t_model self.dp_i2t_model.cuda() self.dp_i2t_model.training = True if self.i2t_train_flag else False self.i2t_crit = criterion.LanguageModelCriterion(self.opt) self.i2t_rl_crit = criterion.RewardCriterion()
def setup(): """Setup for upstream plugin""" models.setup() utils.setup() addbook.setup() covers.setup() merge_authors.setup() import data data.setup() # Types for upstream paths types.register_type('^/authors/[^/]*$', '/type/author') types.register_type('^/books/[^/]*$', '/type/edition') types.register_type('^/languages/[^/]*$', '/type/language') types.register_type('^/subjects/places/[^/]*$', '/type/place') types.register_type('^/subjects/people/[^/]*$', '/type/person') types.register_type('^/subjects/[^/]*$', '/type/subject') # fix photo/cover url pattern ol_code.Author.photo_url_patten = "%s/photo" ol_code.Edition.cover_url_patten = "%s/cover" # setup template globals from openlibrary.i18n import ugettext, ungettext web.template.Template.globals.update({ "gettext": ugettext, "ugettext": ugettext, "_": ugettext, "ungettext": ungettext, "random": random.Random(), "commify": web.commify, "group": web.group, "storage": web.storage, "all": all, "any": any, "locals": locals }); import jsdef web.template.STATEMENT_NODES["jsdef"] = jsdef.JSDefNode setup_jquery_urls()
def build_model(self): self.encoder = image_models.image_model(self.args) self.args.features_size = self.encoder.features_size self.decoder = models.setup(self.args) self.encoder.to(self.device) self.encoder.train() self.decoder.to(self.device) self.decoder.train()
def main(): opt = parse_args() # make dirs print(opt.output_dir) if not os.path.isdir(opt.output_dir): os.makedirs(opt.output_dir) # Load infos infos = load_infos(opt) ignore = ["id", "batch_size", "beam_size", "start_from_best"] for k in vars(infos['opt']).keys(): if k not in ignore: if k in vars(opt): assert vars(opt)[k] == vars( infos['opt'])[k], k + ' option not consistent' else: vars(opt).update({k: vars(infos['opt'])[k] }) # copy over options from model print(opt) # Setup the model model_cnn = models.setup_cnn(opt) model_cnn.cuda() model = models.setup(opt) model.cuda() # Make sure in the evaluation mode model_cnn.eval() model.eval() str_id = ''.join(opt.id.split('_')) path_zip = opt.output_dir + '/results.zip' # zipf = zipfile.ZipFile(path_zip, 'w', zipfile.ZIP_DEFLATED) for dataset in opt.datasets: loader = DataLoaderRaw({ 'folder_path': os.path.join(opt.image_folder, dataset), 'batch_size': opt.batch_size }) loader.ix_to_word = infos['vocab'] # Set sample options predictions = eval_split(model_cnn, model, loader, vars(opt)) path_json = opt.output_dir + '/captions_' + dataset + '_' + str_id + '_results.json' json.dump(predictions, open(path_json, 'w'))
def setup(): """Setup for upstream plugin""" models.setup() utils.setup() addbook.setup() covers.setup() # overwrite ReadableUrlProcessor patterns for upstream ReadableUrlProcessor.patterns = [ (r'/books/OL\d+M', '/type/edition', 'title', 'untitled'), (r'/authors/OL\d+A', '/type/author', 'name', 'noname'), (r'/works/OL\d+W', '/type/work', 'title', 'untitled') ] # Types for upstream paths types.register_type('^/authors/[^/]*$', '/type/author') types.register_type('^/books/[^/]*$', '/type/edition') types.register_type('^/languages/[^/]*$', '/type/language') types.register_type('^/subjects/places/[^/]*$', '/type/place') types.register_type('^/subjects/people/[^/]*$', '/type/person') types.register_type('^/subjects/[^/]*$', '/type/subject') # fix photo/cover url pattern ol_code.Author.photo_url_patten = "%s/photo" ol_code.Edition.cover_url_patten = "%s/cover" # setup template globals from openlibrary.i18n import gettext as _ web.template.Template.globals.update({ "gettext": _, "_": _, "random": random.Random(), "commify": web.commify, "group": web.group, "storage": web.storage, "all": all, "any": any, "locals": locals }); setup_jquery_urls()
def split_eval(): opt = parse_eval_opt() # Load infos infos_file = 'infos-best.json' if opt.load_best else 'info.json' infos_path = os.path.join(opt.log_path, infos_file) with open(infos_path, 'r') as f: infos = json.load(f) infos['opt'] = Bunch(infos['opt']) # override and collect parameters if len(opt.input_att_dir) == 0: opt.input_att_dir = infos['opt'].input_att_dir opt.input_label_h5 = infos['opt'].input_label_h5 if len(opt.input_json) == 0: opt.input_json = infos['opt'].input_json if opt.batch_size == 0: opt.batch_size = infos['opt'].batch_size if len(opt.id) == 0: opt.id = infos['opt'].id ignore = ["id", "batch_size", "beam_size", "start_from", "language_eval", "input_att_dir", "input_label_h5", "input_json"] for k in vars(infos['opt']).keys(): if k not in ignore: if k in vars(opt): assert vars(opt)[k] == vars(infos['opt'])[k], k + ' option not consistent' else: vars(opt).update({k: vars(infos['opt'])[k]}) # copy over options from model # Print out the option variables print("*" * 20) for k, v in opt.__dict__.items(): print("%r: %r" % (k, v)) print("*" * 20) # Setup the model model = models.setup(opt) model_file = 'model-best.pth' if opt.load_best else 'model.pth' model_path = os.path.join(opt.log_path, model_file) model.load_state_dict(torch.load(model_path)) model.to(device) model.eval() crit = utils.LanguageModelCriterion() # Create the Data Loader instance loader = DataLoader(opt) loader.ix_to_word = infos['vocab'] # Set sample options loss, split_predictions, lang_stats = eval_utils.eval_split(model, crit, loader, vars(opt)) print(lang_stats) # dump result json if opt.dump_json: json.dump(split_predictions, open(os.path.join(opt.log_path, opt.split+'_res.json'), 'w'))
def eval_split(model_path, opt): dictionary = Dictionary.load_from_file( 'dictionary.pkl') # question dictionary caption_dictionary = Dictionary.load_from_file('caption_dictionary.pkl') actual_batch_size = 32 eval_dset = CaptionQAIMGDataset('val', dictionary, caption_dictionary) eval_loader = DataLoader(eval_dset, actual_batch_size, shuffle=False, num_workers=0) opt.vocab_size = len(caption_dictionary.word2idx) opt.n_tokens = len(dictionary.word2idx) opt.seq_length = 17 # loader.seq_length model = models.setup(opt) model.load_state_dict(torch.load(model_path)) model.cuda() model.eval() # Make sure in the evaluation mode model.eval() predictions = [] #train_statistics = {} for v, q, a, c, rc, qids in tqdm(iter(eval_loader)): # forward the model to also get generated samples for each image # Only leave one feature for each image, in case duplicate sample # forward the model to also get generated samples for each image v = v.cuda() q = q.cuda() with torch.no_grad(): seq, _ = model._sample_beam(v, q, 0, 0) #seqp, _, _, hidden_states, _, _, correct, addition_output = model._forward(att_feats, labels, reverse_seqs, questions, answers, att_masks, state_weights = state_weights) #seq = seqp.max(2)[1] seq = seq.data # Print beam search sents = decode_sequence( cPickle.load(open('caption_dictionary.pkl'))[1], seq) for k in range(v.size(0)): for j in range(1): sent = sents[k + j] sent = sent.replace('<unk> is', 'it is').replace( '<unk> are', 'there are').replace('<unk> ', '') entry = {'caption': sent, 'question_id': qids[k]} predictions.append(entry) print('image %s: %s' % (entry['question_id'], entry['caption'])) cPickle.dump(predictions, open('predictions_beam.pkl', 'wb')) stat = language_eval('COCO', predictions, 'topdown', 'val') model.train() print(stat) return 0
def set_trainer(config): # load a checkpoint if config.checkpoint is not None: # load data train_loader = load_data(config, 'train', False) model, optimizer, word_map, start_epoch = load_checkpoint(config.checkpoint, device) print('\nLoaded checkpoint from epoch %d.\n' % (start_epoch - 1)) # or initialize model else: start_epoch = 0 # load data train_loader, embeddings, emb_size, word_map, n_classes, vocab_size = load_data(config, 'train', True) model = models.setup( config = config, n_classes = n_classes, vocab_size = vocab_size, embeddings = embeddings, emb_size = emb_size ) optimizer = optim.Adam( params = filter(lambda p: p.requires_grad, model.parameters()), lr = config.lr ) # loss functions loss_function = nn.CrossEntropyLoss() # move to device model = model.to(device) loss_function = loss_function.to(device) trainer = Trainer( num_epochs = config.num_epochs, start_epoch = start_epoch, train_loader = train_loader, model = model, model_name = config.model_name, loss_function = loss_function, optimizer = optimizer, lr_decay = config.lr_decay, word_map = word_map, grad_clip = config.grad_clip, print_freq = config.print_freq, checkpoint_path = config.checkpoint_path, checkpoint_basename = config.checkpoint_basename ) return trainer
def test(opt): logger = Logger(opt) dataset = VISTDataset(opt) opt.vocab_size = dataset.get_vocab_size() opt.seq_length = dataset.get_story_length() dataset.test() test_loader = DataLoader(dataset, batch_size=opt.batch_size, shuffle=False, num_workers=opt.workers) evaluator = Evaluator(opt, 'test') model = models.setup(opt) model.cuda() predictions, metrics = evaluator.test_story(model, dataset, test_loader, opt)
def main(): """Command line interface for setup database""" parser = ArgumentParser(description="Setup ORCM database") parser.add_argument("--alembic-ini", dest='alembic_ini', metavar='/path/to/alembic/config/file', type=str, default="schema_migration.ini", help="A path to the Alembic config file") args = parser.parse_args() save_cwd = os.getcwd() ini_dir, ini_file = os.path.split(args.alembic_ini) try: os.chdir(os.path.join(save_cwd, ini_dir)) models.setup(alembic_ini=ini_file) except OperationalError as e: print("\nException: %s\n" % e) print("Make sure the database exists and the current user has " "proper permissions to create tables and issue DDL & DML " "statements.") finally: os.chdir(save_cwd)
def load_unk_model(args, train_itr, test_itr): """ Load an unknown model. Used for convenience to easily swap unk model """ args.lstm_layers = 2 model = models.setup(args) if not args.train_classifier: model.load_state_dict(torch.load(args.model_path)) else: model = train_unk_model(args, model, train_itr, test_itr) model.to(device) # model.eval() return model
def train(opt): # Deal with feature things before anything crit = LanguageModelCriterion() dictionary = Dictionary.load_from_file( 'data/dictionary.pkl') # question dictionary caption_dictionary = Dictionary.load_from_file( 'data/caption_dictionary.pkl') train_dset = CaptionQAIMGDataset('train', dictionary, caption_dictionary) actual_batch_size = 32 train_loader = DataLoader(train_dset, actual_batch_size, shuffle=False, num_workers=0) eval_dset = CaptionQAIMGDataset('val', dictionary, caption_dictionary) eval_loader = DataLoader(eval_dset, actual_batch_size, shuffle=False, num_workers=0) opt.vocab_size = len(caption_dictionary.word2idx) opt.n_tokens = len(dictionary.word2idx) opt.seq_length = 17 model = models.setup(opt).cuda() model.apply(weights_init_kn) model.train() opt.checkpoint_path = 'saved_models/%s' % (str(datetime.now()).replace( ' ', '_').replace('-', '_').replace('.', '_').replace(':', '_')) os.mkdir(opt.checkpoint_path) log_file = open(opt.checkpoint_path + '/log.txt', 'w') print(opt, file=log_file) log_file.flush() epoch = opt.epoch model.load_state_dict( torch.load( os.path.join(opt.caption_model_path, "caption_%d_6000_model.pth" % epoch))) qid2caption_train = evaluate_caption(model, train_loader, caption_dictionary.idx2word, step=1000000000) cPickle.dump(qid2caption_train, open('data/qid2caption_bs_%d_train.pkl' % epoch, 'wb')) qid2caption_val = evaluate_caption(model, eval_loader, caption_dictionary.idx2word, step=1000000000) cPickle.dump(qid2caption_val, open('data/qid2caption_bs_%d_val.pkl' % epoch, 'wb'))
def test(opt, **kwargs): if opt.use_merged_vocab: # using merged vocabulary for Sequential Learning vocab_path = path_util.get_merged_vocab(opt.collection) else: vocab_path = path_util.get_vocab(opt.collection) with open(vocab_path, 'r') as f: vocab = pickle.load(f) vocab_size = len(vocab) opt.vocab_size = vocab_size # load model best_model_path = path_util.get_best_model( opt) #'%s/model-best.pth' % opt.checkpoint_path best_info_path = path_util.get_best_model_info( opt) #'%s/infos-best.pkl' % opt.checkpoint_path print(best_info_path) print(best_model_path) with open(best_info_path) as f: infos = cPickle.load(f) # opt = infos['opt'] # best_val_score = infos['best_val_score'] ignore = ["batch_size", "language_eval", "beam_size", "num_workers"] for k in vars(infos['opt']).keys(): if k not in ignore: # copy over options from model vars(opt).update({k: vars(infos['opt'])[k]}) model = models.setup(opt) model.cuda() model.load(best_model_path) nnl_crit = nn.CrossEntropyLoss() split = 'test' language_eval = opt.language_eval eval_kwargs = { 'split': split, 'id': 'best-model', 'language_eval': language_eval, 'anno_file': path_util.get_test_anno_file(opt.collection), 'output_dir': path_util.get_output_dir(opt, split), 'beam_size': opt.beam_size, 'verbose': True } test_loss, predictions, lang_stats = eval_utils.eval_split( opt, model, vocab, nnl_crit, eval_kwargs)
def main(): opt = parse_args() # make dirs print(opt.output_dir) if not os.path.isdir(opt.output_dir): os.makedirs(opt.output_dir) # Load infos infos = load_infos(opt) ignore = [ "id", "batch_size", "beam_size", "start_from_best", "checkpoint_best_path" ] for k in vars(infos['opt']).keys(): if k not in ignore: if k in vars(opt): assert vars(opt)[k] == vars( infos['opt'])[k], k + ' option not consistent' else: vars(opt).update({k: vars(infos['opt'])[k] }) # copy over options from model print(opt) # Setup the model model_cnn = models.setup_cnn(opt) model_cnn.cuda() model = models.setup(opt) model.cuda() # Make sure in the evaluation mode model_cnn.eval() model.eval() save_model_best(model, model_cnn, infos, opt) loader = DataLoaderRaw({ 'folder_path': opt.image_folder, 'batch_size': opt.batch_size }) loader.ix_to_word = infos['vocab'] # Set sample options predictions = eval_split(model_cnn, model, loader, vars(opt)) json.dump(predictions, open(opt.output_dir + '/result.json', 'w'))
def run(params, data_generator): model = models.setup(params).to(params.device) print('Training the model!') train(params, model) model = torch.load(params.best_model_file) os.remove(params.best_model_file) pred, y = get_predictions(model, params, data_generator) performance_dict = evaluate(params, pred, y) performance_str = print_performance(performance_dict, params) save_model(model, params, performance_str) return performance_dict
def train_for_document_pair(): # fix cell typ,a nd try different RNN models grid_parameters = { "cell_type": ["gru"], "hidden_unit_num": [200], "dropout_rate": [0.2], #,0.5,0.75,0.8,1] , "model": ["bilstm"], # "contatenate":[0], "lr": [0.001], "batch_size": [128], # "validation_split":[0.05,0.1,0.15,0.2], "validation_split": [0.1], } grid_parameters = { "dropout_rate": [0.3], #,0.5,0.75,0.8,1] , "model": ["cnn"], "filter_size": [25, 50], "lr": [0.001], "batch_size": [64], # "validation_split":[0.05,0.1,0.15,0.2], "validation_split": [0.1], } # Set strategy here: strategy = "POS" token_select = TokenSelection(params) # process the dataset train, test = token_select.get_train(dataset="factcheck", strategy=strategy, POS_category="Noun", selected_ratio=0.9) # val_uncontatenated = process.get_test() parameters = [ arg for index, arg in enumerate( itertools.product(*grid_parameters.values())) if index % args.gpu_num == args.gpu ] for parameter in parameters: print(parameter) params.setup(zip(grid_parameters.keys(), parameter)) model = models.setup(params) # here is the invoking of pair function model.train_matching(train, dev=test, strategy=strategy) # test output predicted = model.predict(test[0]) draw_result(predicted, test[1])
def main(args): # Create the model directory if does not exist if not os.path.exists(args.output_path): os.makedirs(args.output_path) # Normalize the input images transform = transforms.Compose( [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) testset = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform) testloader = torch.utils.data.DataLoader(testset, batch_size=args.batch_size, shuffle=False, num_workers=2) dataiter = iter(testloader) print('Loading Models') # Initialize the models model = models.setup(args) # Load the SAVED model path_to_model = os.path.join(args.model_path, args.model+'-p%d_b%d-%d_%d.pkl' % (args.patch_size, args.coded_size, args.load_epoch, args.load_iter)) model.load_state_dict(torch.load(path_to_model)) print('Starting eval:::::::::::::::::') for i in range(args.num_samples//args.batch_size): imgs, _ = dataiter.next() imsave(torchvision.utils.make_grid(imgs), 'prova_'+str(i)) # Patch the image: patches = to_patches(imgs, args.patch_size) r_patches = [] # Reconstructed Patches if args.residual is None: model.reset_state() for p in patches: if args.residual: outputs = model.sample(Variable(p)) else: outputs = model(Variable(p)) r_patches.append(outputs) # Transform the patches into the image outputs = reconstruct_patches(r_patches) imsave(torchvision.utils.make_grid(outputs), 'prova_'+str(i)+'_decoded')
def run_torch(params): # Loss and Optimizer if params.bert_enabled == True: params.max_sequence_length = 512 params.reader.max_sequence_length = 512 # evaluation=[] # params=dataset.classification.process_embedding(reader,params) qdnn = models.setup(params) model = qdnn criterion = nn.CrossEntropyLoss( ) # optimizer = self.opt.optimizer(_params, lr=self.opt.learning_rate, weight_decay=self.opt.l2reg) optimizer = torch.optim.RMSprop(model.parameters(), lr=0.01) test_x, test_y = params.reader.get_test(iterable=False) test_inputs = torch.Tensor(test_x) test_targets = torch.Tensor(test_y) for i in range(params.epochs): print('epoch: ', i) for sample_batched in params.reader.get_train(iterable=True): model.train() optimizer.zero_grad() inputs = sample_batched['X'].long() targets = sample_batched['y'].long() outputs = model(inputs) loss = criterion(outputs, torch.max(targets, 1)[1]) loss.backward() optimizer.step() n_correct = (torch.argmax(outputs, -1) == torch.argmax(targets, -1)).sum().item() n_total = len(outputs) train_acc = n_correct / n_total print('train_acc: {}, loss: {}'.format(train_acc, loss.item())) test_outputs = model(test_inputs.long()) n_correct = (torch.argmax(test_outputs, -1) == torch.argmax(test_targets, -1)).sum().item() n_total = len(test_outputs) test_acc = n_correct / n_total loss = criterion(test_outputs, torch.max(test_targets, 1)[1]) print('test_acc: {}, loss: {}'.format(test_acc, loss.item()))
def train_grid(opt): if 'gah' in grid_pool['model'] or 'gahs' in grid_pool['model']: opt.load_role = True all_roles = [] for roles in grid_pool['roles']: all_roles += roles all_roles = list(set(all_roles)) opt.all_roles = all_roles # load input data = data_helper.Data_helper(opt) train_test = data.load_train(opt.dataset, dataset_pool[opt.dataset]) # search N times: memo = set() for i in range(args.search_times): print('[search time]:', i, '/', args.search_times) para_str = '' for key in grid_pool: value = random.choice(grid_pool[key]) setattr(opt, key, value) para_str += key para_str += str(value) + '_' # memo skip # if para_str in memo: continue memo.add(para_str) # else run this paras print('[paras]:', para_str) setattr(opt, 'para_str', para_str) # set model and train model = models.setup(opt) if len(dataset_pool[opt.dataset]) > 1: train, test = train_test else: print('lenth of train_Test:', len(train_test)) [train], test = train_test, None # sem encoding if opt.tag_encoding == 1: opt.para_str += 'semEmbed' model.train_tag(train, dev=test, dataset=opt.dataset) else: model.train(train, dev=test, dataset=opt.dataset)
def show(opt): opt.use_att = utils.if_use_att(opt.caption_model) loader = DataLoader(opt, is_show=True) opt.vocab_size = loader.vocab_size opt.seq_length = loader.seq_length infos = {} if opt.start_from is not None: # open old infos and check if models are compatible with open(os.path.join(opt.start_from, 'infos_' + opt.id + '.pkl')) as f: infos = cPickle.load(f) saved_model_opt = infos['opt'] need_be_same = [ "caption_model", "rnn_type", "rnn_size", "num_layers" ] for checkme in need_be_same: assert vars(saved_model_opt)[checkme] == vars( opt )[checkme], "! Command line argument and saved model disagree on '%s' " % checkme model = models.setup(opt) model.cuda() model.load_state_dict( torch.load(os.path.join(opt.start_from, 'model-best.pth'))) crit = utils.LanguageModelCriterion() # eval model eval_kwargs = {} eval_kwargs.update(vars(opt)) eval_kwargs.update({ 'split': 'show', 'dataset': opt.input_json, 'language_eval': 0, 'beam_size': 5, 'print_all_beam': True }) val_loss, predictions, lang_stats = eval_utils.eval_split( model, crit, loader, eval_kwargs)
def i2t_eval(): # Setup the model i2t_model = models.setup(opt) i2t_model.load_state_dict(torch.load(opt.model)) i2t_model.cuda() i2t_model.eval() crit = criterion.LanguageModelCriterion(infos['opt']) # Create the Data Loader instance if len(opt.image_folder) == 0: loader = DataLoader(opt) else: loader = DataLoaderRaw({ 'folder_path': opt.image_folder, 'coco_json': opt.coco_json, 'batch_size': opt.batch_size, 'cnn_model': opt.cnn_model }) # When eval using provided pretrained model, the vocab may be different from what you have in your cocotalk.json # So make sure to use the vocab in infos file. loader.ix_to_word = infos['vocab'] # Set sample options loss, split_predictions, lang_stats, nmt_valid_ppl, nmt_valid_acc = eval_utils.eval_split( opt, loader, i2t_model, None, vars(opt)) print('loss: ', loss) if lang_stats: print(lang_stats) if opt.dump_json == 1: # dump the json dump_json_path = 'tmp/' + opt.model.split( '/')[1] + '_zh_' + opt.dataset + '.json' json.dump( split_predictions, open( 'tmp/' + opt.model.split('/')[1] + '_zh_' + opt.dataset + '.json', 'w')) return dump_json_path
if len(opt.id) == 0: opt.id = infos['opt'].id ignore = ["id", "batch_size", "beam_size", "start_from", "language_eval"] for k in vars(infos['opt']).keys(): if k not in ignore: if k in vars(opt): assert vars(opt)[k] == vars(infos['opt'])[ k], k + ' option not consistent' else: # copy over options from model vars(opt).update({k: vars(infos['opt'])[k]}) vocab = infos['vocab'] # ix -> word mapping # Setup the model model = models.setup(opt) model.load_state_dict(torch.load(opt.model)) model.cuda() model.eval() crit = utils.LanguageModelCriterion() # Create the Data Loader instance if len(opt.image_folder) == 0: loader = DataLoader(opt) else: loader = DataLoaderRaw({'folder_path': opt.image_folder, 'coco_json': opt.coco_json, 'batch_size': opt.batch_size, 'cnn_model': opt.cnn_model}) # When eval using provided pretrained model, the vocab may be different from what you have in your cocotalk.json # So make sure to use the vocab in infos file.
def train(opt): loader = DataLoader(opt) opt.vocab_size = loader.vocab_size opt.seq_length = loader.seq_length model = models.setup(opt) infos = {} if opt.start_from is not None: # open old infos and check if models are compatible with open(os.path.join(opt.start_from, 'infos_'+opt.id+'.pkl')) as f: infos = cPickle.load(f) saved_model_opt = infos['opt'] need_be_same=["caption_model", "rnn_type", "rnn_size", "num_layers"] for checkme in need_be_same: assert vars(saved_model_opt)[checkme] == vars(opt)[checkme], "Command line argument and saved model disagree on '%s' " % checkme iteration = infos.get('iter', 0) epoch = infos.get('epoch', 0) val_result_history = infos.get('val_result_history', {}) loss_history = infos.get('loss_history', {}) loader.iterators = infos.get('iterators', loader.iterators) if opt.load_best_score == 1: best_val_score = infos.get('best_val_score', None) model.build_model() model.build_generator() model.build_decoder() tf_config = tf.ConfigProto() tf_config.intra_op_parallelism_threads=NUM_THREADS tf_config.gpu_options.allow_growth = True with tf.Session(config=tf_config) as sess: # Initialize the variables, and restore the variables form checkpoint if there is. # and initialize the writer model.initialize(sess) # Assign the learning rate if epoch > opt.learning_rate_decay_start and opt.learning_rate_decay_start >= 0: frac = (epoch - opt.learning_rate_decay_start) / opt.learning_rate_decay_every decay_factor = 0.5 ** frac sess.run(tf.assign(model.lr, opt.learning_rate * decay_factor)) # set the decayed rate sess.run(tf.assign(model.cnn_lr, opt.cnn_learning_rate * decay_factor)) else: sess.run(tf.assign(model.lr, opt.learning_rate)) sess.run(tf.assign(model.cnn_lr, opt.cnn_learning_rate)) # Assure in training mode sess.run(tf.assign(model.training, True)) sess.run(tf.assign(model.cnn_training, True)) while True: start = time.time() # Load data from train split (0) data = loader.get_batch('train') print('Read data:', time.time() - start) start = time.time() feed = {model.images: data['images'], model.labels: data['labels'], model.masks: data['masks']} if iteration <= opt.finetune_cnn_after or opt.finetune_cnn_after == -1: train_loss, merged, _ = sess.run([model.cost, model.summaries, model.train_op], feed) else: # Finetune the cnn train_loss, merged, _, __ = sess.run([model.cost, model.summaries, model.train_op, model.cnn_train_op], feed) end = time.time() print("iter {} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}" \ .format(iteration, epoch, train_loss, end - start)) # Update the iteration and epoch iteration += 1 if data['bounds']['wrapped']: epoch += 1 # Write the training loss summary if (iteration % opt.losses_log_every == 0): model.summary_writer.add_summary(merged, iteration) model.summary_writer.flush() loss_history[iteration] = train_loss # make evaluation on validation set, and save model if (iteration % opt.save_checkpoint_every == 0): # eval model eval_kwargs = {'val_images_use': opt.val_images_use, 'split': 'val', 'language_eval': opt.language_eval, 'dataset': opt.input_json} val_loss, predictions, lang_stats = eval_split(sess, model, loader, eval_kwargs) # Write validation result into summary summary = tf.Summary(value=[tf.Summary.Value(tag='validation loss', simple_value=val_loss)]) model.summary_writer.add_summary(summary, iteration) for k,v in lang_stats.iteritems(): summary = tf.Summary(value=[tf.Summary.Value(tag=k, simple_value=v)]) model.summary_writer.add_summary(summary, iteration) model.summary_writer.flush() val_result_history[iteration] = {'loss': val_loss, 'lang_stats': lang_stats, 'predictions': predictions} # Save model if is improving on validation result if opt.language_eval == 1: current_score = lang_stats['CIDEr'] else: current_score = - val_loss if best_val_score is None or current_score > best_val_score: # if true best_val_score = current_score checkpoint_path = os.path.join(opt.checkpoint_path, 'model.ckpt') model.saver.save(sess, checkpoint_path, global_step = iteration) print("model saved to {}".format(checkpoint_path)) # Dump miscalleous informations infos['iter'] = iteration infos['epoch'] = epoch infos['iterators'] = loader.iterators infos['best_val_score'] = best_val_score infos['opt'] = opt infos['val_result_history'] = val_result_history infos['loss_history'] = loss_history infos['vocab'] = loader.get_vocab() with open(os.path.join(opt.checkpoint_path, 'infos_'+opt.id+'.pkl'), 'wb') as f: cPickle.dump(infos, f) # Stop if reaching max epochs if epoch >= opt.max_epochs and opt.max_epochs != -1: break
session.add(movie) try: logging.debug("Processing movie: %s", movie) except: pass for time in itertools.chain(times["soldout"], times["available"]): # For each movie, get the various showtimes showtime = session.query(Showtime).filter(Showtime.movie == movie).filter(Showtime.theater == theater).\ filter(Showtime.time == time).first() if showtime is None: showtime = Showtime(movie, theater, time) session.add(showtime) # And then make a record of whether that showtime is available right now or not status = time in times["available"] availability = Availability(showtime, times["date"], status) session.add(availability) if __name__ == "__main__": logging.basicConfig() for handler in logging.getLogger().handlers: handler.setFormatter(logging.Formatter("%(name)s: %(message)s")) logging.getLogger().setLevel(logging.DEBUG) logging.debug("Starting") db_session = models.setup() showtimes = get_showtimes("3ED25043", "SilverCity_Riverport") #pprint(showtimes) update_database(db_session, showtimes, "Silvercity Riverport") db_session.commit()
from maillib import Mail from add_guests import get_data import models as m; m.setup() from helpers import get_guest_token SERVER = 'google.com' LINK_BASE = 'http://mostsplendiferous.com' TEMPLATE_PATH = './email_template.html' SENDER = '"Lizz Mitchell" <*****@*****.**>' def send_mail(): mailed = [] for data in get_data(): lookup = {'name':('%s %s' % (data.get('first_name',''), data.get('last_name',''))).strip(), 'party_size':data.get('party_size',1) or 1, 'guests':data.get('guests',0) or 0 } guest = m.Guest.get_by(name=lookup.get('name')) if not guest: raise Exception('no guest: %s' % lookup) link = '%s/%s/%s' % (LINK_BASE,guest.id,get_guest_token(guest)) lookup['link'] = link mail = Mail(server=SERVER, sender=SENDER, to=data.get('email_address'), template_path=TEMPLATE_PATH, use_templating=True, replacement_dict=lookup) r = mail.send() mailed.append(r) return mailed
# Make sure our bundled libraries take precedence sys.path.insert(0,os.path.join(os.path.dirname(os.path.abspath(__file__)),'lib')) import config, utils # read configuration file config.settings = utils.get_config(os.path.join(utils.path_for('data'),'config.json')) # Set up logging logging.config.dictConfig(dict(config.settings.logging)) log = logging.getLogger() # load modules import models, controllers models.setup() if __name__ == "__main__": log.info("Starting fetcher.") start = time.time() fc = controllers.FeedController() feeds = fc.get_feeds(all=True) if config.settings.fetcher.pool: if config.settings.fetcher.engine == 'multiprocessing': from multiprocessing import Pool p = Pool(processes=config.settings.fetcher.pool) p.map(controllers.feed_worker, feeds, len(feeds)/config.settings.fetcher.pool) elif config.settings.fetcher.engine == 'gevent': from gevent import monkey; monkey.patch_all() from gevent.pool import Pool p = Pool(config.settings.fetcher.pool)
''' from flask import Flask import config import models app = Flask(__name__) app.config.from_object(config) app.debug = app.config.get('DEBUG', False) app.secret_key = app.config['SECRET_KEY'] app.selnk_key = app.config["SELNK_KEY"] # 初始化进程全局的数据模型对象 models.setup(app) from views import main # 注册 Flask 应用模块 app.register_blueprint(main.module) from flask import render_template @app.errorhandler(404) def page_not_found(error): return render_template('404.html') # 设置logger的输出形式 import logging, os from logging.handlers import TimedRotatingFileHandler
import sys, os from auth import set_user, check_active_login from helpers import set_section import logging import models as m import controllers as c log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) log.addHandler(logging.StreamHandler(sys.stdout)) if __name__ == "__main__": log.info('starting') # setup the db connection m.setup() # setup a tool to rset our db session cherrypy.tools.reset_db = cherrypy.Tool('on_end_resource', m.reset_session) # validates a user is logged in cherrypy.tools.check_active_login = cherrypy.Tool('before_handler', check_active_login, priority = 10) # setting our user from session data cherrypy.tools.set_user = cherrypy.Tool('before_handler', set_user) # set values on the request object for what section / subsection cherrypy.tools.set_section = cherrypy.Tool('before_handler', set_section)
def setup(app): import models models.setup(app) import views.user views.user.setup(app)
def train(opt): opt.use_att = utils.if_use_att(opt.caption_model) loader = DataLoader(opt) opt.vocab_size = loader.vocab_size opt.seq_length = loader.seq_length tf_summary_writer = tf and tf.summary.FileWriter(opt.checkpoint_path) infos = {} histories = {} if opt.start_from is not None: # open old infos and check if models are compatible with open(os.path.join(opt.start_from, 'infos_'+opt.id+'.pkl'),'rb') as f: infos = cPickle.load(f) saved_model_opt = infos['opt'] need_be_same=["caption_model", "rnn_type", "rnn_size", "num_layers"] for checkme in need_be_same: assert vars(saved_model_opt)[checkme] == vars(opt)[checkme], "Command line argument and saved model disagree on '%s' " % checkme if os.path.isfile(os.path.join(opt.start_from, 'histories_'+opt.id+'.pkl')): with open(os.path.join(opt.start_from, 'histories_'+opt.id+'.pkl'),'rb') as f: histories = cPickle.load(f) iteration = infos.get('iter', 0) epoch = infos.get('epoch', 0) val_result_history = histories.get('val_result_history', {}) loss_history = histories.get('loss_history', {}) lr_history = histories.get('lr_history', {}) ss_prob_history = histories.get('ss_prob_history', {}) loader.iterators = infos.get('iterators', loader.iterators) loader.split_ix = infos.get('split_ix', loader.split_ix) if opt.load_best_score == 1: best_val_score = infos.get('best_val_score', None) model = models.setup(opt) model.cuda() if opt.multi_gpu: model=nn.DataParallel(model) update_lr_flag = True # Assure in training mode model.train() crit = utils.LanguageModelCriterion() optimizer = optim.Adam(model.parameters(), lr=opt.learning_rate, weight_decay=opt.weight_decay) # Load the optimizer if vars(opt).get('start_from', None) is not None: optimizer.load_state_dict(torch.load(os.path.join(opt.start_from, 'optimizer.pth'))) while True: if update_lr_flag: # Assign the learning rate if epoch > opt.learning_rate_decay_start and opt.learning_rate_decay_start >= 0: frac = (epoch - opt.learning_rate_decay_start) // opt.learning_rate_decay_every decay_factor = opt.learning_rate_decay_rate ** frac opt.current_lr = opt.learning_rate * decay_factor utils.set_lr(optimizer, opt.current_lr) # set the decayed rate else: opt.current_lr = opt.learning_rate # Assign the scheduled sampling prob if epoch > opt.scheduled_sampling_start and opt.scheduled_sampling_start >= 0: frac = (epoch - opt.scheduled_sampling_start) // opt.scheduled_sampling_increase_every opt.ss_prob = min(opt.scheduled_sampling_increase_prob * frac, opt.scheduled_sampling_max_prob) model.ss_prob = opt.ss_prob update_lr_flag = False start = time.time() # Load data from train split (0) data = loader.get_batch('train') print('Read data:', time.time() - start) torch.cuda.synchronize() start = time.time() tmp = [data['fc_feats'], data['att_feats'], data['labels'], data['masks']] tmp = [Variable(torch.from_numpy(_), requires_grad=False).cuda() for _ in tmp] fc_feats, att_feats, labels, masks = tmp optimizer.zero_grad() loss = crit(model(fc_feats, att_feats, labels), labels[:,1:], masks[:,1:]) loss.backward() utils.clip_gradient(optimizer, opt.grad_clip) optimizer.step() train_loss = loss.data[0] torch.cuda.synchronize() end = time.time() print("iter {} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}" \ .format(iteration, epoch, train_loss, end - start)) # Update the iteration and epoch iteration += 1 if data['bounds']['wrapped']: epoch += 1 update_lr_flag = True # Write the training loss summary if (iteration % opt.losses_log_every == 0): if tf is not None: add_summary_value(tf_summary_writer, 'train_loss', train_loss, iteration) add_summary_value(tf_summary_writer, 'learning_rate', opt.current_lr, iteration) add_summary_value(tf_summary_writer, 'scheduled_sampling_prob', model.ss_prob, iteration) tf_summary_writer.flush() loss_history[iteration] = train_loss lr_history[iteration] = opt.current_lr ss_prob_history[iteration] = model.ss_prob # make evaluation on validation set, and save model if (iteration % opt.save_checkpoint_every == 0): # eval model eval_kwargs = {'split': 'val', 'dataset': opt.input_json} eval_kwargs.update(vars(opt)) val_loss, predictions, lang_stats = eval_utils.eval_split(model, crit, loader, eval_kwargs) # Write validation result into summary if tf is not None: add_summary_value(tf_summary_writer, 'validation loss', val_loss, iteration) for k,v in lang_stats.items(): add_summary_value(tf_summary_writer, k, v, iteration) tf_summary_writer.flush() val_result_history[iteration] = {'loss': val_loss, 'lang_stats': lang_stats, 'predictions': predictions} # Save model if is improving on validation result if opt.language_eval == 1: current_score = lang_stats['CIDEr'] else: current_score = - val_loss best_flag = False if True: # if true if best_val_score is None or current_score > best_val_score: best_val_score = current_score best_flag = True checkpoint_path = os.path.join(opt.checkpoint_path, 'model.pth') torch.save(model.state_dict(), checkpoint_path) print("model saved to {}".format(checkpoint_path)) optimizer_path = os.path.join(opt.checkpoint_path, 'optimizer.pth') torch.save(optimizer.state_dict(), optimizer_path) # Dump miscalleous informations infos['iter'] = iteration infos['epoch'] = epoch infos['iterators'] = loader.iterators infos['split_ix'] = loader.split_ix infos['best_val_score'] = best_val_score infos['opt'] = opt infos['vocab'] = loader.get_vocab() histories['val_result_history'] = val_result_history histories['loss_history'] = loss_history histories['lr_history'] = lr_history histories['ss_prob_history'] = ss_prob_history with open(os.path.join(opt.checkpoint_path, 'infos_'+opt.id+'.pkl'), 'wb') as f: cPickle.dump(infos, f) with open(os.path.join(opt.checkpoint_path, 'histories_'+opt.id+'.pkl'), 'wb') as f: cPickle.dump(histories, f) if best_flag: checkpoint_path = os.path.join(opt.checkpoint_path, 'model-best.pth') torch.save(model.state_dict(), checkpoint_path) print("model saved to {}".format(checkpoint_path)) with open(os.path.join(opt.checkpoint_path, 'infos_'+opt.id+'-best.pkl'), 'wb') as f: cPickle.dump(infos, f) # Stop if reaching max epochs if epoch >= opt.max_epochs and opt.max_epochs != -1: break