def main(): utils.ensure_exists(output_directory) date_of_last_cached_leaderboard = get_date_of_last_cached_leaderboard() log.info('date of the last cached leaderboard is %s', date_of_last_cached_leaderboard) date_of_last_goko_leaderboard = datetime.date.today() one_day_delta = datetime.timedelta(1) date = date_of_last_cached_leaderboard + one_day_delta while date <= datetime.date.today(): log.info('Processing %s', date) if date == datetime.date.today(): log.info('scraping from goko') status = run_scrape_function_with_retries(scrape_leaderboard_from_goko, date) else: log.info('scraping from councilroom') status = run_scrape_function_with_retries(scrape_leaderboard_from_councilroom, date) if status != 200 and date <= datetime.date(2013,01,01): log.info('scraping from bggdl') status = run_scrape_function_with_retries(scrape_leaderboard_from_bggdl, date) if status == 200: pass elif status == 404: log.warning('file not found, so we will assume that it does not exist, and go to the next day') else: log.warning('Unexpected status of %d, please try again later', status) break date += one_day_delta
def run_directory(index): directory = '{sim_dir}/{index}'.format( sim_dir=simulation_store_directory(), index=index) utils.ensure_exists(directory) return directory
def scrape_games(): parser = utils.incremental_date_range_cmd_line_parser() utils.ensure_exists('static/scrape_data') os.chdir('static/scrape_data') args = parser.parse_args() last_month = '' for cur_date in utils.daterange(datetime.date(2010, 10, 15), datetime.date.today()): str_date = time.strftime("%Y%m%d", cur_date.timetuple()) if not utils.includes_day(args, str_date): if DEBUG: print 'skipping', str_date, 'because not in cmd line arg daterange' continue mon = time.strftime("%b%y", cur_date.timetuple()) if mon != last_month: print print mon, cur_date.day*" ", sys.stdout.flush() last_month = mon ret = scrape_date(str_date, cur_date, passive=args.passive) if ret==DOWNLOADED: print 'o', elif ret==REPACKAGED: print 'O', elif ret==ERROR: print '!', elif ret==MISSING: print '_', else: print '.', sys.stdout.flush() print os.chdir('../..')
def get_structure(self, pid, dst=None, parser=None): dirname = '%s/%s' % (self.pdb_dir, pid) if not dst: dst = '%s/%s' % (dirname, pid) if not os.path.exists(dst) or os.path.getsize(dst) < 100: ensure_exists(dirname) download_pdb(pid, dst) if not parser: parser = PDBParser(QUIET=True) def creator(parser=parser): try: ret = parser.get_structure(pid, file=dst) except ValueError as e: # assume it's a .cif if PARSE_CIF: parser = MMCIFParser(QUIET=True) ret = parser.get_structure(pid, dst) else: raise e finally: self.freemem() return ret if self.cache: # warn: Leaky Code! return self.cache.get(key=pid, createfunc=creator) elif os.path.getsize(dst) > consts.PDB_SIZE_LIMIT: raise ValueError('file size exceeds %s' % consts.PDB_SIZE_LIMIT) else: return creator()
def train(self, x, y, epochs, batch_size=128, sample_interval=50, sample_path="samples/unknown"): """ Trains the GAN. :param x: The training data. :param y: The labels for the training data. :param epochs: The number of epochs to train. :param batch_size: The size of an epoch. :param sample_interval: How often to save sample images. :param sample_path: Where to save sample images. """ ensure_exists(sample_path) half_batch = int(batch_size / 2) for epoch in range(epochs): # --------------------- # Train Discriminator # --------------------- # Select a random half batch of images idx = np.random.randint(0, x.shape[0], half_batch) imgs = x[idx] labels = y[idx] # Generate a half batch of new images noise = np.random.normal(0, 1, (half_batch, self.noise_size)) gen_imgs = self.generator.predict({"noise": noise, "label": labels}) # Train the discriminator d_loss_real = self.discriminator.train_on_batch({"image": imgs, "label": labels}, np.ones((half_batch, 1))) d_loss_fake = self.discriminator.train_on_batch({"image": gen_imgs, "label": labels}, np.zeros((half_batch, 1))) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) # --------------------- # Train Generator # --------------------- # The generator wants the discriminator to label the generated samples # as valid (ones) valid_y = np.array([1] * batch_size) # Train the generator on random labels noise = np.random.normal(0, 1, (batch_size, self.noise_size)) idx = np.random.randint(0, x.shape[0], batch_size) labels = y[idx] g_loss = self.combined.train_on_batch({"noise": noise, "label": labels}, valid_y) # Plot the progress print("%d [D loss: %f, acc.: %.2f%%] [G loss: %f]" % (epoch, d_loss[0], 100 * d_loss[1], g_loss)) # If at save interval => save generated image samples if sample_interval and epoch % sample_interval == 0: self.save_sample(epoch, sample_path)
def save_summary(self, path): path = path.rstrip('/') ensure_exists(path) with open(f'{path}/summary.txt', 'w') as f: def write_to_summary_file(text): f.write(f"{text}\n") self.generator.summary(print_fn=write_to_summary_file) self.discriminator.summary(print_fn=write_to_summary_file) tf.keras.utils.plot_model(self.discriminator, to_file=f"{path}/d.png") tf.keras.utils.plot_model(self.generator, to_file=f"{path}/g.png")
def save(self, path): """Saves the GAN to a folder.""" path = path.rstrip('/') ensure_exists(path) with open(f"{path}/config.json", 'w') as f: json.dump({"k": float(self.k.numpy())}, f) self.generator.save_weights(f"{path}/g.h5", save_format='h5') self.discriminator.save_weights(f"{path}/d.h5", save_format='h5') try: self.save_summary(path) except: pass
def main(args, log): BEEN_PARSED_KEY = 'day_analyzed' if args.incremental: log.info("Performing incremental parsing from %s to %s", args.startdate, args.enddate) else: log.info("Performing non-incremental (re)parsing from %s to %s", args.startdate, args.enddate) connection = pymongo.MongoClient() db = connection.test # RT: changed. raw_games = db.raw_games raw_games.ensure_index('game_date') utils.ensure_exists('parsed_out') day_status_col = db.day_status days = day_status_col.find({'raw_games_loaded': True}) for day in days: year_month_day = day['_id'] if not utils.includes_day(args, year_month_day): log.debug( "Raw games for %s available in the database but not in date range, skipping", year_month_day) continue if BEEN_PARSED_KEY not in day: day[BEEN_PARSED_KEY] = False day_status_col.save(day) if day[BEEN_PARSED_KEY] and args.incremental: log.debug( "Raw games for %s have been parsed, and we're running incrementally, skipping", year_month_day) continue try: log.info("Parsing %s", year_month_day) convert_to_json(log, raw_games, year_month_day) continue day[BEEN_PARSED_KEY] = True day_status_col.save(day) except ParseTurnHeaderError as e: log.error("ParseTurnHeaderError occurred while parsing %s: %s", year_month_day, e) return except Exception as e: log.error("Exception occurred while parsing %s: %s", year_month_day, e) return
def read_usage(): """ Read the usage information from ~/cluster-load/info. Information is expected to be updated through another process. The function creates ~/cluster-load/info if it doesn't exists. """ # Because the file transfer might change files as they're being read, this could fail # Retry until success or num_retries num_retries = 10 info_path = os.path.expanduser('~/cluster-load/info/') utils.ensure_exists(info_path) for retries in range(num_retries): try: cmd = 'cat {path}* 2>/dev/null'.format(path=info_path) output_lines = subprocess.check_output( cmd, shell=True).decode('utf8').split('\n') num_tries = retries + 1 if num_tries > 1: print('warning: cpu usage reading attempt success at try {x}'. format(x=num_tries)) break except subprocess.CalledProcessError as e: if (retries == num_retries - 1): raise (e) else: continue line_parts = [ line.split() for line in output_lines if line[:7] == '10.0.0.' ] cpu_usage = { line[0]: string_to_rounded_int(line[1]) for line in line_parts } temp = { line[0]: int(float(line[2]) * 100 / settings.pi_max_temp) for line in line_parts } return cpu_usage, temp
def dump_chpt(eval_batcher, hps, model, sess, saver, eval_loss_best, early_stop=False): dump_model = False # Run evals on development set and print their perplexity. previous_losses = [eval_loss_best] eval_losses = [] eval_accuracies = [] stop_flag = False while True: batch = eval_batcher.next_batch() if not batch[0]: eval_batcher.reset() break eval_inputs, eval_conditions, eval_targets = \ data.prepare_dis_pretraining_batch(batch) eval_inputs = np.split(eval_inputs, 2)[0] eval_conditions = np.split(eval_conditions, 2)[0] eval_targets = np.split(eval_targets, 2)[0] eval_results = model.run_one_batch(sess, eval_inputs, eval_conditions, eval_targets, update=False) eval_losses.append(eval_results["loss"]) eval_accuracies.append(eval_results["accuracy"]) eval_loss = sum(eval_losses) / len(eval_losses) eval_accuracy = sum(eval_accuracies) / len(eval_accuracies) previous_losses.append(eval_loss) sys.stdout.flush() threshold = 10 if eval_loss > 0.99 * previous_losses[-2]: sess.run( model.learning_rate.assign( tf.maximum( hps.learning_rate_decay_factor * model.learning_rate, 1e-4))) if len(previous_losses) > threshold and \ eval_loss > max(previous_losses[-threshold-1:-1]) and \ eval_loss_best < min(previous_losses[-threshold:]): if early_stop: stop_flag = True else: stop_flag = False print("Proper time to stop...") if eval_loss < eval_loss_best: dump_model = True eval_loss_best = eval_loss # Save checkpoint and zero timer and loss. if dump_model: checkpoint_path = ensure_exists( join_path(hps.model_dir, "discriminator")) + "/model.ckpt" saver.save(sess, checkpoint_path, global_step=model.global_step) print("Saving the checkpoint to %s" % checkpoint_path) return eval_accuracy, eval_loss, stop_flag, eval_loss_best
def main(): utils.ensure_exists(output_directory) one_day_delta = datetime.timedelta(1) date = get_date_of_last_cached_leaderboard() + one_day_delta date_of_current_isotropic_leaderboard = get_date_of_current_isotropic_leaderboard() success = True while success and date <= date_of_current_isotropic_leaderboard: print date if date == date_of_current_isotropic_leaderboard: success = scrape_leaderboard_from_isotropic(date) else: success = scrape_leaderboard_from_online_cache(date) date += one_day_delta
def main(args, log): BEEN_PARSED_KEY = 'day_analyzed' if args.incremental: log.info("Performing incremental parsing from %s to %s", args.startdate, args.enddate) else: log.info("Performing non-incremental (re)parsing from %s to %s", args.startdate, args.enddate) connection = pymongo.Connection() db = connection.test raw_games = db.raw_games raw_games.ensure_index('game_date') utils.ensure_exists('parsed_out') day_status_col = db.day_status days = day_status_col.find({'raw_games_loaded': True}) for day in days: year_month_day = day['_id'] if not utils.includes_day(args, year_month_day): log.debug("Raw games for %s available in the database but not in date range, skipping", year_month_day) continue if BEEN_PARSED_KEY not in day: day[BEEN_PARSED_KEY] = False day_status_col.save(day) if day[BEEN_PARSED_KEY] and args.incremental: log.debug("Raw games for %s have been parsed, and we're running incrementally, skipping", year_month_day) continue try: log.info("Parsing %s", year_month_day) convert_to_json(log, raw_games, year_month_day) continue day[BEEN_PARSED_KEY] = True day_status_col.save(day) except ParseTurnHeaderError, e: log.error("ParseTurnHeaderError occurred while parsing %s: %s", year_month_day, e) return except Exception, e: log.error("Exception occurred while parsing %s: %s", year_month_day, e) return
def save(self, path): """Saves the GAN to a folder.""" path = path.rstrip('/') ensure_exists(path) config = { "rows": self.img_rows, "cols": self.img_cols, "chans": self.channels, "noise_size": self.noise_size } with open(f"{path}/config.json", 'w') as f: json.dump(config, f) self.generator.save(f"{path}/g.h5") self.discriminator.save(f"{path}/d.h5") try: self.save_summary(path) except: pass
def init_files(self, filename): data_dir = self.pdb_dir with open(filename) as f: ids = f.read().split(',') if not os.path.exists(data_dir): os.mkdir(data_dir) for pid in ids: pid_dir = "%s/%s" % (data_dir, pid) dst = '%s/%s' % (pid_dir, pid) old_dst = '%s/%s.pdb' % (pid_dir, pid) ensure_exists(pid_dir) if os.path.exists(old_dst): print("moving %s->%s" % (old_dst, dst)) shutil.move(old_dst, dst) if not os.path.exists(dst) or os.path.getsize(dst) < 100: download_pdb(pid, dst) else: print("found %s\t%10s Bytes" % (dst, os.path.getsize(dst)))
def main(): utils.ensure_exists(output_directory) date_of_last_cached_leaderboard = get_date_of_last_cached_leaderboard() print 'date of the last cached leaderboard is', date_of_last_cached_leaderboard date_of_current_isotropic_leaderboard = get_date_of_current_isotropic_leaderboard() if date_of_current_isotropic_leaderboard is None: print 'could not determine the date of the current isotropic leaderboard, so please try again later' return print 'date of the current isotropic leaderboard is', date_of_current_isotropic_leaderboard one_day_delta = datetime.timedelta(1) date = date_of_last_cached_leaderboard + one_day_delta while date <= date_of_current_isotropic_leaderboard: print print date if date == date_of_current_isotropic_leaderboard: print 'scraping from isotropic' status = run_scrape_function_with_retries(scrape_leaderboard_from_isotropic, date) else: print 'scraping from councilroom' status = run_scrape_function_with_retries(scrape_leaderboard_from_councilroom, date) if status != 200: print 'scraping from bggdl' status = run_scrape_function_with_retries(scrape_leaderboard_from_bggdl, date) if status == 200: pass elif status == 404: print 'file not found, so we will assume that it does not exist, and go to the next day' else: print 'please try again later' break date += one_day_delta
def main(): utils.ensure_exists(output_directory) date_of_last_cached_leaderboard = get_date_of_last_cached_leaderboard() log.info('date of the last cached leaderboard is %s', date_of_last_cached_leaderboard) date_of_current_isotropic_leaderboard = get_date_of_current_isotropic_leaderboard() if date_of_current_isotropic_leaderboard is None: log.warning('could not determine the date of the current isotropic leaderboard, so please try again later') return log.info('date of the current isotropic leaderboard is %s', date_of_current_isotropic_leaderboard) one_day_delta = datetime.timedelta(1) date = date_of_last_cached_leaderboard + one_day_delta while date <= date_of_current_isotropic_leaderboard: log.info('Processing %s', date) if date == date_of_current_isotropic_leaderboard: log.info('scraping from isotropic') status = run_scrape_function_with_retries(scrape_leaderboard_from_isotropic, date) else: log.info('scraping from councilroom') status = run_scrape_function_with_retries(scrape_leaderboard_from_councilroom, date) if status != 200: log.info('scraping from bggdl') status = run_scrape_function_with_retries(scrape_leaderboard_from_bggdl, date) if status == 200: pass elif status == 404: log.warning('file not found, so we will assume that it does not exist, and go to the next day') else: log.warning('Unexpected status of %d, please try again later', status) break date += one_day_delta
def scrape_games(): parser = utils.incremental_date_range_cmd_line_parser() utils.ensure_exists('static/scrape_data') os.chdir('static/scrape_data') args = parser.parse_args() last_month = '' yesterday = datetime.date.today() - datetime.timedelta(days=1) #Goko updates logs in real time; wait a day so the list is finalized. for cur_date in utils.daterange(default_startdate, yesterday, reverse=True): str_date = time.strftime("%Y%m%d", cur_date.timetuple()) if not utils.includes_day(args, str_date): if DEBUG: print 'skipping', str_date, 'because not in cmd line arg daterange' continue mon = time.strftime("%b%y", cur_date.timetuple()) if mon != last_month: print print mon, cur_date.day*" ", sys.stdout.flush() last_month = mon ret = scrape_date(str_date, cur_date, passive=args.passive) if ret==DOWNLOADED: print 'o', elif ret==REPACKAGED: print 'O', elif ret==ERROR: print '!', elif ret==MISSING: print '_', else: print '.', sys.stdout.flush() print os.chdir('../..')
# taken from # http://stackoverflow.com/questions/1060279/iterating-through-a-range-of-dates-in-python import datetime import time import os import urllib import utils parser = utils.IncrementalDateRangeCmdLineParser() # if the size of the game log is less than this assume we got an error page SMALL_FILE_SIZE = 5000 utils.ensure_exists('static/scrape_data') os.chdir('static/scrape_data') # make I should just adopt the isotropic format for consistency? ISOTROPIC_FORMAT = '%(year)d%(month)02d/%(day)02d/all.tar.bz2' COUNCILROOM_FORMAT = '%(year)d%(month)02d%(day)02d/%(year)d%(month)02d%(day)02d.all.tar.bz2' def FormatDate(fmt, date): return fmt % { 'year': cur_date.year, 'month': cur_date.month, 'day': cur_date.day }
def train(self, x, epochs, batch_size=128, sample_interval=50, sample_path="samples/unknown", starting_epoch=0, save_interval=5000): ensure_exists(sample_path) half_batch = int(batch_size / 2) exp_replay = [] epoch = starting_epoch while epoch < epochs: # --------------------- # Train Discriminator # --------------------- # Select a random half batch of images idx = np.random.randint(0, x.shape[0], half_batch) imgs = x[idx] # Generate a half batch of new images noise = np.random.normal(0, 1, (half_batch, self.noise_size)) gen_imgs = self.generator.predict(noise) # save one random generated image for experience replay r_idx = np.random.randint(0, half_batch) exp_replay.append(gen_imgs[r_idx]) # Train the discriminator # If we have enough points, do experience replay if len(exp_replay) == half_batch: generated_images = np.array(exp_replay) d_loss_replay = self.discriminator.train_on_batch( generated_images, np.zeros((half_batch, 1))) exp_replay = [] d_loss_real = self.discriminator.train_on_batch( imgs, np.array([0.9] * half_batch)) d_loss_fake = self.discriminator.train_on_batch( gen_imgs, np.zeros((half_batch, 1))) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) # --------------------- # Train Generator # --------------------- # The generator wants the discriminator to label the generated samples # as valid (ones) valid_y = np.array([1] * batch_size) # Train the generator on random labels noise = np.random.normal(0, 1, (batch_size, self.noise_size)) g_loss = self.combined.train_on_batch(noise, valid_y) # Plot the progress print("%d [D loss: %f, acc.: %.2f%%] [G loss: %f]" % (epoch, d_loss[0], 100 * d_loss[1], g_loss)) # If at save interval => save generated image samples if epoch % sample_interval == 0: self.save_sample(epoch, sample_path) # save model if epoch % save_interval == 0: self.save(f"temp/{epoch}") epoch += 1
def simulation_store_directory(): directory = '{root_dir}/simulations'.format(root_dir=settings.root_dir) utils.ensure_exists(directory) return directory
days.sort() for year_month_day in days: if not utils.includes_day(args, year_month_day): continue if args.incremental and os.path.exists("parsed_out/%s-0.json" % year_month_day): print "skipping", year_month_day, "because already done" continue try: print "trying", year_month_day convert_to_json(year_month_day) except ParseTurnHeaderError, e: print e return # def profilemain(): # import hotshot, hotshot.stats # prof = hotshot.Profile("parse_game.prof") # prof.runcall(t) # prof.close() # stats = hotshot.stats.load("parse_game.prof") # stats.strip_dirs() # stats.sort_stats('time', 'calls') # stats.print_stats(20) if __name__ == "__main__": utils.ensure_exists("parsed_out") main()
for year_month_day in days: if not utils.includes_day(args, year_month_day): continue if args.incremental and os.path.exists( 'parsed_out/%s-0.json' % year_month_day): print 'skipping', year_month_day, 'because already done' continue try: print 'trying', year_month_day convert_to_json(year_month_day) except ParseTurnHeaderError, e: print e return # def profilemain(): # import hotshot, hotshot.stats # prof = hotshot.Profile("parse_game.prof") # prof.runcall(t) # prof.close() # stats = hotshot.stats.load("parse_game.prof") # stats.strip_dirs() # stats.sort_stats('time', 'calls') # stats.print_stats(20) if __name__ == '__main__': utils.ensure_exists('parsed_out') main()
tf.app.flags.DEFINE_float('gan_lr', 0.0005, 'learning rate for the gen in GAN training') FLAGS = tf.app.flags.FLAGS assert FLAGS.mode in [ "pretrain_gen", "pretrain_dis", "train_gan", "decode", "test" ] if FLAGS.mode == "train_gan": FLAGS.single_pass = False if FLAGS.min_dec_steps > FLAGS.max_dec_steps / 2: FLAGS.min_dec_steps = int(FLAGS.max_dec_steps / 2) ensure_exists(FLAGS.model_dir) def pretrain_generator(model, batcher, sess, val_batcher, model_saver, model_dir, val_saver, val_dir): """Repeatedly runs training iterations, logging loss to screen and writing summaries""" print("starting run_training") best_loss = None # will hold the best loss achieved so far best_loss = get_best_loss_from_chpt(val_dir) # get the val loss score coverage_loss = None hps = model.hps # this is where checkpoints of best models are saved running_avg_loss = 0 # the eval job keeps a smoother, running average loss to tell it when to
import os import time from datetime import date import utils import sys utils.ensure_exists('static/status') cmds = [ ('python scrape.py', False), # downloads gamelogs from isotropic ('python parse_game.py', True), # parses data into useable format ('python load_parsed_data.py', False), # loads data into database ('python analyze.py', False), # produces data for graphs ('python goals.py', False), ('python count_buys.py', False) ] extra_args = sys.argv[1:] # should think about how to parrallelize this for multiprocessor machines. while True: for cmd, spittable in cmds: status_fn = (date.today().isoformat() + '-' + time.strftime('%H:%M:%S') + '-' + cmd.replace(' ', '_') + '.txt') cmd = cmd + ' ' + ' '.join(extra_args) + ' 2>&1 | tee -a ' + status_fn print cmd os.system(cmd) os.system('mv %s static/status' % status_fn) print 'sleeping'
os.stat(fn).st_size <= SMALL_FILE_SIZE): print 'removing small existing file', fn os.unlink(fn) class MyURLOpener(urllib.FancyURLopener): def http_error_default(self, *args, **kwargs): urllib.URLopener.http_error_default(self, *args, **kwargs) if __name__ == '__main__': parser = utils.incremental_date_range_cmd_line_parser() args = parser.parse_args() utils.ensure_exists('static/scrape_data') os.chdir('static/scrape_data') for cur_date in utils.daterange(datetime.date(2010, 10, 15), datetime.date.today()): str_date = time.strftime("%Y%m%d", cur_date.timetuple()) if not utils.includes_day(args, str_date): print 'skipping', str_date, 'because not in cmd line arg daterange' continue directory = str_date print str_date games_short_name = str_date + '.all.tar.bz2' saved_games_bundle = directory + '/' + games_short_name if utils.at_least_as_big_as(saved_games_bundle, SMALL_FILE_SIZE): print 'skipping because exists', str_date, saved_games_bundle, \ 'and not small (size=', os.stat(saved_games_bundle).st_size, ')'
if __name__ == "__main__": INPUT_FILE = sys.argv[1] log.debug(f"input file: {INPUT_FILE}") OUTPUT_FILE: Text = os.path.join( "/tmp/vim", basename(dirname(INPUT_FILE)), re.sub( r"^(.*)\.(?:r?md|m(?:ark)?down)$", r"\1.html", basename(INPUT_FILE), re.IGNORECASE | re.MULTILINE, ), ) log.debug(f"output file: {OUTPUT_FILE}") ensure_exists(OUTPUT_FILE) with open(OUTPUT_FILE, "w", encoding="utf-8") as output: cmd = PandocCmd(INPUT_FILE) output.write(cmd.execute()) print(f"Cmd: {' '.join(cmd.args)}") print(f'Output: {output.name}') # vim:foldmethod=manual:
import os import time from datetime import date, timedelta import utils import sys utils.ensure_exists('static/status') cmds = [ 'python scrape.py', 'python parse_game.py', 'python load_parsed_data.py ', 'python analyze.py', 'python goals.py', 'python count_buys.py', 'python run_trueskill.py', 'python optimal_card_ratios.py', 'python goal_stats.py', 'python scrape_leaderboard.py', 'python load_leaderboard.py', ] extra_args = sys.argv[1:] # should think about how to parrallelize this for multiprocessor machines. while True: for cmd in cmds: month_ago = (date.today() - timedelta(days=30)).strftime('%Y%m%d') fmt_dict = {'month_ago': month_ago} cmd = (cmd % fmt_dict)
def main(argv): tf.set_random_seed(111) # a seed value for randomness # Create a batcher object that will create minibatches of data # TODO change to pass number # --------------- building graph --------------- hparam_gen = [ 'mode', 'model_dir', 'adagrad_init_acc', 'steps_per_checkpoint', 'batch_size', 'beam_size', 'cov_loss_wt', 'coverage', 'emb_dim', 'rand_unif_init_mag', 'gen_vocab_file', 'gen_vocab_size', 'hidden_dim', 'gen_lr', 'gen_max_gradient', 'max_dec_steps', 'max_enc_steps', 'min_dec_steps', 'trunc_norm_init_std', 'single_pass', 'log_root', 'data_path', ] hps_dict = {} for key, val in FLAGS.__flags.iteritems(): # for each flag if key in hparam_gen: # if it's in the list hps_dict[key] = val # add it to the dict hps_gen = namedtuple("HParams4Gen", hps_dict.keys())(**hps_dict) print("Building vocabulary for generator ...") gen_vocab = Vocab(join_path(hps_gen.data_path, hps_gen.gen_vocab_file), hps_gen.gen_vocab_size) hparam_dis = [ 'mode', 'vocab_type', 'model_dir', 'dis_vocab_size', 'steps_per_checkpoint', 'learning_rate_decay_factor', 'dis_vocab_file', 'num_class', 'layer_size', 'conv_layers', 'max_steps', 'kernel_size', 'early_stop', 'pool_size', 'pool_layers', 'dis_max_gradient', 'batch_size', 'dis_lr', 'lr_decay_factor', 'cell_type', 'max_enc_steps', 'max_dec_steps', 'single_pass', 'data_path', 'num_models', ] hps_dict = {} for key, val in FLAGS.__flags.iteritems(): # for each flag if key in hparam_dis: # if it's in the list hps_dict[key] = val # add it to the dict hps_dis = namedtuple("HParams4Dis", hps_dict.keys())(**hps_dict) if hps_gen.gen_vocab_file == hps_dis.dis_vocab_file: hps_dis = hps_dis._replace(vocab_type="word") hps_dis = hps_dis._replace(layer_size=hps_gen.emb_dim) hps_dis = hps_dis._replace(dis_vocab_size=hps_gen.gen_vocab_size) else: hps_dis = hps_dis._replace(max_enc_steps=hps_dis.max_enc_steps * 2) hps_dis = hps_dis._replace(max_dec_steps=hps_dis.max_dec_steps * 2) if FLAGS.mode == "train_gan": hps_gen = hps_gen._replace(batch_size=hps_gen.batch_size * hps_dis.num_models) if FLAGS.mode != "pretrain_dis": with tf.variable_scope("generator"): generator = PointerGenerator(hps_gen, gen_vocab) print("Building generator graph ...") gen_decoder_scope = generator.build_graph() if FLAGS.mode != "pretrain_gen": print("Building vocabulary for discriminator ...") dis_vocab = Vocab(join_path(hps_dis.data_path, hps_dis.dis_vocab_file), hps_dis.dis_vocab_size) if FLAGS.mode in ['train_gan', 'pretrain_dis']: with tf.variable_scope("discriminator"), tf.device("/gpu:0"): discriminator = Seq2ClassModel(hps_dis) print("Building discriminator graph ...") discriminator.build_graph() hparam_gan = [ 'mode', 'model_dir', 'gan_iter', 'gan_gen_iter', 'gan_dis_iter', 'gan_lr', 'rollout_num', 'sample_num', ] hps_dict = {} for key, val in FLAGS.__flags.iteritems(): # for each flag if key in hparam_gan: # if it's in the list hps_dict[key] = val # add it to the dict hps_gan = namedtuple("HParams4GAN", hps_dict.keys())(**hps_dict) hps_gan = hps_gan._replace(mode="train_gan") if FLAGS.mode == 'train_gan': with tf.device("/gpu:0"): print("Creating rollout...") rollout = Rollout(generator, 0.8, gen_decoder_scope) # --------------- initializing variables --------------- all_variables = tf.get_collection_ref(tf.GraphKeys.GLOBAL_VARIABLES) + \ tf.get_collection_ref(tf.GraphKeys.WEIGHTS) + \ tf.get_collection_ref(tf.GraphKeys.BIASES) sess = tf.Session(config=utils.get_config()) sess.run(tf.variables_initializer(all_variables)) if FLAGS.mode == "pretrain_gen": val_dir = ensure_exists( join_path(FLAGS.model_dir, 'generator', FLAGS.val_dir)) model_dir = ensure_exists(join_path(FLAGS.model_dir, 'generator')) print("Restoring the generator model from the latest checkpoint...") gen_saver = tf.train.Saver( max_to_keep=3, var_list=[ v for v in all_variables if "generator" in v.name and "GAN" not in v.name ]) gen_dir = ensure_exists(join_path(FLAGS.model_dir, "generator")) # gen_dir = ensure_exists(FLAGS.model_dir) # temp_saver = tf.train.Saver( # var_list=[v for v in all_variables if "generator" in v.name and "Adagrad" not in v.name]) ckpt_path = utils.load_ckpt(gen_saver, sess, gen_dir) print('going to restore embeddings from checkpoint') if not ckpt_path: emb_path = join_path(FLAGS.model_dir, "generator", "init_embed") if emb_path: generator.saver.restore( sess, tf.train.get_checkpoint_state( emb_path).model_checkpoint_path) print( colored( "successfully restored embeddings form %s" % emb_path, 'green')) else: print( colored("failed to restore embeddings form %s" % emb_path, 'red')) elif FLAGS.mode in ["decode", "train_gan"]: print("Restoring the generator model from the best checkpoint...") dec_saver = tf.train.Saver( max_to_keep=3, var_list=[v for v in all_variables if "generator" in v.name]) gan_dir = ensure_exists( join_path(FLAGS.model_dir, 'generator', FLAGS.gan_dir)) gan_val_dir = ensure_exists( join_path(FLAGS.model_dir, 'generator', FLAGS.gan_dir, FLAGS.val_dir)) gan_saver = tf.train.Saver( max_to_keep=3, var_list=[v for v in all_variables if "generator" in v.name]) gan_val_saver = tf.train.Saver( max_to_keep=3, var_list=[v for v in all_variables if "generator" in v.name]) utils.load_ckpt(dec_saver, sess, val_dir, (FLAGS.mode in ["train_gan", "decode"])) if FLAGS.mode in ["pretrain_dis", "train_gan"]: dis_saver = tf.train.Saver( max_to_keep=3, var_list=[v for v in all_variables if "discriminator" in v.name]) dis_dir = ensure_exists(join_path(FLAGS.model_dir, 'discriminator')) ckpt = utils.load_ckpt(dis_saver, sess, dis_dir) if not ckpt: if hps_dis.vocab_type == "word": discriminator.init_emb( sess, join_path(FLAGS.model_dir, "generator", "init_embed")) else: discriminator.init_emb( sess, join_path(FLAGS.model_dir, "discriminator", "init_embed")) # --------------- train models --------------- if FLAGS.mode != "pretrain_dis": gen_batcher_train = GenBatcher("train", gen_vocab, hps_gen, single_pass=hps_gen.single_pass) decoder = Decoder(sess, generator, gen_vocab) gen_batcher_val = GenBatcher("val", gen_vocab, hps_gen, single_pass=True) val_saver = tf.train.Saver( max_to_keep=10, var_list=[ v for v in all_variables if "generator" in v.name and "GAN" not in v.name ]) if FLAGS.mode != "pretrain_gen": dis_val_batch_size = hps_dis.batch_size * hps_dis.num_models \ if hps_dis.mode == "train_gan" else hps_dis.batch_size * hps_dis.num_models * 2 dis_batcher_val = DisBatcher( hps_dis.data_path, "eval", gen_vocab, dis_vocab, dis_val_batch_size, single_pass=True, max_art_steps=hps_dis.max_enc_steps, max_abs_steps=hps_dis.max_dec_steps, ) if FLAGS.mode == "pretrain_gen": # get reload the print('Going to pretrain the generator') try: pretrain_generator(generator, gen_batcher_train, sess, gen_batcher_val, gen_saver, model_dir, val_saver, val_dir) except KeyboardInterrupt: tf.logging.info("Caught keyboard interrupt on worker....") elif FLAGS.mode == "pretrain_dis": print('Going to pretrain the discriminator') dis_batcher = DisBatcher( hps_dis.data_path, "decode", gen_vocab, dis_vocab, hps_dis.batch_size * hps_dis.num_models, single_pass=hps_dis.single_pass, max_art_steps=hps_dis.max_enc_steps, max_abs_steps=hps_dis.max_dec_steps, ) try: pretrain_discriminator(sess, discriminator, dis_batcher_val, dis_vocab, dis_batcher, dis_saver) except KeyboardInterrupt: tf.logging.info("Caught keyboard interrupt on worker....") elif FLAGS.mode == "train_gan": gen_best_loss = get_best_loss_from_chpt(val_dir) gen_global_step = 0 print('Going to tune the two using Gan') for i_gan in range(hps_gan.gan_iter): # Train the generator for one step g_losses = [] current_speed = [] for it in range(hps_gan.gan_gen_iter): start_time = time.time() batch = gen_batcher_train.next_batch() # generate samples enc_states, dec_in_state, n_samples, n_targets_padding_mask = decoder.mc_generate( batch, include_start_token=True, s_num=hps_gan.sample_num) # get rewards for the samples n_rewards = rollout.get_reward(sess, gen_vocab, dis_vocab, batch, enc_states, dec_in_state, n_samples, hps_gan.rollout_num, discriminator) # fine tune the generator n_sample_targets = [samples[:, 1:] for samples in n_samples] n_targets_padding_mask = [ padding_mask[:, 1:] for padding_mask in n_targets_padding_mask ] n_samples = [samples[:, :-1] for samples in n_samples] # sample_target_padding_mask = pad_sample(sample_target, gen_vocab, hps_gen) n_samples = [ np.where( np.less(samples, hps_gen.gen_vocab_size), samples, np.array([[gen_vocab.word2id(data.UNKNOWN_TOKEN)] * hps_gen.max_dec_steps] * hps_gen.batch_size)) for samples in n_samples ] results = generator.run_gan_batch(sess, batch, n_samples, n_sample_targets, n_targets_padding_mask, n_rewards) gen_global_step = results["global_step"] # for visualization g_loss = results["loss"] if not math.isnan(g_loss): g_losses.append(g_loss) else: print(colored('a nan in gan loss', 'red')) current_speed.append(time.time() - start_time) # Test # if FLAGS.gan_gen_iter and (i_gan % 100 == 0 or i_gan == hps_gan.gan_iter - 1): if i_gan % 100 == 0 or i_gan == hps_gan.gan_iter - 1: print('Going to test the generator.') current_speed = sum(current_speed) / (len(current_speed) * hps_gen.batch_size) everage_g_loss = sum(g_losses) / len(g_losses) # one more process hould be opened for the evaluation eval_loss, gen_best_loss = save_ckpt( sess, generator, gen_best_loss, gan_dir, gan_saver, gen_batcher_val, gan_val_dir, gan_val_saver, gen_global_step) if eval_loss: print("\nDashboard for " + colored("GAN Generator", 'green') + " updated %s, " "finished steps:\t%s\n" "\tBatch size:\t%s\n" "\tVocabulary size:\t%s\n" "\tCurrent speed:\t%.4f seconds/article\n" "\tAverage training loss:\t%.4f; " "eval loss:\t%.4f" % ( datetime.datetime.now().strftime( "on %m-%d at %H:%M"), gen_global_step, FLAGS.batch_size, hps_gen.gen_vocab_size, current_speed, everage_g_loss.item(), eval_loss.item(), )) # Train the discriminator print('Going to train the discriminator.') dis_best_loss = 1000 dis_losses = [] dis_accuracies = [] for d_gan in range(hps_gan.gan_dis_iter): batch = gen_batcher_train.next_batch() enc_states, dec_in_state, k_samples_words, _ = decoder.mc_generate( batch, s_num=hps_gan.sample_num) # shuould first tanslate to words to avoid unk articles_oovs = batch.art_oovs for samples_words in k_samples_words: dec_batch_words = batch.target_batch conditions_words = batch.enc_batch_extend_vocab if hps_dis.vocab_type == "char": samples = gen_vocab2dis_vocab(samples_words, gen_vocab, articles_oovs, dis_vocab, hps_dis.max_dec_steps, STOP_DECODING) dec_batch = gen_vocab2dis_vocab( dec_batch_words, gen_vocab, articles_oovs, dis_vocab, hps_dis.max_dec_steps, STOP_DECODING) conditions = gen_vocab2dis_vocab( conditions_words, gen_vocab, articles_oovs, dis_vocab, hps_dis.max_enc_steps, PAD_TOKEN) else: samples = samples_words dec_batch = dec_batch_words conditions = conditions_words # the unknown in target inputs = np.concatenate([samples, dec_batch], 0) conditions = np.concatenate([conditions, conditions], 0) targets = [[1, 0] for _ in samples] + [[0, 1] for _ in dec_batch] targets = np.array(targets) # randomize the samples assert len(inputs) == len(conditions) == len( targets ), "lengthes of the inputs, conditions and targests should be the same." indices = np.random.permutation(len(inputs)) inputs = np.split(inputs[indices], 2) conditions = np.split(conditions[indices], 2) targets = np.split(targets[indices], 2) assert len(inputs) % 2 == 0, "the length should be mean" results = discriminator.run_one_batch( sess, inputs[0], conditions[0], targets[0]) dis_accuracies.append(results["accuracy"].item()) dis_losses.append(results["loss"].item()) results = discriminator.run_one_batch( sess, inputs[1], conditions[1], targets[1]) dis_accuracies.append(results["accuracy"].item()) ave_dis_acc = sum(dis_accuracies) / len(dis_accuracies) if d_gan == hps_gan.gan_dis_iter - 1: if (sum(dis_losses) / len(dis_losses)) < dis_best_loss: dis_best_loss = sum(dis_losses) / len(dis_losses) checkpoint_path = ensure_exists( join_path(hps_dis.model_dir, "discriminator")) + "/model.ckpt" dis_saver.save(sess, checkpoint_path, global_step=results["global_step"]) print_dashboard("GAN Discriminator", results["global_step"].item(), hps_dis.batch_size, hps_dis.dis_vocab_size, results["loss"].item(), 0.00, 0.00, 0.00) print("Average training accuracy: \t%.4f" % ave_dis_acc) if ave_dis_acc > 0.9: break # --------------- decoding samples --------------- elif FLAGS.mode == "decode": print('Going to decode from the generator.') decoder.bs_decode(gen_batcher_train) print("Finished decoding..") # decode for generating corpus for discriminator sess.close()
# and since these are shown, they are 1 indexed show_turn_id = '%s-show-turn-%d' % (player, turn_no + 1) ret += '<div id="%s"></div>' % turn_id ret += '<a name="%s"></a><a href="#%s">%s</a>' % ( show_turn_id, show_turn_id, cur_match.group()) contents = contents[cur_match.end():] else: break before_end = contents.find('</html') ret = ret + contents[:before_end] ret += '<div id="end-game">\n' ret += '</div> <br>\n' * 10 contents = contents[before_end:] return ret + contents # def profilemain(): # import hotshot, hotshot.stats # prof = hotshot.Profile("parse_game.prof") # prof.runcall(t) # prof.close() # stats = hotshot.stats.load("parse_game.prof") # stats.strip_dirs() # stats.sort_stats('time', 'calls') # stats.print_stats(20) if __name__ == '__main__': utils.ensure_exists('parsed_out') main()
def train(self, x, epochs, k_lambda, gamma, batch_size=16, sample_interval=50, sample_path="samples/unknown", starting_epoch=0, save_interval=5000): ensure_exists(sample_path) epoch = starting_epoch steps_per_epoch = len(x) // batch_size decay_every = 16000 initial_lr = 0.0001 step = tf.Variable(0) lr = initial_lr * pow(0.5, epoch // decay_every) print(f"Initialized initial learning rate at {lr}") self.optimizer = tf.train.AdamOptimizer(lr) # set up train operations # dataset def datagen(): while True: idx = np.random.randint(0, x.shape[0], batch_size) imgs = x[idx] yield imgs dataset = tf.data.Dataset.from_generator(datagen, tf.float32, tf.TensorShape( (batch_size, self.img_rows, self.img_cols, self.channels))).repeat() # generator noise = tf.random_uniform((x.shape[0], batch_size), -1.0, 1.0) fake = self.generator(noise) g_loss = self.l1loss(fake, self.discriminator(fake)) # discriminator real = dataset.get_next() d_loss_real = self.l1loss(real, self.discriminator(real)) d_loss_gen = g_loss d_loss = d_loss_real - self.k * d_loss_gen # training g_opt = self.optimizer.minimize(g_loss, var_list=self.generator.trainable_variables) d_opt = self.optimizer.minimize(d_loss, global_step=step, var_list=self.discriminator.trainable_variables) # updating balance = gamma * d_loss_real - g_loss measure = d_loss_real + tf.abs(balance) with tf.control_dependencies([g_opt, d_opt]): k_update = tf.assign(self.k, tf.clip_by_value(self.k + k_lambda * balance, 0, 1)) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) while epoch < epochs: result = sess.run({ "k_update": k_update, "m": measure, "g_loss": g_loss, "d_loss_real": d_loss_real, "d_loss_fake": d_loss_gen, "d_loss": d_loss, "k": self.k }) # --------------------- # LR Decay # --------------------- if epoch % decay_every == 0: lr = initial_lr * pow(0.5, epoch // decay_every) print(f"Decaying learning rate to {lr}") self.optimizer = tf.train.AdamOptimizer(lr) # --------------------- # Status report # --------------------- # calculate convergence factor # Plot the progress print("%d [M: %f] [D loss: %f = %f + %f] [G loss: %f] [K: %f]" % ( epoch, result['m'], result['d_loss'], result['d_loss_real'], result['d_loss_fake'], result['g_loss'], result['k'])) # If at save interval => save generated image samples if epoch % sample_interval == 0: self.save_sample(epoch, sample_path, x, sess) # save model if epoch % save_interval == 0: self.save(f"temp/{epoch}") epoch += 1
def train(self, x, y, epochs, batch_size=128, sample_interval=50, sample_path="samples/unknown", starting_epoch=0, save_interval=5000): """ Trains the GAN. :param x: The training data. :param y: The labels for the training data. :param epochs: The number of epochs to train. :param batch_size: The size of an epoch. :param sample_interval: How often to save sample images. :param sample_path: Where to save sample images. """ ensure_exists(sample_path) half_batch = int(batch_size / 2) exp_replay = [] epoch = starting_epoch while epoch < epochs: # --------------------- # Train Discriminator # --------------------- # Select a random half batch of images idx = np.random.randint(0, x.shape[0], half_batch) imgs = x[idx] labels = y[idx] # Generate a half batch of new images noise = np.random.normal(0, 1, (half_batch, self.noise_size)) gen_imgs = self.generator.predict({"noise": noise, "label": labels}) # save one random generated image for experience replay r_idx = np.random.randint(0, half_batch) exp_replay.append((gen_imgs[r_idx], labels[r_idx])) # Train the discriminator # If we have enough points, do experience replay if len(exp_replay) == half_batch: generated_images = np.array([p[0] for p in exp_replay]) labels = np.array([p[1] for p in exp_replay]) d_loss_replay = self.discriminator.train_on_batch([generated_images, labels], np.zeros((half_batch, 1))) exp_replay = [] d_loss_real = self.discriminator.train_on_batch({"image": imgs, "label": labels}, np.ones((half_batch, 1))) d_loss_fake = self.discriminator.train_on_batch({"image": gen_imgs, "label": labels}, np.zeros((half_batch, 1))) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) # --------------------- # Train Generator # --------------------- # The generator wants the discriminator to label the generated samples # as valid (ones) valid_y = np.array([1] * batch_size) # Train the generator on random labels noise = np.random.normal(0, 1, (batch_size, self.noise_size)) idx = np.random.randint(0, x.shape[0], batch_size) labels = y[idx] g_loss = self.combined.train_on_batch({"noise": noise, "label": labels}, valid_y) # Plot the progress print("%d [D loss: %f, acc.: %.2f%%] [G loss: %f]" % (epoch, d_loss[0], 100 * d_loss[1], g_loss)) # If at save interval => save generated image samples if epoch % sample_interval == 0: self.save_sample(epoch, sample_path) # save model if epoch % save_interval == 0: self.save(f"temp/{epoch}") epoch += 1
def trainNet(net, opt, cri, sch, cp_dir, epoch_range, training_set, val_set, batch_size_big, rep=1): tb_dir = os.path.join(cp_dir, 'tb') utils.ensure_exists(tb_dir) fout = open(os.path.join(cp_dir, 'train.log'), 'a') for epoch in epoch_range: net.train() running_loss = 0 start_time = time.time() tl = len(training_set) for e in range(rep): for i, data in enumerate(training_set, 0): com = data['com'].float().cuda() org = data['org'].float().cuda() opt.zero_grad() if com.shape[0] > batch_size_big: com_big = com[:batch_size_big, :, :, :] org_big = org[:batch_size_big, :, :, :] ret_big = net(com_big) loss_big = cri(ret_big, org_big) _, _, h, w = com.shape new_h = 32 new_w = 32 top = 8 * np.random.randint(0, (h - new_h) // 8) left = 8 * np.random.randint(0, (w - new_w) // 8) com_small = com[batch_size_big:, :, top:top + new_h, left:left + new_w] org_small = org[batch_size_big:, :, top:top + new_h, left:left + new_w] ret_small = net(com_small) loss_small = cri(ret_small, org_small) loss = loss_big + loss_small loss.backward() nn.utils.clip_grad_norm_(net.parameters(), 5) opt.step() else: ret = net(com) loss = cri(ret, org) loss.backward() nn.utils.clip_grad_norm_(net.parameters(), 5) opt.step() running_loss += loss.item() if i % 100 == 0: print('[Running epoch %2d, batch %4d] loss: %.3f' % (epoch + 1, i + 1, \ 10000 * running_loss / (e * tl + i + 1), ), end='\n') else: print('[Running epoch %2d, batch %4d] loss: %.3f' % (epoch + 1, i + 1, \ 10000 * running_loss / (e * tl + i + 1), ), end='\r') if not (epoch + 1) % 1: timestamp = time.time() print('[timestamp %d, epoch %2d] loss: %.3f, time: %6ds ' % (timestamp, epoch + 1, 10000 * running_loss / ((i + 1) * rep), timestamp - start_time), end='\n') with torch.no_grad(): p_psnr = utils.evalPsnr(net, val_set, fout=fout) save_model(net, opt, os.path.join(cp_dir, str(epoch + 1) + '_withopt')) torch.save(net.state_dict(), os.path.join(cp_dir, str(epoch + 1))) sch.step() print('cur_lr: %.5f' % sch.get_lr()[0])
def trainNet(net, opt, cri, sch, cp_dir, epoch_range, training_set, val_set, batch_size_big, rep=1): tb_dir = os.path.join(cp_dir, 'tb') utils.ensure_exists(tb_dir) fout = open(os.path.join(cp_dir, 'train.log'), 'a') for epoch in epoch_range: net.train() running_loss = 0 start_time = time.time() tl = len(training_set) for e in range(rep): for i, data in enumerate(training_set, 0): com = data['com'].float().cuda() c_2 = data['com_2'].float().cuda() c_4 = data['com_4'].float().cuda() org = data['org'].float().cuda() o_2 = data['org_2'].float().cuda() o_4 = data['org_4'].float().cuda() com_pair = (c_4, c_2, com) org_pair = (o_4, o_2, org) opt.zero_grad() ret = net(com_pair) loss, MSE4, MSE2, MSEp, MSEd = cri(ret, org_pair) loss.backward() nn.utils.clip_grad_norm_(net.parameters(), 10) opt.step() running_loss += loss.item() if i % 100 == 0: print('[Running epoch %2d, batch %4d] loss: %.3f' % (epoch + 1, i + 1, \ 10000 * running_loss / (e * tl + i + 1), ), end='\n') else: print('[Running epoch %2d, batch %4d] loss: %.3f' % (epoch + 1, i + 1, \ 10000 * running_loss / (e * tl + i + 1), ), end='\r') if not (epoch + 1) % 1: timestamp = time.time() print('[timestamp %d, epoch %2d] loss: %.3f, time: %6ds ' % (timestamp, epoch + 1, 10000 * running_loss / ((i + 1) * rep), timestamp - start_time), end='\n') with torch.no_grad(): p_psnr = utils.evalPsnr(net, val_set, fout=fout) save_model(net, opt, os.path.join(cp_dir, str(epoch + 1) + '_withopt')) torch.save(net.state_dict(), os.path.join(cp_dir, str(epoch + 1))) sch.step() print('cur_lr: %.5f' % sch.get_lr()[0])