def model_fn(features, labels, mode, params): global printed_model_yet global initial_weights_already_loaded print_model = not printed_model_yet built = ModelUtils.build_model_from_tfrecords_features( features, mode, print_model, trainlog, model_config, pos_len, num_batches_per_epoch, lr_scale) if mode == tf.estimator.ModeKeys.PREDICT: model = built predictions = {} predictions["policy_output"] = model.policy_output predictions["value_output"] = model.value_output return tf.estimator.EstimatorSpec(mode, predictions=predictions) if mode == tf.estimator.ModeKeys.EVAL: (model, target_vars, metrics) = built wsum = tf.Variable(0.0, dtype=tf.float32, name="wsum", trainable=False, collections=[ tf.GraphKeys.LOCAL_VARIABLES, tf.GraphKeys.METRIC_VARIABLES ], synchronization=tf.VariableSynchronization.ON_READ, aggregation=tf.VariableAggregation.SUM) wsum_op = tf.assign_add(wsum, target_vars.weight_sum) return tf.estimator.EstimatorSpec( mode, loss=target_vars.opt_loss / tf.constant(batch_size, dtype=tf.float32), eval_metric_ops={ "wsum": (wsum.read_value(), wsum_op), "p0loss": tf.metrics.mean(target_vars.policy_loss_unreduced, weights=target_vars.target_weight_used), "p1loss": tf.metrics.mean(target_vars.policy1_loss_unreduced, weights=target_vars.target_weight_used), "vloss": tf.metrics.mean(target_vars.value_loss_unreduced, weights=target_vars.target_weight_used), "smloss": tf.metrics.mean(target_vars.scoremean_loss_unreduced, weights=target_vars.target_weight_used), "sbpdfloss": tf.metrics.mean(target_vars.scorebelief_pdf_loss_unreduced, weights=target_vars.target_weight_used), "sbcdfloss": tf.metrics.mean(target_vars.scorebelief_cdf_loss_unreduced, weights=target_vars.target_weight_used), "bbpdfloss": tf.metrics.mean(target_vars.bonusbelief_pdf_loss_unreduced, weights=target_vars.target_weight_used), "bbcdfloss": tf.metrics.mean(target_vars.bonusbelief_cdf_loss_unreduced, weights=target_vars.target_weight_used), "uvloss": tf.metrics.mean(target_vars.utilityvar_loss_unreduced, weights=target_vars.target_weight_used), "oloss": tf.metrics.mean(target_vars.ownership_loss_unreduced, weights=target_vars.target_weight_used), "rwlloss": tf.metrics.mean(target_vars.winloss_reg_loss_unreduced, weights=target_vars.target_weight_used), "rsmloss": tf.metrics.mean(target_vars.scoremean_reg_loss_unreduced, weights=target_vars.target_weight_used), "rsdloss": tf.metrics.mean(target_vars.scorestdev_reg_loss_unreduced, weights=target_vars.target_weight_used), "roloss": tf.metrics.mean(target_vars.ownership_reg_loss_unreduced, weights=target_vars.target_weight_used), "rloss": tf.metrics.mean(target_vars.reg_loss_per_weight, weights=target_vars.weight_sum), "rscloss": tf.metrics.mean(target_vars.scale_reg_loss_unreduced, weights=target_vars.target_weight_used), "pacc1": tf.metrics.mean(metrics.accuracy1_unreduced, weights=target_vars.target_weight_used), "ventr": tf.metrics.mean(metrics.value_entropy_unreduced, weights=target_vars.target_weight_used), "ptentr": tf.metrics.mean(metrics.policy_target_entropy_unreduced, weights=target_vars.target_weight_used) }) if mode == tf.estimator.ModeKeys.TRAIN: (model, target_vars, metrics, global_step, global_step_float, per_sample_learning_rate, train_step) = built printed_model_yet = True def moving_mean(name, x, weights): sumwx = tf.reduce_sum(x * weights) sumw = tf.reduce_sum(weights, name="printstats/" + name) ema = tf.train.ExponentialMovingAverage(decay=0.999) op = ema.apply([sumwx, sumw]) avg = ema.average(sumwx) / ema.average(sumw) return (avg, op) (p0loss, p0loss_op) = moving_mean("p0loss", target_vars.policy_loss_unreduced, weights=target_vars.target_weight_used) (p1loss, p1loss_op) = moving_mean("p1loss", target_vars.policy1_loss_unreduced, weights=target_vars.target_weight_used) (vloss, vloss_op) = moving_mean("vloss", target_vars.value_loss_unreduced, weights=target_vars.target_weight_used) (smloss, smloss_op) = moving_mean("smloss", target_vars.scoremean_loss_unreduced, weights=target_vars.target_weight_used) (sbpdfloss, sbpdfloss_op) = moving_mean( "sbpdfloss", target_vars.scorebelief_pdf_loss_unreduced, weights=target_vars.target_weight_used) (sbcdfloss, sbcdfloss_op) = moving_mean( "sbcdfloss", target_vars.scorebelief_cdf_loss_unreduced, weights=target_vars.target_weight_used) (bbpdfloss, bbpdfloss_op) = moving_mean( "bbpdfloss", target_vars.bonusbelief_pdf_loss_unreduced, weights=target_vars.target_weight_used) (bbcdfloss, bbcdfloss_op) = moving_mean( "bbcdfloss", target_vars.bonusbelief_cdf_loss_unreduced, weights=target_vars.target_weight_used) (uvloss, uvloss_op) = moving_mean("uvloss", target_vars.utilityvar_loss_unreduced, weights=target_vars.target_weight_used) (oloss, oloss_op) = moving_mean("oloss", target_vars.ownership_loss_unreduced, weights=target_vars.target_weight_used) (rwlloss, rwlloss_op) = moving_mean("rwlloss", target_vars.winloss_reg_loss_unreduced, weights=target_vars.target_weight_used) (rsmloss, rsmloss_op) = moving_mean("rsmloss", target_vars.scoremean_reg_loss_unreduced, weights=target_vars.target_weight_used) (rsdloss, rsdloss_op) = moving_mean("rsdloss", target_vars.scorestdev_reg_loss_unreduced, weights=target_vars.target_weight_used) (roloss, roloss_op) = moving_mean("roloss", target_vars.ownership_reg_loss_unreduced, weights=target_vars.target_weight_used) (rloss, rloss_op) = moving_mean("rloss", target_vars.reg_loss_per_weight, weights=target_vars.weight_sum) (rscloss, rscloss_op) = moving_mean("rscloss", target_vars.scale_reg_loss_unreduced, weights=target_vars.target_weight_used) (pacc1, pacc1_op) = moving_mean("pacc1", metrics.accuracy1_unreduced, weights=target_vars.target_weight_used) (ventr, ventr_op) = moving_mean("ventr", metrics.value_entropy_unreduced, weights=target_vars.target_weight_used) (ptentr, ptentr_op) = moving_mean("ptentr", metrics.policy_target_entropy_unreduced, weights=target_vars.target_weight_used) (wmean, wmean_op) = tf.metrics.mean(target_vars.weight_sum) print_train_loss_every_batches = 100 logging_hook = tf.train.LoggingTensorHook( { "nsamp": global_step * tf.constant(batch_size, dtype=tf.int64), "wsum": global_step_float * wmean, "p0loss": p0loss, "p1loss": p1loss, "vloss": vloss, "smloss": smloss, "sbpdfloss": sbpdfloss, "sbcdfloss": sbcdfloss, "bbpdfloss": bbpdfloss, "bbcdfloss": bbcdfloss, "uvloss": uvloss, "oloss": oloss, "rwlloss": rwlloss, "rsmloss": rsmloss, "rsdloss": rsdloss, "roloss": roloss, "rloss": rloss, "rscloss": rscloss, "pacc1": pacc1, "ventr": ventr, "ptentr": ptentr, "pslr": per_sample_learning_rate }, every_n_iter=print_train_loss_every_batches) printed_model_yet = True sys.stdout.flush() sys.stderr.flush() initial_weights_dir = os.path.join(traindir, "initial_weights") if os.path.exists( initial_weights_dir) and not initial_weights_already_loaded: print("Initial weights found at: " + initial_weights_dir) checkpoint_path = os.path.join(initial_weights_dir, "model") vars_in_checkpoint = tf.contrib.framework.list_variables( checkpoint_path) print("Checkpoint contains:") for var in vars_in_checkpoint: print(var) print( "Modifying graph to load weights from checkpoint upon init...") sys.stdout.flush() sys.stderr.flush() variables_to_restore = tf.trainable_variables() assignment_mapping = { v.name.split(":")[0]: v for v in variables_to_restore } tf.train.init_from_checkpoint(checkpoint_path, assignment_mapping) initial_weights_already_loaded = True return tf.estimator.EstimatorSpec( mode, loss=(target_vars.opt_loss / tf.constant(batch_size, dtype=tf.float32)), train_op=tf.group(train_step, p0loss_op, p1loss_op, vloss_op, smloss_op, sbpdfloss_op, sbcdfloss_op, bbpdfloss_op, bbcdfloss_op, uvloss_op, oloss_op, rwlloss_op, rsmloss_op, rsdloss_op, roloss_op, rloss_op, rscloss_op, pacc1_op, ventr_op, ptentr_op, wmean_op), training_hooks=[logging_hook])
dataset = dataset.map(parse_input) iterator = dataset.make_one_shot_iterator() features = iterator.get_next() elif using_npz: features = tfrecordio.make_raw_input_feature_placeholders( model_config, pos_len, batch_size) # Model ---------------------------------------------------------------- mode = tf.estimator.ModeKeys.EVAL print_model = False if name_scope is not None: with tf.compat.v1.variable_scope(name_scope): (model, target_vars, metrics) = ModelUtils.build_model_from_tfrecords_features( features, mode, print_model, log, model_config, pos_len, batch_size) else: (model, target_vars, metrics) = ModelUtils.build_model_from_tfrecords_features( features, mode, print_model, log, model_config, pos_len, batch_size) total_parameters = 0 for variable in tf.compat.v1.trainable_variables(): shape = variable.get_shape() variable_parameters = 1 for dim in shape: variable_parameters *= dim.value total_parameters += variable_parameters log("Built model, %d total parameters" % total_parameters)
def model_fn(features, labels, mode, params): global printed_model_yet global initial_weights_already_loaded print_model = not printed_model_yet built = ModelUtils.build_model_from_tfrecords_features( features, mode, print_model, trainlog, model_config, pos_len, batch_size, lr_scale, gnorm_clip_scale, num_gpus_used) if mode == tf.estimator.ModeKeys.PREDICT: model = built predictions = {} predictions["policy_output"] = model.policy_output predictions["value_output"] = model.value_output return tf.estimator.EstimatorSpec(mode, predictions=predictions) if mode == tf.estimator.ModeKeys.EVAL: (model, target_vars, metrics) = built wsum = tf.Variable(0.0, dtype=tf.float32, name="wsum", trainable=False, collections=[ tf.compat.v1.GraphKeys.LOCAL_VARIABLES, tf.compat.v1.GraphKeys.METRIC_VARIABLES ], synchronization=tf.VariableSynchronization.ON_READ, aggregation=tf.VariableAggregation.SUM) wsum_op = tf.assign_add(wsum, target_vars.weight_sum) eval_metric_ops = { #"wsum": (wsum.read_value(),wsum_op), "p0loss": tf.compat.v1.metrics.mean(target_vars.policy_loss_unreduced, weights=target_vars.target_weight_used), "p1loss": tf.compat.v1.metrics.mean(target_vars.policy1_loss_unreduced, weights=target_vars.target_weight_used), "vloss": tf.compat.v1.metrics.mean(target_vars.value_loss_unreduced, weights=target_vars.target_weight_used), "tdvloss": tf.compat.v1.metrics.mean(target_vars.td_value_loss_unreduced, weights=target_vars.target_weight_used), "smloss": tf.compat.v1.metrics.mean(target_vars.scoremean_loss_unreduced, weights=target_vars.target_weight_used), "leadloss": tf.compat.v1.metrics.mean(target_vars.lead_loss_unreduced, weights=target_vars.target_weight_used), "vtimeloss": tf.compat.v1.metrics.mean(target_vars.variance_time_loss_unreduced, weights=target_vars.target_weight_used), "sbpdfloss": tf.compat.v1.metrics.mean( target_vars.scorebelief_pdf_loss_unreduced, weights=target_vars.target_weight_used), "sbcdfloss": tf.compat.v1.metrics.mean( target_vars.scorebelief_cdf_loss_unreduced, weights=target_vars.target_weight_used), "oloss": tf.compat.v1.metrics.mean(target_vars.ownership_loss_unreduced, weights=target_vars.target_weight_used), "sloss": tf.compat.v1.metrics.mean(target_vars.scoring_loss_unreduced, weights=target_vars.target_weight_used), "fploss": tf.compat.v1.metrics.mean(target_vars.futurepos_loss_unreduced, weights=target_vars.target_weight_used), "rsdloss": tf.compat.v1.metrics.mean( target_vars.scorestdev_reg_loss_unreduced, weights=target_vars.target_weight_used), "rloss": tf.compat.v1.metrics.mean(target_vars.reg_loss_per_weight, weights=target_vars.weight_sum), "rscloss": tf.compat.v1.metrics.mean(target_vars.scale_reg_loss_unreduced, weights=target_vars.target_weight_used), "pacc1": tf.compat.v1.metrics.mean(metrics.accuracy1_unreduced, weights=target_vars.target_weight_used), "ventr": tf.compat.v1.metrics.mean(metrics.value_entropy_unreduced, weights=target_vars.target_weight_used), "ptentr": tf.compat.v1.metrics.mean(metrics.policy_target_entropy_unreduced, weights=target_vars.target_weight_used) } if model.version >= 9: eval_metric_ops["evstloss"] = tf.compat.v1.metrics.mean( target_vars.shortterm_value_error_loss_unreduced, weights=target_vars.target_weight_used) eval_metric_ops["esstloss"] = tf.compat.v1.metrics.mean( target_vars.shortterm_score_error_loss_unreduced, weights=target_vars.target_weight_used) if model.version >= 10: eval_metric_ops["tdsloss"] = tf.compat.v1.metrics.mean( target_vars.td_score_loss_unreduced, weights=target_vars.target_weight_used) return tf.estimator.EstimatorSpec( mode, loss=target_vars.opt_loss / tf.constant(batch_size, dtype=tf.float32), eval_metric_ops=eval_metric_ops) if mode == tf.estimator.ModeKeys.TRAIN: (model, target_vars, metrics, global_step, global_step_float, per_sample_learning_rate, train_step) = built printed_model_yet = True def moving_mean(name, x, weights): sumwx = tf.reduce_sum(x * weights, name="printstats/wx/" + name) sumw = tf.reduce_sum(weights, name="printstats/w/" + name) moving_wx = tf.compat.v1.get_variable(initializer=tf.zeros([]), name=(name + "/moving_wx"), trainable=False) moving_w = tf.compat.v1.get_variable(initializer=tf.zeros([]), name=(name + "/moving_w"), trainable=False) decay = 0.999 with tf.compat.v1.variable_scope(name): wx_op = tf.keras.backend.moving_average_update( moving_wx, sumwx, decay) w_op = tf.keras.backend.moving_average_update( moving_w, sumw, decay) op = tf.group(wx_op, w_op) avg = (moving_wx + sumwx * (1.0 - decay)) / (moving_w + sumw * (1.0 - decay)) return (avg, op) (p0loss, p0loss_op) = moving_mean("p0loss", target_vars.policy_loss_unreduced, weights=target_vars.target_weight_used) (p1loss, p1loss_op) = moving_mean("p1loss", target_vars.policy1_loss_unreduced, weights=target_vars.target_weight_used) (vloss, vloss_op) = moving_mean("vloss", target_vars.value_loss_unreduced, weights=target_vars.target_weight_used) (tdvloss, tdvloss_op) = moving_mean("tdvloss", target_vars.td_value_loss_unreduced, weights=target_vars.target_weight_used) (smloss, smloss_op) = moving_mean("smloss", target_vars.scoremean_loss_unreduced, weights=target_vars.target_weight_used) (leadloss, leadloss_op) = moving_mean("leadloss", target_vars.lead_loss_unreduced, weights=target_vars.target_weight_used) (vtimeloss, vtimeloss_op) = moving_mean("vtimeloss", target_vars.variance_time_loss_unreduced, weights=target_vars.target_weight_used) (sbpdfloss, sbpdfloss_op) = moving_mean( "sbpdfloss", target_vars.scorebelief_pdf_loss_unreduced, weights=target_vars.target_weight_used) (sbcdfloss, sbcdfloss_op) = moving_mean( "sbcdfloss", target_vars.scorebelief_cdf_loss_unreduced, weights=target_vars.target_weight_used) (oloss, oloss_op) = moving_mean("oloss", target_vars.ownership_loss_unreduced, weights=target_vars.target_weight_used) (sloss, sloss_op) = moving_mean("sloss", target_vars.scoring_loss_unreduced, weights=target_vars.target_weight_used) (fploss, fploss_op) = moving_mean("fploss", target_vars.futurepos_loss_unreduced, weights=target_vars.target_weight_used) (skloss, skloss_op) = moving_mean("skloss", target_vars.seki_loss_unreduced, weights=target_vars.target_weight_used) (rsdloss, rsdloss_op) = moving_mean("rsdloss", target_vars.scorestdev_reg_loss_unreduced, weights=target_vars.target_weight_used) (rloss, rloss_op) = moving_mean("rloss", target_vars.reg_loss_per_weight, weights=target_vars.weight_sum) (rscloss, rscloss_op) = moving_mean("rscloss", target_vars.scale_reg_loss_unreduced, weights=target_vars.target_weight_used) if model.version >= 9: (evstloss, evstloss_op) = moving_mean( "evstloss", target_vars.shortterm_value_error_loss_unreduced, weights=target_vars.target_weight_used) (esstloss, esstloss_op) = moving_mean( "esstloss", target_vars.shortterm_score_error_loss_unreduced, weights=target_vars.target_weight_used) # (evstm,evstm_op) = moving_mean("evstm",metrics.shortterm_value_error_mean_unreduced, weights=target_vars.target_weight_used) # (evstv,evstv_op) = moving_mean("evstv",metrics.shortterm_value_error_var_unreduced, weights=target_vars.target_weight_used) # (esstm,esstm_op) = moving_mean("esstm",metrics.shortterm_score_error_mean_unreduced, weights=target_vars.target_weight_used) # (esstv,esstv_op) = moving_mean("esstv",metrics.shortterm_score_error_var_unreduced, weights=target_vars.target_weight_used) if model.version >= 10: (tdsloss, tdsloss_op) = moving_mean("tdsloss", target_vars.td_score_loss_unreduced, weights=target_vars.target_weight_used) (pacc1, pacc1_op) = moving_mean("pacc1", metrics.accuracy1_unreduced, weights=target_vars.target_weight_used) (ptentr, ptentr_op) = moving_mean("ptentr", metrics.policy_target_entropy_unreduced, weights=target_vars.target_weight_used) #NOTE: These two are going to be smaller if using more GPUs since it's the gradient norm as measured on the instance batch #rather than the global batch. #Also, somewhat awkwardly, we say the weight is 1.0 rather than 1.0/num_gpus_used because tensorflow seems to have "meany" #behavior where it updates sumw via the mean of the two separate updates of the gpus rather than the sum. (gnorm, gnorm_op) = moving_mean("gnorm", metrics.gnorm, weights=1.0) (exgnorm, exgnorm_op) = moving_mean("excessgnorm", metrics.excess_gnorm, weights=1.0) (wmean, wmean_op) = tf.compat.v1.metrics.mean(target_vars.weight_sum) # print_op = tf.print( # metrics.gnorm, # target_vars.weight_sum, # target_vars.opt_loss, # metrics.tmp, # foo[0], # output_stream=sys.stdout # ) print_train_loss_every_batches = 100 logvars = { "nsamp": global_step * tf.constant(batch_size, dtype=tf.int64), "wsum": global_step_float * wmean * tf.constant(float(num_gpus_used)), "p0loss": p0loss, "p1loss": p1loss, "vloss": vloss, "tdvloss": tdvloss, "smloss": smloss, "leadloss": leadloss, "vtimeloss": vtimeloss, "sbpdfloss": sbpdfloss, "sbcdfloss": sbcdfloss, "oloss": oloss, "sloss": sloss, "fploss": fploss, "skloss": skloss, "skw": target_vars.seki_weight_scale, "rsdloss": rsdloss, "rloss": rloss, "rscloss": rscloss, "pacc1": pacc1, "ptentr": ptentr, "pslr": per_sample_learning_rate, "gnorm": gnorm, "exgnorm": exgnorm } if model.version >= 9: logvars["evstloss"] = evstloss logvars["esstloss"] = esstloss # logvars["evstm"] = evstm # logvars["evstv"] = evstv # logvars["esstm"] = esstm # logvars["esstv"] = esstv if model.version >= 10: logvars["tdsloss"] = tdsloss logging_hook = CustomLoggingHook( logvars, every_n_iter=print_train_loss_every_batches, handle_logging_values=update_global_latest_extra_stats) printed_model_yet = True sys.stdout.flush() sys.stderr.flush() initial_weights_dir = os.path.join(traindir, "initial_weights") if os.path.exists( initial_weights_dir) and not initial_weights_already_loaded: print("Initial weights dir found at: " + initial_weights_dir) checkpoint_path = None for initial_weights_file in os.listdir(initial_weights_dir): if initial_weights_file.startswith( "model") and initial_weights_file.endswith(".index"): checkpoint_path = os.path.join( initial_weights_dir, initial_weights_file[0:len(initial_weights_file) - len(".index")]) break if checkpoint_path is not None: print("Initial weights checkpoint to use found at: " + checkpoint_path) vars_in_checkpoint = tf.contrib.framework.list_variables( checkpoint_path) varname_in_checkpoint = {} print("Checkpoint contains:") for varandshape in vars_in_checkpoint: print(varandshape) varname_in_checkpoint[varandshape[0]] = True print( "Modifying graph to load weights from checkpoint upon init..." ) sys.stdout.flush() sys.stderr.flush() variables_to_restore = tf.compat.v1.global_variables() assignment_mapping = {} for v in variables_to_restore: name = v.name.split(":")[ 0] # drop the ":0" at the end of each var if name in varname_in_checkpoint: assignment_mapping[name] = v tf.compat.v1.train.init_from_checkpoint( checkpoint_path, assignment_mapping) initial_weights_already_loaded = True ops = [ train_step, p0loss_op, p1loss_op, vloss_op, tdvloss_op, smloss_op, leadloss_op, vtimeloss_op, sbpdfloss_op, sbcdfloss_op, oloss_op, sloss_op, fploss_op, skloss_op, rsdloss_op, rloss_op, rscloss_op, pacc1_op, ptentr_op, wmean_op, gnorm_op, exgnorm_op ] if model.version >= 9: ops.append(evstloss_op) ops.append(esstloss_op) # ops.append(evstm_op) # ops.append(evstv_op) # ops.append(esstm_op) # ops.append(esstv_op) if model.version >= 10: ops.append(tdsloss_op) return tf.estimator.EstimatorSpec( mode, loss=(target_vars.opt_loss / tf.constant(batch_size, dtype=tf.float32)), train_op=tf.group(*ops), training_hooks=[logging_hook])
"ptncm": tf.placeholder(tf.float32,[batch_size,NUM_POLICY_TARGETS,pos_len*pos_len+1]), "gtnc": tf.placeholder(tf.float32,[batch_size,NUM_GLOBAL_TARGETS]), "sdn": tf.placeholder(tf.float32,[batch_size,pos_len*pos_len*2+EXTRA_SCORE_DISTR_RADIUS*2]), "sbsn": tf.placeholder(tf.float32,[batch_size,BONUS_SCORE_RADIUS*2+1]), "vtnchw": tf.placeholder(tf.float32,[batch_size,NUM_VALUE_SPATIAL_TARGETS,pos_len,pos_len]) } # Model ---------------------------------------------------------------- mode = tf.estimator.ModeKeys.EVAL print_model = False num_batches_per_epoch = 1 #doesn't matter if name_scope is not None: with tf.name_scope(name_scope): (model,target_vars,metrics) = ModelUtils.build_model_from_tfrecords_features(features,mode,print_model,log,model_config,pos_len,num_batches_per_epoch) else: (model,target_vars,metrics) = ModelUtils.build_model_from_tfrecords_features(features,mode,print_model,log,model_config,pos_len,num_batches_per_epoch) total_parameters = 0 for variable in tf.trainable_variables(): shape = variable.get_shape() variable_parameters = 1 for dim in shape: variable_parameters *= dim.value total_parameters += variable_parameters log("Built model, %d total parameters" % total_parameters) # Testing ------------------------------------------------------------