def main(_): logging.set_verbosity(logging.INFO) tf.enable_v2_behavior() tf.enable_resource_variables() tf.enable_control_flow_v2() logging.info('Executing eagerly: %s', tf.executing_eagerly()) logging.info('parsing config files: %s', FLAGS.gin_file) gin.parse_config_files_and_bindings( FLAGS.gin_file, FLAGS.gin_bindings, skip_unknown=True) trainer.train(root_dir, eval_metrics_callback=metrics_callback)
def predict_with_saved_model(feature): # [0, 1] is snoring # [1, 0] is non-snoring tf.enable_v2_behavior() loaded_model = tf2.saved_model.load(saved_model_path) pred = loaded_model(feature).numpy() # Take out snoring possibilities # ~1 is more likely snoring; ~0 is not snoring pred = pred[:, 1] reframed = frame_multi_seconds(pred) return np.round(reframed, 3)
def getDominantColor(): tfv1.disable_v2_behavior() json_data = request.get_json( force=True) #json 데이터를 받음. cropped 내부에 있을 파일명만 전송해주면 됨. filename = json_data['filename'] filename = os.path.join('../cropped/', filename + '.png') if 'isDemo' in json_data.keys(): isDemo = json_data['isDemo'] else: isDemo = False c = Classifier() image = cv2.imread(filename) result = c.predict(image, isDemo) tfv1.enable_v2_behavior() return jsonify(result)
import numpy as np sys.path.append(os.getcwd()) from tools.project_tools import get_project_name, get_project_paths from tools.model_info import save_graph_plot, save_graph_json config = ConfigProto() config.gpu_options.allow_growth = True session = InteractiveSession(config=config) epochs = 11 batch_size = 50 validation_split = 0.2 skip_epoch = 3 skip_from = 1 tfds.disable_progress_bar() tf.enable_v2_behavior() project_paths = get_project_paths(sys.argv[0], to_tmp=False) logs = project_paths["weights"] + "/reg_model_history_log.csv" csv_logger = CSVLogger(logs, append=True) (train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.mnist.load_data() train_images, test_images = train_images / 255.0, test_images / 255.0 """ ## Prepare the data """ # Model / data parameters num_classes = 10 input_shape = (28, 28, 1)
return os.environ["KMP_AFFINITY"] = "noverbose" os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' import warnings # warnings.simplefilter(action='ignore', category=FutureWarning) # warnings.simplefilter(action='ignore', category=DeprecationWarning) import tensorflow as tf tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR) _maybe_disable_warnings() import tensorflow.compat.v1 as _tfv1 _tfv1.enable_v2_behavior() import tensorflow as tf if int(tf.__version__[0]) < 2: warnings.warn( f"You are using TensorFlow version {tf.__version__}. This zfit version ({__version__}) works" f" with TF >= 2 and will likely break with an older version. Please consider upgrading as this" f" will raise an error in the future.") # EXPERIMENTAL_FUNCTIONS_RUN_EAGERLY = False # tf.config.experimental_run_functions_eagerly(EXPERIMENTAL_FUNCTIONS_RUN_EAGERLY) from . import z from . import z as ztf # legacy from .settings import ztypes
def __init__(self, source_vocab_size, target_vocab_size, buckets, size, num_layers, max_gradient_norm, batch_size, learning_rate, learning_rate_decay_factor, use_lstm=False, num_samples=512, forward_only=False): """Create the model. Args: source_vocab_size: size of the source vocabulary. target_vocab_size: size of the target vocabulary. buckets: a list of pairs (I, O), where I specifies maximum input length that will be processed in that bucket, and O specifies maximum output length. Training instances that have inputs longer than I or outputs longer than O will be pushed to the next bucket and padded accordingly. We assume that the list is sorted, e.g., [(2, 4), (8, 16)]. size: number of units in each layer of the model. num_layers: number of layers in the model. max_gradient_norm: gradients will be clipped to maximally this norm. batch_size: the size of the batches used during training; the model construction is independent of batch_size, so it can be changed after initialization if this is convenient, e.g., for decoding. learning_rate: learning rate to start with. learning_rate_decay_factor: decay learning rate by this much when needed. use_lstm: if true, we use LSTM cells instead of GRU cells. num_samples: number of samples for sampled softmax. forward_only: if set, we do not construct the backward pass in the model. """ self.source_vocab_size = source_vocab_size self.target_vocab_size = target_vocab_size self.buckets = buckets self.batch_size = batch_size import tensorflow.compat.v1 as tf tf.enable_v2_behavior() self.learning_rate = tf.Variable(float(learning_rate), trainable=False) self.learning_rate_decay_op = self.learning_rate.assign( self.learning_rate * learning_rate_decay_factor) self.global_step = tf.Variable(0, trainable=False) # If we use sampled softmax, we need an output projection. output_projection = None softmax_loss_function = None # Sampled softmax only makes sense if we sample less than vocabulary size. if num_samples > 0 and num_samples < self.target_vocab_size: w = tf.get_variable("proj_w", [size, self.target_vocab_size]) w_t = tf.transpose(w) b = tf.get_variable("proj_b", [self.target_vocab_size]) output_projection = (w, b) ''' def sampled_loss(inputs, labels): labels = tf.reshape(labels, [-1, 1]) return tf.nn.sampled_softmax_loss(w_t, b, inputs, labels, num_samples, self.target_vocab_size) ''' def sampled_loss(labels, logits): labels = tf.reshape(labels, [-1, 1]) return tf.nn.sampled_softmax_loss(tf.transpose(w), b, labels, logits, num_samples, self.target_vocab_size) softmax_loss_function = sampled_loss # Create the internal multi-layer cell for our RNN. single_cell = tf.nn.rnn_cell.GRUCell(size) if use_lstm: single_cell = tf.nn.rnn_cell.BasicLSTMCell(size) cell = single_cell cell = tf.nn.rnn_cell.DropoutWrapper(cell, output_keep_prob=0.5) if num_layers > 1: cell = tf.nn.rnn_cell.MultiRNNCell([single_cell] * num_layers) # The seq2seq function: we use embedding for the input and attention. def seq2seq_f(encoder_inputs, decoder_inputs, do_decode): #return tf.nn.seq2seq.embedding_attention_seq2seq( return tf.contrib.legacy_seq2seq.embedding_attention_seq2seq( encoder_inputs, decoder_inputs, cell, num_encoder_symbols=source_vocab_size, num_decoder_symbols=target_vocab_size, embedding_size=size, output_projection=output_projection, feed_previous=do_decode) # Feeds for inputs. self.encoder_inputs = [] self.decoder_inputs = [] self.target_weights = [] for i in xrange(buckets[-1][0]): # Last bucket is the biggest one. self.encoder_inputs.append( tf.placeholder(tf.int32, shape=[None], name="encoder{0}".format(i))) for i in xrange(buckets[-1][1] + 1): self.decoder_inputs.append( tf.placeholder(tf.int32, shape=[None], name="decoder{0}".format(i))) self.target_weights.append( tf.placeholder(tf.float32, shape=[None], name="weight{0}".format(i))) # Our targets are decoder inputs shifted by one. targets = [ self.decoder_inputs[i + 1] for i in xrange(len(self.decoder_inputs) - 1) ] # Training outputs and losses. if forward_only: self.outputs, self.losses = tf.nn.seq2seq.model_with_buckets( self.encoder_inputs, self.decoder_inputs, targets, self.target_weights, buckets, lambda x, y: seq2seq_f(x, y, True), softmax_loss_function=softmax_loss_function) # If we use output projection, we need to project outputs for decoding. if output_projection is not None: for b in xrange(len(buckets)): self.outputs[b] = [ tf.matmul(output, output_projection[0]) + output_projection[1] for output in self.outputs[b] ] else: import tensorflow as tf #tf.disable_v2_behavior() #self.outputs, self.losses = tf.nn.seq2seq.model_with_buckets( self.outputs, self.losses = tf.contrib.legacy_seq2seq.model_with_buckets( self.encoder_inputs, self.decoder_inputs, targets, self.target_weights, buckets, lambda x, y: seq2seq_f(x, y, False), softmax_loss_function=softmax_loss_function) # Gradients and SGD update operation for training the model. params = tf.trainable_variables() if not forward_only: self.gradient_norms = [] self.updates = [] opt = tf.train.AdamOptimizer() for b in xrange(len(buckets)): gradients = tf.gradients(self.losses[b], params) clipped_gradients, norm = tf.clip_by_global_norm( gradients, max_gradient_norm) self.gradient_norms.append(norm) self.updates.append( opt.apply_gradients(zip(clipped_gradients, params), global_step=self.global_step)) self.saver = tf.train.Saver(tf.global_variables())
def main(argv): del argv # Unused if hasattr(tf, 'enable_v2_behavior'): tf.enable_v2_behavior() tf.test.main()
def run_experiment( model_dir, data_dir=None, xid=None, batch_size_per_device=128, eval_frequency=500, checkpoint_frequency=10000, save_checkpoints=True, restore_checkpoint=True, num_eval_steps=None, epochs=None, max_train_steps=1000000, # 1 million max_train_length=512, train_summary_frequency=100, max_eval_length=None, model_cls=models.FlaxLM): """Run experiment. Args: model_dir: Directory to save checkpoints and metrics to. data_dir: Directory to load data. xid: Optional experiment id. batch_size_per_device: Batch size per device. eval_frequency: Steps per eval. checkpoint_frequency: How often to checkpoint. If None, only checkpoint once at end of run. save_checkpoints: If True, checkpoints model according to checkpoint_frequency restore_checkpoint: If True, will restore checkpoint from directory. Useful for robustness to preemption. num_eval_steps: Number of eval steps to take on eval dataset. epochs: Number of train epochs. max_train_steps: Stop training after N steps. max_train_length: Crop training sequences to this length. train_summary_frequency: Frequency to write train metrics. max_eval_length: Maximum eval length. Defaults to max_train_length. model_cls: Model class to use. Returns: FlaxLM resulting from running training. """ if xid is not None: model_dir = os.path.join(model_dir, '%s_l%s' % (str(xid), max_train_length)) tf.enable_v2_behavior() if jax.host_id() == 0: summary_writer = tf_summary.create_file_writer(os.path.join( model_dir, 'metrics'), max_queue=1, flush_millis=1000) train_summary_writer = logging_lib.ScalarSummary(step=None, scope='train/', enable_tf=True, verbose=0) eval_summary_writer = logging_lib.ScalarSummary(step=None, scope='eval/', enable_tf=True, verbose=0) batch_size = batch_size_per_device * jax.local_device_count() max_eval_length = max_eval_length or max_train_length train_files, test_files = data.get_train_valid_files(directory=data_dir) train_ds, eval_ds = data.load_dataset(train_files=train_files, test_files=test_files, batch_size=batch_size, max_train_length=max_train_length, max_eval_length=max_eval_length, shuffle_buffer=16384) with contextlib.ExitStack() as stack: # pylint: disable=using-constant-test if jax.host_id() == 0: # Only need metric writer context manager on host 0. stack.enter_context(summary_writer.as_default()) model = model_cls(domain=data.protein_domain, batch_size=batch_size) if restore_checkpoint: try: model.load_checkpoint(model_dir) except ValueError: # No checkpoint to load -> raises ValueError. pass start_step = model.train_step train_ds = train_ds.repeat(epochs) train_iter = iter(train_ds) train_metrics = [] tick = time.time() if jax.host_id() == 0: _write_gin_configs(os.path.join(model_dir, 'config.gin')) num_evals = 0 for step, batch in zip(range(start_step, max_train_steps), train_iter): batch = jax.tree_map(lambda x: x._numpy(), batch) # pylint: disable=protected-access metrics = model.fit_batch(batch) train_metrics.append(metrics) if jax.host_id() == 0 and ( (save_checkpoints and checkpoint_frequency and step % checkpoint_frequency == 0 and step > 0) or step == max_train_steps - 1): model.save_checkpoint(model_dir) if (step + 1) % train_summary_frequency == 0: summary = evaluation.combine_metrics(train_metrics) logging.info('train in step: %d, loss: %.4f', step, summary['loss']) if jax.host_id() == 0: tock = time.time() steps_per_sec = eval_frequency / (tock - tick) tick = tock train_summary_writer('steps per second', steps_per_sec, step) for key, val in summary.items(): if jnp.isnan(val): raise ValueError(f'NaN in {key} at step {step}.') train_summary_writer(key, val, step) # reset metric accumulation for next evaluation cycle. train_metrics = [] if eval_frequency and (step + 1) % eval_frequency == 0: eval_summary = evaluation.evaluate( model=model, eval_ds=eval_ds, num_eval_steps=num_eval_steps) logging.info('eval in step: %d, loss: %.4f', step, eval_summary['loss']) if jax.host_id() == 0: for key, val in eval_summary.items(): eval_summary_writer(key, val, step) tf_summary.flush() summary_writer.flush() if num_evals == 0: # Write out config on first eval. _write_gin_configs( os.path.join(model_dir, 'config_after_eval.gin')) num_evals += 1 if jax.host_id() == 0: tf_summary.flush() summary_writer.close() _write_gin_configs(os.path.join(model_dir, 'config_end.gin')) return model
def main(_): if FLAGS.is_tempscale: tf.enable_v2_behavior() params = { 'num_epochs': FLAGS.num_epochs, 'fix_len': FLAGS.fix_len, 'batch_size': FLAGS.batch_size, 'n_class': FLAGS.n_class, 'emb_size': FLAGS.emb_size, 'vocab_size': FLAGS.vocab_size, 'hidden_lstm_size': FLAGS.hidden_lstm_size, 'dropout_rate': FLAGS.dropout_rate, 'dropout_rate_lstm': FLAGS.dropout_rate_lstm, 'learning_rate': FLAGS.learning_rate, 'reg_weight': FLAGS.reg_weight, 'tr_out_dir': FLAGS.tr_out_dir, 'data_pkl_file': FLAGS.data_pkl_file, 'master': FLAGS.master, 'clip_norm': FLAGS.clip_norm, 'random_seed': FLAGS.random_seed, 'variational': FLAGS.variational, 'n_class_in': None, 'n_train': None, } # load in-dist. and skewed in-dist. datasets data = classifier.load_np_dataset(params['data_pkl_file']) # load OOD dataset n_ood = 5600 test_lm1b_x_pad, _ = load_ood_dataset(n_ood, params['fix_len'], data.vocab, params['vocab_size']) # list of ckpt dir model_dir = os.path.join(FLAGS.model_dir, FLAGS.method) ckpt_dirs = tf.io.gfile.listdir(model_dir) # how many replicates for ensemble if FLAGS.is_ensemble: assert len(ckpt_dirs) > 1 n_ensemble = len(ckpt_dirs) if n_ensemble == 0: logging.fatal('no model ckpt') else: n_ensemble = 1 pred = {} # dict for final prediction score # dict for saving pred from different models pred_accum = {'in': [], 'skew': [], 'ood': []} for i in range(n_ensemble): ckpt_dir = os.path.join(model_dir, ckpt_dirs[i], 'model') if not tf.io.gfile.isdir(ckpt_dir): continue print('ckpt_dir={}'.format(ckpt_dir)) # load params with tf.gfile.GFile(os.path.join(ckpt_dir, 'params.json'), mode='rb') as f: params_json = yaml.safe_load(f) params.update(params_json) params['master'] = '' print('params after load={}'.format(params)) tf.reset_default_graph() # create model model = classifier.rnn_model( params, training_dr_lstm=params['dropout_rate_lstm'] != 0.0, training_dr_ll=params['dropout_rate'] != 0.0) # load model model.load_weights(ckpt_dir + '/model.ckpt') # predict if FLAGS.method in ['ll-svi', 'dropout', 'll-dropout']: # need to run multiple times and get mean prediction assert FLAGS.n_pred_sample > 1 else: FLAGS.n_pred_sample = 1 pred_k = {'in': [], 'skew': [], 'ood': []} for _ in range(FLAGS.n_pred_sample): pred_tr_in = model.predict(data.in_sample_examples) acc_tr_in = np.mean( data.in_sample_labels == np.argmax(pred_tr_in, axis=1)) pred_test_in = model.predict(data.test_in_sample_examples) acc_test_in = np.mean( data.test_in_sample_labels == np.argmax(pred_test_in, axis=1)) print('in-dist. acc_tr={}, acc_test={}'.format( acc_tr_in, acc_test_in)) pred_test_skew = model.predict(data.test_oos_examples) pred_test_ood = model.predict(test_lm1b_x_pad) if FLAGS.is_tempscale: # temperature scaling # logits for temp scaling last_layer_model = models.Model( inputs=model.input, outputs=model.get_layer('last_layer').output) logits = last_layer_model.predict(data.dev_in_sample_examples) opt_temp = calibration_lib.find_scaling_temperature( data.dev_in_sample_labels, logits, temp_range=(1e-5, 1e5)) pred_test_in = calibration_lib.apply_temperature_scaling( opt_temp, pred_test_in) pred_test_skew = calibration_lib.apply_temperature_scaling( opt_temp, pred_test_skew) pred_test_ood = calibration_lib.apply_temperature_scaling( opt_temp, pred_test_ood) # save in a list pred_k['in'].append(pred_test_in) pred_k['skew'].append(pred_test_skew) pred_k['ood'].append(pred_test_ood) pred_k_in_mean = np.mean(np.stack(pred_k['in']), axis=0) pred_k_skew_mean = np.mean(np.stack(pred_k['skew']), axis=0) pred_k_ood_mean = np.mean(np.stack(pred_k['ood']), axis=0) pred_accum['in'].append(pred_k_in_mean) pred_accum['skew'].append(pred_k_skew_mean) pred_accum['ood'].append(pred_k_ood_mean) # if ensemble, then take the mean pred['in'] = np.mean(np.stack(pred_accum['in']), axis=0) pred['skew'] = np.mean(np.stack(pred_accum['skew']), axis=0) pred['ood'] = np.mean(np.stack(pred_accum['ood']), axis=0) # prediction accuracy for in-dist. pred['in_true_labels'] = data.test_in_sample_labels acc = np.mean(data.test_in_sample_labels == np.argmax(pred['in'], axis=1)) print('== (optionally ensemble) acc={} =='.format(acc)) print('== eval in and skew using max(Py|x) ==') neg = list(np.max(pred['in'], axis=1)) pos = list(np.max(pred['skew'], axis=1)) print('auc={}'.format(compute_auc(neg, pos, pos_label=0))) print('== eval in and ood using max(Py|x) ==') neg = list(np.max(pred['in'], axis=1)) pos = list(np.max(pred['ood'], axis=1)) print('auc={}'.format(compute_auc(neg, pos, pos_label=0))) # save the predictions pred_file_name = 'pred_nensemb{}_npred{}_tempscale{}.pkl'.format( len(pred_accum['in']), FLAGS.n_pred_sample, FLAGS.is_tempscale) with tf.gfile.Open(os.path.join(model_dir, pred_file_name), 'wb') as f: pickle.dump(pred, f, protocol=2)