def main(_):
logging.set_verbosity(logging.INFO)
tf.enable_v2_behavior()
tf.enable_resource_variables()
tf.enable_control_flow_v2()
logging.info('Executing eagerly: %s', tf.executing_eagerly())
logging.info('parsing config files: %s', FLAGS.gin_file)
gin.parse_config_files_and_bindings(
FLAGS.gin_file, FLAGS.gin_bindings, skip_unknown=True)
trainer.train(root_dir, eval_metrics_callback=metrics_callback)
def predict_with_saved_model(feature):
# [0, 1] is snoring
# [1, 0] is non-snoring
tf.enable_v2_behavior()
loaded_model = tf2.saved_model.load(saved_model_path)
pred = loaded_model(feature).numpy()
# Take out snoring possibilities
# ~1 is more likely snoring; ~0 is not snoring
pred = pred[:, 1]
reframed = frame_multi_seconds(pred)
return np.round(reframed, 3)
def getDominantColor():
tfv1.disable_v2_behavior()
json_data = request.get_json(
force=True) #json 데이터를 받음. cropped 내부에 있을 파일명만 전송해주면 됨.
filename = json_data['filename']
filename = os.path.join('../cropped/', filename + '.png')
if 'isDemo' in json_data.keys():
isDemo = json_data['isDemo']
else:
isDemo = False
c = Classifier()
image = cv2.imread(filename)
result = c.predict(image, isDemo)
tfv1.enable_v2_behavior()
return jsonify(result)
import numpy as np
sys.path.append(os.getcwd())
from tools.project_tools import get_project_name, get_project_paths
from tools.model_info import save_graph_plot, save_graph_json
config = ConfigProto()
config.gpu_options.allow_growth = True
session = InteractiveSession(config=config)
epochs = 11
batch_size = 50
validation_split = 0.2
skip_epoch = 3
skip_from = 1
tfds.disable_progress_bar()
tf.enable_v2_behavior()
project_paths = get_project_paths(sys.argv[0], to_tmp=False)
logs = project_paths["weights"] + "/reg_model_history_log.csv"
csv_logger = CSVLogger(logs, append=True)
(train_images,
train_labels), (test_images,
test_labels) = tf.keras.datasets.mnist.load_data()
train_images, test_images = train_images / 255.0, test_images / 255.0
"""
## Prepare the data
"""
# Model / data parameters
num_classes = 10
input_shape = (28, 28, 1)
return
os.environ["KMP_AFFINITY"] = "noverbose"
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
import warnings
# warnings.simplefilter(action='ignore', category=FutureWarning)
# warnings.simplefilter(action='ignore', category=DeprecationWarning)
import tensorflow as tf
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
_maybe_disable_warnings()
import tensorflow.compat.v1 as _tfv1
_tfv1.enable_v2_behavior()
import tensorflow as tf
if int(tf.__version__[0]) < 2:
warnings.warn(
f"You are using TensorFlow version {tf.__version__}. This zfit version ({__version__}) works"
f" with TF >= 2 and will likely break with an older version. Please consider upgrading as this"
f" will raise an error in the future.")
# EXPERIMENTAL_FUNCTIONS_RUN_EAGERLY = False
# tf.config.experimental_run_functions_eagerly(EXPERIMENTAL_FUNCTIONS_RUN_EAGERLY)
from . import z
from . import z as ztf # legacy
from .settings import ztypes
def __init__(self,
source_vocab_size,
target_vocab_size,
buckets,
size,
num_layers,
max_gradient_norm,
batch_size,
learning_rate,
learning_rate_decay_factor,
use_lstm=False,
num_samples=512,
forward_only=False):
"""Create the model.
Args:
source_vocab_size: size of the source vocabulary.
target_vocab_size: size of the target vocabulary.
buckets: a list of pairs (I, O), where I specifies maximum input length
that will be processed in that bucket, and O specifies maximum output
length. Training instances that have inputs longer than I or outputs
longer than O will be pushed to the next bucket and padded accordingly.
We assume that the list is sorted, e.g., [(2, 4), (8, 16)].
size: number of units in each layer of the model.
num_layers: number of layers in the model.
max_gradient_norm: gradients will be clipped to maximally this norm.
batch_size: the size of the batches used during training;
the model construction is independent of batch_size, so it can be
changed after initialization if this is convenient, e.g., for decoding.
learning_rate: learning rate to start with.
learning_rate_decay_factor: decay learning rate by this much when needed.
use_lstm: if true, we use LSTM cells instead of GRU cells.
num_samples: number of samples for sampled softmax.
forward_only: if set, we do not construct the backward pass in the model.
"""
self.source_vocab_size = source_vocab_size
self.target_vocab_size = target_vocab_size
self.buckets = buckets
self.batch_size = batch_size
import tensorflow.compat.v1 as tf
tf.enable_v2_behavior()
self.learning_rate = tf.Variable(float(learning_rate), trainable=False)
self.learning_rate_decay_op = self.learning_rate.assign(
self.learning_rate * learning_rate_decay_factor)
self.global_step = tf.Variable(0, trainable=False)
# If we use sampled softmax, we need an output projection.
output_projection = None
softmax_loss_function = None
# Sampled softmax only makes sense if we sample less than vocabulary size.
if num_samples > 0 and num_samples < self.target_vocab_size:
w = tf.get_variable("proj_w", [size, self.target_vocab_size])
w_t = tf.transpose(w)
b = tf.get_variable("proj_b", [self.target_vocab_size])
output_projection = (w, b)
'''
def sampled_loss(inputs, labels):
labels = tf.reshape(labels, [-1, 1])
return tf.nn.sampled_softmax_loss(w_t, b, inputs, labels, num_samples,
self.target_vocab_size)
'''
def sampled_loss(labels, logits):
labels = tf.reshape(labels, [-1, 1])
return tf.nn.sampled_softmax_loss(tf.transpose(w), b, labels,
logits, num_samples,
self.target_vocab_size)
softmax_loss_function = sampled_loss
# Create the internal multi-layer cell for our RNN.
single_cell = tf.nn.rnn_cell.GRUCell(size)
if use_lstm:
single_cell = tf.nn.rnn_cell.BasicLSTMCell(size)
cell = single_cell
cell = tf.nn.rnn_cell.DropoutWrapper(cell, output_keep_prob=0.5)
if num_layers > 1:
cell = tf.nn.rnn_cell.MultiRNNCell([single_cell] * num_layers)
# The seq2seq function: we use embedding for the input and attention.
def seq2seq_f(encoder_inputs, decoder_inputs, do_decode):
#return tf.nn.seq2seq.embedding_attention_seq2seq(
return tf.contrib.legacy_seq2seq.embedding_attention_seq2seq(
encoder_inputs,
decoder_inputs,
cell,
num_encoder_symbols=source_vocab_size,
num_decoder_symbols=target_vocab_size,
embedding_size=size,
output_projection=output_projection,
feed_previous=do_decode)
# Feeds for inputs.
self.encoder_inputs = []
self.decoder_inputs = []
self.target_weights = []
for i in xrange(buckets[-1][0]): # Last bucket is the biggest one.
self.encoder_inputs.append(
tf.placeholder(tf.int32,
shape=[None],
name="encoder{0}".format(i)))
for i in xrange(buckets[-1][1] + 1):
self.decoder_inputs.append(
tf.placeholder(tf.int32,
shape=[None],
name="decoder{0}".format(i)))
self.target_weights.append(
tf.placeholder(tf.float32,
shape=[None],
name="weight{0}".format(i)))
# Our targets are decoder inputs shifted by one.
targets = [
self.decoder_inputs[i + 1]
for i in xrange(len(self.decoder_inputs) - 1)
]
# Training outputs and losses.
if forward_only:
self.outputs, self.losses = tf.nn.seq2seq.model_with_buckets(
self.encoder_inputs,
self.decoder_inputs,
targets,
self.target_weights,
buckets,
lambda x, y: seq2seq_f(x, y, True),
softmax_loss_function=softmax_loss_function)
# If we use output projection, we need to project outputs for decoding.
if output_projection is not None:
for b in xrange(len(buckets)):
self.outputs[b] = [
tf.matmul(output, output_projection[0]) +
output_projection[1] for output in self.outputs[b]
]
else:
import tensorflow as tf
#tf.disable_v2_behavior()
#self.outputs, self.losses = tf.nn.seq2seq.model_with_buckets(
self.outputs, self.losses = tf.contrib.legacy_seq2seq.model_with_buckets(
self.encoder_inputs,
self.decoder_inputs,
targets,
self.target_weights,
buckets,
lambda x, y: seq2seq_f(x, y, False),
softmax_loss_function=softmax_loss_function)
# Gradients and SGD update operation for training the model.
params = tf.trainable_variables()
if not forward_only:
self.gradient_norms = []
self.updates = []
opt = tf.train.AdamOptimizer()
for b in xrange(len(buckets)):
gradients = tf.gradients(self.losses[b], params)
clipped_gradients, norm = tf.clip_by_global_norm(
gradients, max_gradient_norm)
self.gradient_norms.append(norm)
self.updates.append(
opt.apply_gradients(zip(clipped_gradients, params),
global_step=self.global_step))
self.saver = tf.train.Saver(tf.global_variables())
def main(argv):
del argv # Unused
if hasattr(tf, 'enable_v2_behavior'):
tf.enable_v2_behavior()
tf.test.main()
def run_experiment(
model_dir,
data_dir=None,
xid=None,
batch_size_per_device=128,
eval_frequency=500,
checkpoint_frequency=10000,
save_checkpoints=True,
restore_checkpoint=True,
num_eval_steps=None,
epochs=None,
max_train_steps=1000000, # 1 million
max_train_length=512,
train_summary_frequency=100,
max_eval_length=None,
model_cls=models.FlaxLM):
"""Run experiment.
Args:
model_dir: Directory to save checkpoints and metrics to.
data_dir: Directory to load data.
xid: Optional experiment id.
batch_size_per_device: Batch size per device.
eval_frequency: Steps per eval.
checkpoint_frequency: How often to checkpoint. If None, only checkpoint once
at end of run.
save_checkpoints: If True, checkpoints model according to
checkpoint_frequency
restore_checkpoint: If True, will restore checkpoint from directory. Useful
for robustness to preemption.
num_eval_steps: Number of eval steps to take on eval dataset.
epochs: Number of train epochs.
max_train_steps: Stop training after N steps.
max_train_length: Crop training sequences to this length.
train_summary_frequency: Frequency to write train metrics.
max_eval_length: Maximum eval length. Defaults to max_train_length.
model_cls: Model class to use.
Returns:
FlaxLM resulting from running training.
"""
if xid is not None:
model_dir = os.path.join(model_dir,
'%s_l%s' % (str(xid), max_train_length))
tf.enable_v2_behavior()
if jax.host_id() == 0:
summary_writer = tf_summary.create_file_writer(os.path.join(
model_dir, 'metrics'),
max_queue=1,
flush_millis=1000)
train_summary_writer = logging_lib.ScalarSummary(step=None,
scope='train/',
enable_tf=True,
verbose=0)
eval_summary_writer = logging_lib.ScalarSummary(step=None,
scope='eval/',
enable_tf=True,
verbose=0)
batch_size = batch_size_per_device * jax.local_device_count()
max_eval_length = max_eval_length or max_train_length
train_files, test_files = data.get_train_valid_files(directory=data_dir)
train_ds, eval_ds = data.load_dataset(train_files=train_files,
test_files=test_files,
batch_size=batch_size,
max_train_length=max_train_length,
max_eval_length=max_eval_length,
shuffle_buffer=16384)
with contextlib.ExitStack() as stack: # pylint: disable=using-constant-test
if jax.host_id() == 0:
# Only need metric writer context manager on host 0.
stack.enter_context(summary_writer.as_default())
model = model_cls(domain=data.protein_domain, batch_size=batch_size)
if restore_checkpoint:
try:
model.load_checkpoint(model_dir)
except ValueError:
# No checkpoint to load -> raises ValueError.
pass
start_step = model.train_step
train_ds = train_ds.repeat(epochs)
train_iter = iter(train_ds)
train_metrics = []
tick = time.time()
if jax.host_id() == 0:
_write_gin_configs(os.path.join(model_dir, 'config.gin'))
num_evals = 0
for step, batch in zip(range(start_step, max_train_steps), train_iter):
batch = jax.tree_map(lambda x: x._numpy(), batch) # pylint: disable=protected-access
metrics = model.fit_batch(batch)
train_metrics.append(metrics)
if jax.host_id() == 0 and (
(save_checkpoints and checkpoint_frequency
and step % checkpoint_frequency == 0 and step > 0)
or step == max_train_steps - 1):
model.save_checkpoint(model_dir)
if (step + 1) % train_summary_frequency == 0:
summary = evaluation.combine_metrics(train_metrics)
logging.info('train in step: %d, loss: %.4f', step,
summary['loss'])
if jax.host_id() == 0:
tock = time.time()
steps_per_sec = eval_frequency / (tock - tick)
tick = tock
train_summary_writer('steps per second', steps_per_sec,
step)
for key, val in summary.items():
if jnp.isnan(val):
raise ValueError(f'NaN in {key} at step {step}.')
train_summary_writer(key, val, step)
# reset metric accumulation for next evaluation cycle.
train_metrics = []
if eval_frequency and (step + 1) % eval_frequency == 0:
eval_summary = evaluation.evaluate(
model=model,
eval_ds=eval_ds,
num_eval_steps=num_eval_steps)
logging.info('eval in step: %d, loss: %.4f', step,
eval_summary['loss'])
if jax.host_id() == 0:
for key, val in eval_summary.items():
eval_summary_writer(key, val, step)
tf_summary.flush()
summary_writer.flush()
if num_evals == 0:
# Write out config on first eval.
_write_gin_configs(
os.path.join(model_dir, 'config_after_eval.gin'))
num_evals += 1
if jax.host_id() == 0:
tf_summary.flush()
summary_writer.close()
_write_gin_configs(os.path.join(model_dir, 'config_end.gin'))
return model
def main(_):
if FLAGS.is_tempscale:
tf.enable_v2_behavior()
params = {
'num_epochs': FLAGS.num_epochs,
'fix_len': FLAGS.fix_len,
'batch_size': FLAGS.batch_size,
'n_class': FLAGS.n_class,
'emb_size': FLAGS.emb_size,
'vocab_size': FLAGS.vocab_size,
'hidden_lstm_size': FLAGS.hidden_lstm_size,
'dropout_rate': FLAGS.dropout_rate,
'dropout_rate_lstm': FLAGS.dropout_rate_lstm,
'learning_rate': FLAGS.learning_rate,
'reg_weight': FLAGS.reg_weight,
'tr_out_dir': FLAGS.tr_out_dir,
'data_pkl_file': FLAGS.data_pkl_file,
'master': FLAGS.master,
'clip_norm': FLAGS.clip_norm,
'random_seed': FLAGS.random_seed,
'variational': FLAGS.variational,
'n_class_in': None,
'n_train': None,
}
# load in-dist. and skewed in-dist. datasets
data = classifier.load_np_dataset(params['data_pkl_file'])
# load OOD dataset
n_ood = 5600
test_lm1b_x_pad, _ = load_ood_dataset(n_ood, params['fix_len'], data.vocab,
params['vocab_size'])
# list of ckpt dir
model_dir = os.path.join(FLAGS.model_dir, FLAGS.method)
ckpt_dirs = tf.io.gfile.listdir(model_dir)
# how many replicates for ensemble
if FLAGS.is_ensemble:
assert len(ckpt_dirs) > 1
n_ensemble = len(ckpt_dirs)
if n_ensemble == 0:
logging.fatal('no model ckpt')
else:
n_ensemble = 1
pred = {} # dict for final prediction score
# dict for saving pred from different models
pred_accum = {'in': [], 'skew': [], 'ood': []}
for i in range(n_ensemble):
ckpt_dir = os.path.join(model_dir, ckpt_dirs[i], 'model')
if not tf.io.gfile.isdir(ckpt_dir):
continue
print('ckpt_dir={}'.format(ckpt_dir))
# load params
with tf.gfile.GFile(os.path.join(ckpt_dir, 'params.json'),
mode='rb') as f:
params_json = yaml.safe_load(f)
params.update(params_json)
params['master'] = ''
print('params after load={}'.format(params))
tf.reset_default_graph()
# create model
model = classifier.rnn_model(
params,
training_dr_lstm=params['dropout_rate_lstm'] != 0.0,
training_dr_ll=params['dropout_rate'] != 0.0)
# load model
model.load_weights(ckpt_dir + '/model.ckpt')
# predict
if FLAGS.method in ['ll-svi', 'dropout', 'll-dropout']:
# need to run multiple times and get mean prediction
assert FLAGS.n_pred_sample > 1
else:
FLAGS.n_pred_sample = 1
pred_k = {'in': [], 'skew': [], 'ood': []}
for _ in range(FLAGS.n_pred_sample):
pred_tr_in = model.predict(data.in_sample_examples)
acc_tr_in = np.mean(
data.in_sample_labels == np.argmax(pred_tr_in, axis=1))
pred_test_in = model.predict(data.test_in_sample_examples)
acc_test_in = np.mean(
data.test_in_sample_labels == np.argmax(pred_test_in, axis=1))
print('in-dist. acc_tr={}, acc_test={}'.format(
acc_tr_in, acc_test_in))
pred_test_skew = model.predict(data.test_oos_examples)
pred_test_ood = model.predict(test_lm1b_x_pad)
if FLAGS.is_tempscale:
# temperature scaling
# logits for temp scaling
last_layer_model = models.Model(
inputs=model.input,
outputs=model.get_layer('last_layer').output)
logits = last_layer_model.predict(data.dev_in_sample_examples)
opt_temp = calibration_lib.find_scaling_temperature(
data.dev_in_sample_labels, logits, temp_range=(1e-5, 1e5))
pred_test_in = calibration_lib.apply_temperature_scaling(
opt_temp, pred_test_in)
pred_test_skew = calibration_lib.apply_temperature_scaling(
opt_temp, pred_test_skew)
pred_test_ood = calibration_lib.apply_temperature_scaling(
opt_temp, pred_test_ood)
# save in a list
pred_k['in'].append(pred_test_in)
pred_k['skew'].append(pred_test_skew)
pred_k['ood'].append(pred_test_ood)
pred_k_in_mean = np.mean(np.stack(pred_k['in']), axis=0)
pred_k_skew_mean = np.mean(np.stack(pred_k['skew']), axis=0)
pred_k_ood_mean = np.mean(np.stack(pred_k['ood']), axis=0)
pred_accum['in'].append(pred_k_in_mean)
pred_accum['skew'].append(pred_k_skew_mean)
pred_accum['ood'].append(pred_k_ood_mean)
# if ensemble, then take the mean
pred['in'] = np.mean(np.stack(pred_accum['in']), axis=0)
pred['skew'] = np.mean(np.stack(pred_accum['skew']), axis=0)
pred['ood'] = np.mean(np.stack(pred_accum['ood']), axis=0)
# prediction accuracy for in-dist.
pred['in_true_labels'] = data.test_in_sample_labels
acc = np.mean(data.test_in_sample_labels == np.argmax(pred['in'], axis=1))
print('== (optionally ensemble) acc={} =='.format(acc))
print('== eval in and skew using max(Py|x) ==')
neg = list(np.max(pred['in'], axis=1))
pos = list(np.max(pred['skew'], axis=1))
print('auc={}'.format(compute_auc(neg, pos, pos_label=0)))
print('== eval in and ood using max(Py|x) ==')
neg = list(np.max(pred['in'], axis=1))
pos = list(np.max(pred['ood'], axis=1))
print('auc={}'.format(compute_auc(neg, pos, pos_label=0)))
# save the predictions
pred_file_name = 'pred_nensemb{}_npred{}_tempscale{}.pkl'.format(
len(pred_accum['in']), FLAGS.n_pred_sample, FLAGS.is_tempscale)
with tf.gfile.Open(os.path.join(model_dir, pred_file_name), 'wb') as f:
pickle.dump(pred, f, protocol=2)