def load_estimator(self):
"""
Returns
-------
estimator
A tf.estimator.DNNClassifier
"""
# Feature columns describe how to use the input.
my_feature_columns = []
for key in self.train_x.keys():
my_feature_columns.append(
tf.feature_column.numeric_column(key=key))
run_config = RunConfig()
run_config = run_config.replace(model_dir=self.model_path)
return tf.estimator.DNNClassifier(
feature_columns=my_feature_columns,
# Two hidden layers of 10 nodes each.
hidden_units=[10, 10],
# The model must choose between 3 classes.
n_classes=3,
# Use runconfig to load model,
config=run_config,
model_dir=self.model_path)
def main():
tf.logging.set_verbosity(tf.logging.DEBUG)
parsed_args = get_parser().parse_args()
session_config = tf.ConfigProto(allow_soft_placement=True)
session_config.gpu_options.allow_growth = True
run_config = RunConfig(session_config=session_config)
run_config = run_config.replace(model_dir=get_model_dir(parsed_args))
params = HParams(learning_rate=parsed_args.lr,
train_steps=parsed_args.train_steps,
steps_per_eval=parsed_args.steps_per_eval,
batch_size=parsed_args.batch_size,
vgg_model_path=parsed_args.vgg_model_path,
selector=parsed_args.selector,
dropout=parsed_args.dropout,
ctx2out=parsed_args.ctx2out,
prev2out=parsed_args.prev2out,
dataset=parsed_args.dataset,
eval_steps=parsed_args.eval_steps,
hard_attention=parsed_args.hard_attention,
use_sampler=parsed_args.use_sampler,
bin_size=14)
learn_runner.run(experiment_fn=experiment_fn_inner,
run_config=run_config,
schedule="continuous_train_and_eval",
hparams=params)
def main(_):
mnist = input_data.read_data_sets("/tmp/data")
X_train = mnist.train.images
X_test = mnist.test.images
Y_train = mnist.train.labels.astype("int")
Y_test = mnist.test.labels.astype("int")
config = RunConfig(tf_random_seed=42, save_checkpoints_secs=10)
feature_cols = tf.contrib.learn.infer_real_valued_columns_from_input(
X_train)
validation_monitor = monitors.ValidationMonitor(x=X_test,
y=Y_test,
every_n_steps=100)
dnn_clf = DNNClassifier(
hidden_units=[300, 100],
n_classes=10,
feature_columns=feature_cols,
config=config,
model_dir="/home/mtb/Projects/machine_learning/tensorflow/mnist")
dnn_clf.fit(X_train,
Y_train,
batch_size=50,
steps=4000,
monitors=[validation_monitor])
accuracy_score = dnn_clf.evaluate(x=X_test, y=Y_test)["accuracy"]
print(' accuracy_score: {0} '.format(accuracy_score))
def get_unthreaded_estimator(model_path, feature_columns):
return tf.estimator.DNNClassifier(
feature_columns=feature_columns,
hidden_units=[10, 10],
n_classes=3,
config=RunConfig().replace(model_dir=model_path),
model_dir=model_path)
def generate_results(model_dir, results_path):
os.makedirs(os.path.dirname(results_path), exist_ok=True)
vocab = np.load('output/processed-annotations/vocab.npy')
run_config = RunConfig(model_dir=model_dir)
hparams = get_hparams(model_dir=model_dir, create=False)
print(hparams)
estimator = Estimator(model_fn=model_fn, config=run_config, params=hparams)
val_path = tf.flags.FLAGS.batch_path
splits = tf.flags.FLAGS.batch_splits
batch_size = tf.flags.FLAGS.batch_size
hook = FeedFnHook(path_fmt=val_path,
splits=splits,
batch_size=batch_size,
predict=True,
single_pass=True)
results = []
it = tqdm(desc='Generating results')
for prediction in estimator.predict(input_fn=predict_input_fn,
hooks=[hook]):
caption = calc_caption(prediction=prediction, vocab=vocab)
results.append({
'image_id': np.asscalar(prediction['image_ids']),
'caption': caption
})
it.update(1)
with open(results_path, 'w') as f:
json.dump(results, f)
def start_prediction(output_directory,
data_directory,
dataset_name,
model_dir,
network_name,
batch_size,
batch_threads,
num_classes=None):
dataset_factory = DatasetFactory(dataset_name=dataset_name,
data_directory=data_directory,
augment=False)
if num_classes is None:
num_classes = dataset_factory.get_dataset('train').num_classes()
run_config = RunConfig(keep_checkpoint_max=10, save_checkpoints_steps=None)
# Instantiate Estimator
estimator = tf.estimator.Estimator(model_fn=get_model_function(
model_dir, network_name, num_classes),
model_dir=model_dir,
config=run_config,
params={})
image_size = nets_factory.get_input_size(network_name)
run_prediction_and_evaluation(output_directory, batch_size, batch_threads,
dataset_factory, estimator, image_size)
def main(_):
mnist = load_dataset('mnist')
train_data = mnist.train.images
train_labels = mnist.train.labels
test_data = mnist.test.images
test_labels = mnist.test.labels
session_config = tf.ConfigProto()
session_config.gpu_options.allow_growth = True
config = RunConfig(session_config=session_config)
mnist_classifier = tf.estimator.Estimator(model_fn=cnn_model_fn,
model_dir='./model/',
config=config)
#tf.estimator.EstimatorSpec()
tensors_to_log = {"probabilities": "softmax_tensor"}
logging_hook = tf.train.LoggingTensorHook(tensors=tensors_to_log,
every_n_iter=50)
train_input_fn = tf.estimator.inputs.numpy_input_fn(x={'x': train_data},
y=train_labels,
batch_size=100,
num_epochs=None,
shuffle=True)
mnist_classifier.train(input_fn=train_input_fn,
steps=20000,
hooks=[logging_hook])
eval_input_fn = tf.estimator.inputs.numpy_input_fn(x={'x': test_data},
y=test_labels,
num_epochs=1,
shuffle=False)
eval_result = mnist_classifier.evaluate(input_fn=eval_input_fn)
print(eval_result)
def build_estimator(model_dir):
config = RunConfig(save_checkpoints_secs=180)
print('----------------------TF_CONFIG--------------------------')
print(os.environ['TF_CONFIG'])
print('---------------------------------------------------------')
return tf.estimator.Estimator(model_fn=_cnn_model_fn,
model_dir=model_dir,
config=config)
def start_training(data_directory, dataset_name, output_directory,
network_name, batch_size, learning_rate, batch_threads,
num_epochs, initial_checkpoint, checkpoint_exclude_scopes,
ignore_missing_variables, trainable_scopes,
not_trainable_scopes, fixed_learning_rate,
learning_rate_decay_rate, do_evaluation,
learning_rate_decay_steps):
dataset_factory = DatasetFactory(dataset_name=dataset_name,
data_directory=data_directory)
model_params = {
'learning_rate':
learning_rate,
'fixed_learning_rate':
fixed_learning_rate,
'learning_rate_decay_rate':
learning_rate_decay_rate,
'learning_rate_decay_steps':
(dataset_factory.get_dataset('train').get_number_of_samples() if
learning_rate_decay_steps is None else learning_rate_decay_steps) //
batch_size
}
run_config = RunConfig(keep_checkpoint_max=10, save_checkpoints_steps=None)
# Instantiate Estimator
estimator = tf.estimator.Estimator(model_fn=get_model_function(
output_directory, network_name,
dataset_factory.get_dataset('train').num_classes(), initial_checkpoint,
checkpoint_exclude_scopes, ignore_missing_variables, trainable_scopes,
not_trainable_scopes),
params=model_params,
model_dir=output_directory,
config=run_config)
image_size = nets_factory.get_input_size(network_name)
dataset = dataset_factory.get_dataset('train')
evaluation_summary_writer = get_evaluation_summary_writer(
do_evaluation, output_directory)
for epoch in range(num_epochs):
run_training(dataset=dataset,
batch_size=batch_size,
batch_threads=batch_threads,
epoch=epoch,
estimator=estimator,
num_epochs=num_epochs,
image_size=image_size)
if do_evaluation:
run_evaluation_conserving_best(
estimator=estimator,
batch_size=2 * batch_size,
batch_threads=batch_threads,
dataset_factory=dataset_factory,
image_size=image_size,
evaluation_summary_writer=evaluation_summary_writer)
print('Finished training')
def main(_argv):
model_dir = tf.flags.FLAGS.model_dir
run_config = RunConfig(model_dir=model_dir)
hparams = HParams(generator_steps=1,
discriminator_steps=1,
latent_units=100,
dis_lr=1e-4,
gen_lr=1e-3)
tf.contrib.learn.learn_runner.run(experiment_fn=experiment_fn,
run_config=run_config,
schedule=tf.flags.FLAGS.schedule,
hparams=hparams)
def start_training(data_directory, dataset_name, mean, output_directory,
network_name, batch_size, learning_rate, learning_rate_gen,
beta1_gen, separable_conv, batch_threads, num_epochs,
initial_checkpoint, checkpoint_exclude_scopes,
ignore_missing_variables, trainable_scopes,
not_trainable_scopes, fixed_learning_rate,
learning_rate_decay_rate, do_evaluation,
learning_rate_decay_steps, img_size):
dataset_factory = DatasetFactory(dataset_name=dataset_name,
data_directory=data_directory,
mean=mean)
model_params = {
'learning_rate':
learning_rate,
'learning_rate_gen':
learning_rate_gen,
'beta1_gen':
beta1_gen,
'fixed_learning_rate':
fixed_learning_rate,
'learning_rate_decay_rate':
learning_rate_decay_rate,
'separable_conv':
separable_conv,
'learning_rate_decay_steps':
(dataset_factory.get_dataset('train').get_number_of_samples() if
learning_rate_decay_steps is None else learning_rate_decay_steps) //
batch_size
}
run_config = RunConfig(keep_checkpoint_max=10, save_checkpoints_steps=None)
# Instantiate Estimator
estimator = tf.estimator.Estimator(model_fn=get_model_function(
output_directory, network_name,
dataset_factory.get_dataset('train').num_classes(), initial_checkpoint,
checkpoint_exclude_scopes, ignore_missing_variables, trainable_scopes,
not_trainable_scopes),
params=model_params,
model_dir=output_directory,
config=run_config)
image_size = img_size
for epoch in range(num_epochs):
run_training(dataset_factory,
batch_size=batch_size,
batch_threads=batch_threads,
epoch=epoch,
estimator=estimator,
num_epochs=num_epochs,
image_size=image_size)
print('Finished training')
def main(_argv):
if tf.flags.FLAGS.debug:
enable_debugging_monkey_patch()
model_dir = tf.flags.FLAGS.model_dir
os.makedirs(model_dir, exist_ok=True)
print("model_dir={}".format(model_dir))
run_config = RunConfig(model_dir=model_dir)
hparams = get_hparams(model_dir, create=True)
estimator = tf.contrib.learn.learn_runner.run(
experiment_fn=experiment_fn,
run_config=run_config,
schedule=tf.flags.FLAGS.schedule,
hparams=hparams)
def main(_argv, config: TextConfig):
model_dir = tf.flags.FLAGS.model_dir
os.makedirs(model_dir, exist_ok=True)
print("model_dir={}".format(model_dir))
run_config = RunConfig(
model_dir=model_dir,
save_checkpoints_steps=tf.flags.FLAGS.save_checkpoints_steps)
# save_checkpoints_secs=tf.flags.FLAGS.save_checkpoints_secs)
hparams = get_hparams(model_dir, create=True)
estimator = run(experiment_fn=make_experiment_fn(config),
run_config=run_config,
schedule=tf.flags.FLAGS.schedule,
hparams=hparams)
def _build_run_config(self):
valid_runconfig_keys = ['save_summary_steps', 'save_checkpoints_secs', 'save_checkpoints_steps',
'keep_checkpoint_max', 'keep_checkpoint_every_n_hours', 'log_step_count_steps']
runconfig_params = {k: v for k, v in self.customer_params.items() if k in valid_runconfig_keys}
logger.info("creating RunConfig:")
logger.info(runconfig_params)
run_config = RunConfig(
model_dir=self.model_path,
**runconfig_params
)
return run_config
def main():
hparams = HParams(**HPARAMS)
run_config = RunConfig(model_dir='./save')
if len(argv) < 2 or argv[1] == 'train':
learn_runner.run(
experiment_fn=experiment_fn,
run_config=run_config,
schedule="train_and_evaluate",
hparams=hparams,
)
elif argv[1] == 'predict':
pass
else:
print('Unknown Operation.')
def main(_argv):
# Pass command-line arguments to RunConfig
run_config = RunConfig(
model_dir=tf.flags.FLAGS.model_dir,
save_checkpoints_steps=tf.flags.FLAGS.save_checkpoints_steps)
# Default hyperparameters
hparams = HParams(l2=1e-3, lr=1e-3, hidden_layers=3, hidden_units=200) \
# Parse the hparams command-line argument
hparams.parse(tf.flags.FLAGS.hparams)
# Run the experiment
run(
experiment_fn=experiment_fn,
run_config=run_config,
schedule=tf.flags.FLAGS.schedule,
hparams=hparams)
def start_training(data_directory, dataset_name, output_directory, network_name, batch_size, learning_rate, batch_threads, num_epochs, initial_checkpoint, checkpoint_exclude_scopes,ignore_missing_variables, trainable_scopes, fixed_learning_rate, learning_rate_decay_rate, num_classes):
dataset_factory = DatasetFactory(dataset_name=dataset_name, data_directory=data_directory)
model_params = {'learning_rate': learning_rate,'fixed_learning_rate': fixed_learning_rate,'learning_rate_decay_rate': learning_rate_decay_rate,'learning_rate_decay_steps': dataset_factory.get_dataset('train').get_number_of_samples() // batch_size}
run_config = RunConfig(keep_checkpoint_max=10, save_checkpoints_steps=None)
# Instantiate Estimator
estimator = tf.estimator.Estimator(model_fn=get_model_function(output_directory, network_name, dataset_factory.get_dataset('train').num_classes() if num_classes is None else num_classes, initial_checkpoint,checkpoint_exclude_scopes, ignore_missing_variables, trainable_scopes),params=model_params,model_dir=output_directory,config=run_config)
image_size = nets_factory.get_input_size(network_name)
for epoch in range(num_epochs):
run_training(dataset_factory, batch_size, batch_threads, epoch, estimator, num_epochs, image_size)
#run_validation(dataset_factory, batch_size, batch_threads, estimator, image_size)
run_evaluation(batch_size, batch_threads, dataset_factory, estimator, image_size)
def main():
parsed_args = get_parser().parse_args()
with open(os.path.join("data/challenger.ai", 'word_to_idx.pkl'), 'rb') as f:
word_to_idx = pickle.load(f)
hparams = HParams(vocab_size=len(word_to_idx),
batch_size=parsed_args.batch_size,
selector=parsed_args.selector,
dropout=parsed_args.dropout,
ctx2out=parsed_args.ctx2out,
prev2out=parsed_args.prev2out,
hard_attention=parsed_args.hard_attention,
bin_size=14)
run_config = RunConfig(model_dir=parsed_args.model_dir)
estimator = Estimator(
model_fn=model_fn_inner,
params=hparams,
config=run_config)
dataset = ChallengerAI("data/challenger.ai")
input_fn = dataset.get_tfrecords_test_input_fn(bin_size=hparams.bin_size)
val_init_hook = IteratorInitializerHook("infer")
idx_to_word = {v: k for k, v in word_to_idx.items()}
del word_to_idx
results = estimator.predict(input_fn, hooks=[val_init_hook])
all_predicions = []
image_ids = []
num_generated = 0
for batch_result in results:
image_id, pred = batch_result["image_id"], batch_result["predictions"]
result = ''.join([idx_to_word[idx] for idx in pred if idx != 0 and idx != 2])
all_predicions.append(result)
image_ids.append(image_id.decode("utf-8").split(".")[0])
num_generated = num_generated + 1
if num_generated % 1000 == 0:
print("Generated %d" % num_generated)
total_results = [{"image_id": img_id, "caption": pred}
for img_id, pred
in zip(image_ids, all_predicions)]
with open("result.json", "w", encoding="utf-8") as f:
json.dump(total_results, f, ensure_ascii=False)
def start_prediction(data_directory, dataset_name, mean, model_dir,
network_name, batch_size, batch_threads, num_classes,
result_dir, img_size, model, mode):
dataset_factory = DatasetFactory(dataset_name=dataset_name,
data_directory=data_directory,
mean=mean,
augment=False,
num_classes=num_classes)
run_config = RunConfig(keep_checkpoint_max=10, save_checkpoints_steps=None)
# Instantiate Estimator
estimator = tf.estimator.Estimator(model_fn=get_model_function(
model_dir, network_name,
dataset_factory.get_dataset('train').num_classes()),
model_dir=model_dir,
config=run_config,
params={})
image_size = img_size
run_prediction_and_evaluation(batch_size, batch_threads, dataset_factory,
estimator, image_size, result_dir, mode)
def generate_captions(model_dir, output_dir):
os.makedirs(output_dir, exist_ok=True)
vocab = np.load('output/processed-annotations/vocab.npy')
run_config = RunConfig(model_dir=model_dir)
hparams = get_hparams(model_dir, create=False)
print(hparams)
estimator = Estimator(
model_fn=model_fn,
config=run_config,
params=hparams)
val_path = tf.flags.FLAGS.batch_path
use_slot_vocab = hparams.use_slot_vocab
hook = FeedFnHook(path_fmt=val_path, splits=1, batch_size=tf.flags.FLAGS.batch_size, predict=True)
with open(os.path.join(output_dir, 'captions.csv'), 'w', newline='') as f:
w = csv.writer(f)
w.writerow(['Index', 'Caption'])
for i, prediction in enumerate(estimator.predict(input_fn=predict_input_fn, hooks=[hook])):
caption = write_prediction(os.path.join(output_dir, '{:08d}'.format(i)),
prediction=prediction, vocab=vocab, use_slot_vocab=use_slot_vocab)
w.writerow([i, caption])
if i > 100:
break
def _build_estimator(self):
valid_runconfig_keys = [
'save_summary_steps', 'save_checkpoints_secs',
'save_checkpoints_steps', 'keep_checkpoint_max',
'keep_checkpoint_every_n_hours', 'log_step_count_steps'
]
runconfig_params = {
k: v
for k, v in self.customer_params.items()
if k in valid_runconfig_keys
}
logging.info("creating RunConfig:")
logging.info(runconfig_params)
run_config = RunConfig(model_dir=self.model_path, **runconfig_params)
if hasattr(self.customer_script, 'estimator_fn'):
logging.info("invoking estimator_fn")
return self.customer_script.estimator_fn(run_config,
self.customer_params)
elif hasattr(self.customer_script, 'keras_model_fn'):
logging.info("involing keras_model_fn")
model = self.customer_script.keras_model_fn(self.customer_params)
return tf.keras.estimator.model_to_estimator(keras_model=model,
config=run_config)
else:
logging.info("creating the estimator")
# transforming hyperparameters arg to params, which is required by tensorflow
def _model_fn(features, labels, mode, params):
return self.customer_script.model_fn(features, labels, mode,
params)
return tf.estimator.Estimator(model_fn=_model_fn,
params=self.customer_params,
config=run_config)
def main():
hparams = HParams(**HPARAMS)
run_config = RunConfig(model_dir='./save')
if len(argv) < 2 or argv[1] == 'train':
learn_runner.run(
experiment_fn=experiment_fn,
run_config=run_config,
schedule="train_and_evaluate",
hparams=hparams,
)
elif argv[1] == 'predict':
input_fn_factory = ModelInputs(hparams.vocab_paths, 1)
predict_input_fn, predict_input_hook = input_fn_factory.get_inputs(
hparams.predict_dataset_path,
mode=estimator.ModeKeys.PREDICT,
num_infer=1)
classifier = get_estimator(run_config, hparams)
predictions = classifier.predict(input_fn=predict_input_fn,
hooks=[predict_input_hook])
print_predictions(predictions, hparams)
else:
print('Unknown Operation.')
def main():
parsed_args = get_parser().parse_args()
with open(os.path.join("data", 'word_to_idx.pkl'), 'rb') as f:
word_to_idx = pickle.load(f)
hparams = HParams(vocab_size=len(word_to_idx),
batch_size=parsed_args.batch_size,
selector=parsed_args.selector,
dropout=parsed_args.dropout,
ctx2out=parsed_args.ctx2out,
prev2out=parsed_args.prev2out)
run_config = RunConfig(model_dir=parsed_args.model_dir)
estimator = Estimator(model_fn=model_fn, params=hparams, config=run_config)
image_ids, input_fn = get_input_fn()
val_init_hook = IteratorInitializerHook("infer")
idx_to_word = {v: k for k, v in word_to_idx.items()}
del word_to_idx
pred_results = estimator.predict(input_fn, hooks=[val_init_hook])
all_predicions = []
num_generated = 0
for pred in pred_results:
result = ' '.join(
[idx_to_word[idx] for idx in pred if idx != 0 and idx != 2])
all_predicions.append(result)
num_generated = num_generated + 1
if num_generated % 1000 == 0:
print("Generated %d" % num_generated)
total_results = [{
"image_id": img_id,
"caption": pred
} for img_id, pred in zip(image_ids, all_predicions)]
with open("result.json", "w") as f:
json.dump(total_results, f)
clip_max=1.0,
time_shift_samples=0,
)
result[FINGERPRINT_KEY]=getMfcc(getTransformedAudioLocal(**audio_options))
yield result
test_input_fn = generator_input_fn(
x=test_data_generator,
batch_size=TEST_BATCH_SIZE,
shuffle=False,
num_epochs=1,
queue_capacity= 10 * TEST_BATCH_SIZE,
num_threads=1,
)
model = create_estimator(
config=RunConfig(model_dir=model_dir),
hparams=HParams(**params),
)
it = model.predict(input_fn=test_input_fn)
submission = dict()
for t in tqdm(it):
fname, label = t['fname'].decode(), id2name[t['label']]
submission[fname] = label
with open(os.path.join(model_dir, 'submission.csv'), 'w') as fout:
fout.write('fname,label\n')
for fname, label in submission.items():
fout.write('{},{}\n'.format(fname, label))
from tensorflow.contrib.layers import one_hot_column, real_valued_column, sparse_column_with_keys
from tensorflow.contrib.layers.python.layers.feature_column import _OneHotColumn, _RealValuedColumn, _SparseColumnKeys
from tensorflow.contrib.learn.python.learn.utils.input_fn_utils import InputFnOps
import numpy
import os
import pandas
import shutil
import tempfile
import tensorflow as tf
tf.logging.set_verbosity(tf.logging.INFO)
#tf.reset_default_graph()
estimator_conf = RunConfig(num_cores = 1, tf_random_seed = 42)
def load_csv(name):
return pandas.read_csv("csv/" + name)
def store_csv(df, name):
df.to_csv("csv/" + name, index = False)
def store_savedmodel(estimator, serving_input_fn, name):
savemodel_dir = estimator.export_savedmodel(tempfile.mkdtemp(), serving_input_fn = serving_input_fn, as_text = True)
savemodel_dir = savemodel_dir.decode("UTF-8")
if(os.path.isdir("savedmodel/" + name)):
shutil.rmtree("savedmodel/" + name)
shutil.move(savemodel_dir, "savedmodel/" + name)
def setUpClass(self):
""" setup test env
"""
tf.logging.set_verbosity(tf.logging.INFO)
self.estimator_conf = RunConfig(num_cores=1, tf_random_seed=42)
self.data = TestData()
args = parser.parse_args()
# Input pipe settings
input_param = {
'data_dir': args.data_dir,
'batch_size': args.batch_size,
'buffer_size': args.buffer_size,
'epochs': args.train_epochs,
'num_parallel_calls': args.num_parallel_calls,
'img_sizes': input_pipe.get_tf_record_image_size(args.data_dir),
'padding': args.unet_padding
}
# Create run configuration default
run_config = RunConfig()
run_config = run_config.replace(model_dir=os.path.join(args.output_dir, args.model_type))
run_config = run_config.replace(save_summary_steps=args.save_summary_steps)
run_config = run_config.replace(save_checkpoints_steps=args.save_checkpoints_steps)
# Define model and input parameters
hparams = HParams(
learning_rate=args.learning_rate,
l2_gain=args.l2_gain,
model_type=args.model_type,
rmsprop_momentum=args.rmsprop_momentum,
opt_epsilon=args.opt_epsilon,
rmsprop_decay=args.rmsprop_decay,
padding=args.unet_padding,
optimizer=args.optimizer,
model_dir=run_config.model_dir