def main():
# enable TensorFlow logging
tf.logging.set_verbosity(tf.logging.INFO)
tf_logging._get_logger().propagate = False # fix double messages
# directory with the exported model
saved_model_dir = root_dir('export/final_model')
# image size that the model accepts
image_size = 48
# load the images from the dataset
_, imgs = load_data()
# get test images and crop them to the right size
imgs = get_test_dataset(imgs, image_size)
# load the model
predict_fn = tf.contrib.predictor.from_saved_model(saved_model_dir)
# get predictions
res = predict_fn({'image': imgs})
# print predicted spikes
pprint(res['spikes'])
def __init__(self, config: dict):
super().__init__(config)
# load the data
df, imgs = load_data()
train_imgs, train_labels, eval_imgs, eval_labels = \
get_train_datasets(df, imgs, config['data']['eval_size'], config['model']['image_size'],
config['data'].get('random_seed', RANDOM_SEED))
train_nan_mask = np.logical_not(np.isnan(train_labels))
eval_nan_mask = np.logical_not(np.isnan(eval_labels))
train_labels = np.nan_to_num(train_labels)
eval_labels = np.nan_to_num(eval_labels)
# mask all the columns that are not supposed to be trained
column_id = config['training'].get('only_column_id')
if column_id is not None:
train_nan_mask[:, :column_id] = False
train_nan_mask[:, (column_id + 1):] = False
eval_nan_mask[:, :column_id] = False
eval_nan_mask[:, (column_id + 1):] = False
# build input functions
self._train_input_fn = build_input_fn(train_imgs, train_labels, train_nan_mask,
distortions=config['data'].get('distortions'), num_epochs=None)
self._eval_train_input_fn = build_input_fn(train_imgs, train_labels, train_nan_mask, num_epochs=None)
self._eval_input_fn = build_input_fn(eval_imgs, eval_labels, eval_nan_mask, num_epochs=1)
self._serving_input_receiver_fn = build_serving_input_receiver_fn(config['model']['image_size'])
def main():
tf.logging.set_verbosity(tf.logging.INFO)
models_dir = 'training/single_column/models3'
# load the data
df, stim = load_data()
columns = list(df.columns)[1:]
test_data = {'image': stim[:50, 16:-16, 16:-16]}
# get predictions from all the models
column_predictions = {}
for i, column_name in enumerate(columns):
print('Predicting values for the column "%s"...' % column_name)
# find the model directory
best_models_path = root_dir('%s/%d_%s/export/best' % (models_dir, i, column_name))
latest_model_subdir = sorted(os.listdir(best_models_path), reverse=True)[0]
latest_model_dir = os.path.join(best_models_path, latest_model_subdir)
# create predictor
predict_fn = predictor.from_saved_model(latest_model_dir)
# get predictions
column_predictions[column_name] = predict_fn(test_data)['spike']
# generate a submission file
with open(root_dir('data/submission/single_column/3.csv'), 'w') as f:
writer = csv.writer(f)
writer.writerow(['Id'] + columns)
for i in range(len(test_data['image'])):
writer.writerow([i] + [column_predictions[column_name][i] for column_name in columns])
def __init__(self, config: dict):
super().__init__(config)
# load the data
df, imgs = load_data()
train_imgs, train_labels, eval_imgs, eval_labels = \
get_train_datasets(df, imgs, config['data']['eval_size'], config['model']['image_size'])
# build input functions
batch_size = config['training'].get('batch_size')
self._train_input_fn = build_input_fn(train_imgs,
train_labels,
batch_size=batch_size,
num_epochs=None,
shuffle=True)
self._eval_input_fn = build_input_fn(eval_imgs,
eval_labels,
num_epochs=1)
self._serving_input_receiver_fn = build_serving_input_receiver_fn(
config['model']['image_size'])
def generate_submission(model_type, model_dir, submission_num):
submission_dir = root_dir('data/submission/%s/%s' % (model_type, submission_num))
if os.path.isdir(submission_dir):
raise ValueError('Submission #%d already exists' % submission_num)
os.makedirs(submission_dir)
# load the data
df, stim = load_data()
columns = list(df.columns)[1:]
config = load_model_config(model_dir)
test_data = {'image': get_test_dataset(stim, config['model']['image_size'])}
# create the predictor
export_dir = root_dir(os.path.join(model_dir, 'export', 'best'))
latest_model_subdir = sorted(os.listdir(export_dir), reverse=True)[0]
latest_model_dir = os.path.join(export_dir, latest_model_subdir)
# get predictor
predict_fn = predictor.from_saved_model(latest_model_dir)
# get predictions
predictions = predict_fn(test_data)['spikes']
# generate a submission file
with open(os.path.join(submission_dir, 'submission_%d.csv' % submission_num), 'w') as f:
writer = csv.writer(f)
writer.writerow(['Id'] + columns)
for i in range(len(test_data['image'])):
writer.writerow([i] + list(predictions[i]))
# copy config file
config_path = get_model_config_path(model_dir)
copyfile(config_path, os.path.join(submission_dir, CONFIG_FILENAME))
# copy the model
copytree(latest_model_dir, os.path.join(submission_dir, 'model'))
def main():
tf.logging.set_verbosity(tf.logging.INFO)
eval_steps = 10
eval_size = 50
export_best_models = True
train_column_id = None
models_dir = 'training/single_column/models3'
models_postfix = ''
# load the data
df, imgs = load_data()
columns = list(df.columns)[1:]
# train a model for each column
for i, column_name in enumerate(columns):
if (train_column_id is not None) and (i != train_column_id):
continue
print('Training the model for the column "%s"...' % column_name)
train_imgs, train_labels, eval_imgs, eval_labels = get_column_datasets(column_name, df, imgs, eval_size)
print('Train size: %d, eval size: %d' % (len(train_labels), len(eval_labels)))
# create the estimator
model_dir = root_dir('%s/%d_%s' % (models_dir, i, column_name))
if models_postfix:
model_dir += '_' + models_postfix
estimator = tf.estimator.Estimator(
model_fn=model_fn,
config=tf.estimator.RunConfig(
model_dir=model_dir,
save_checkpoints_steps=eval_steps,
save_summary_steps=eval_steps,
)
)
# training input function
train_data = {'image': train_imgs}
train_input_fn = tf.estimator.inputs.numpy_input_fn(x=train_data, y=train_labels,
batch_size=len(train_labels), num_epochs=None, shuffle=True)
# evaluation training function
eval_data = {'image': eval_imgs}
eval_input_fn = tf.estimator.inputs.numpy_input_fn(x=eval_data, y=eval_labels, num_epochs=1, shuffle=False)
# hooks
early_stopping_hook = early_stopping.stop_if_no_decrease_hook(estimator, 'rmse', eval_steps * 10,
run_every_secs=None, run_every_steps=eval_steps)
exporter = tf.estimator.BestExporter(name='best',
serving_input_receiver_fn=serving_input_receiver_fn,
exports_to_keep=1,
compare_fn=lambda best_eval_result, current_eval_result:
# should be "<=" to export the best model on the 1st evaluation
current_eval_result['rmse'] <= best_eval_result['rmse'],
)
# train and evaluate
train_spec = tf.estimator.TrainSpec(input_fn=train_input_fn, hooks=[early_stopping_hook])
eval_spec = tf.estimator.EvalSpec(input_fn=eval_input_fn,
exporters=(exporter if export_best_models else None),
throttle_secs=0)
tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)