'image_feature_size', 196,
'Multiplication of width and height of image feature\'s dimension, e.g 14x14=196 in the original paper.'
)
flags.DEFINE_integer(
'image_feature_depth', 1024,
'Depth dimension of image feature, e.g 512 if you extract features at conv-5 of VGG-16 model.'
)
flags.DEFINE_integer('lstm_hidden_size', 1536,
'Hidden layer size for LSTM cell.')
flags.DEFINE_integer('time_steps', 31,
'Number of time steps to be iterating through.')
flags.DEFINE_integer('embed_dim', 512,
'Embedding space size for embedding tokens.')
flags.DEFINE_integer('beam_size', 3, 'Beam size for inference phase.')
flags.DEFINE_float('dropout', 0.5, 'Dropout portion.')
flags.DEFINE_boolean('prev2out', True, 'Link previous hidden state to output.')
flags.DEFINE_boolean('ctx2out', True, 'Link context features to output.')
flags.DEFINE_boolean(
'enable_selector', True,
'Enable selector to determine how much important the image context is at every time step.'
)
"""Training parameters"""
flags.DEFINE_string('optimizer', 'rmsprop',
'Optimizer used to update model\'s weights.')
flags.DEFINE_float('learning_rate', 0.0001, 'Initial learning rate.')
flags.DEFINE_integer('num_epochs', 10, 'Number of epochs.')
flags.DEFINE_integer('batch_size', 128, 'Number of examples per mini-batch.')
flags.DEFINE_integer('snapshot_steps', 10,
'Logging every snapshot_steps steps.')
flags.DEFINE_integer(
'eval_steps', 100,
'Vocabulary is made of vocab_size most frequent words in the dataset. ' +
'Leave it to default value means not using it.')
flags.DEFINE_string(
'model_name', 'resnet', 'Model used to extract features of images.' +
'It should be vgg or resnet.')
flags.DEFINE_string(
'model_num_layers', '101', 'Number of layers for model\'s architecture.' +
'If model_name is vgg, this variable can take values of 16 or 19.' +
'If model_name is resnet, this variable can take values of 50, 101 or 152.'
)
flags.DEFINE_boolean(
'use_tf', False,
'Whether to use Tensorflow slim model to extract features of images or use pytorch.'
+
'Using Pytorch will be faster but it will keep you from finetuning the extracting model.'
+ 'Default is set to use Pytorch models.')
flags.DEFINE_string(
'model_ckpt', '',
'Model checkpoint to load model\'s weights to extract images.' +
'Only use this variable when you use tensorflow model.')
def _process_caption_data(phase,
caption_file=None,
image_dir=None,
max_length=None):
if phase == 'test' or phase == 'val':
if phase == 'val':
import numpy
import os
import sys
import tensorflow as tf
from tensorflow import flags
from third_party.node2vec import node2vec
flags.DEFINE_string('input', '',
'Path to edge-list textfile. Required unless '
'--only_simulate_walks is set.')
flags.DEFINE_boolean('only_simulate_walks', False,
'If set, train.txt.npy will be read from --output_dir, '
'random walks will be simulated, out their output will be '
'written to --output_dir')
flags.DEFINE_string('output_dir', '',
'Directory where training files will be written.')
flags.DEFINE_boolean('directed', False, 'Must be set if graph is directed.')
flags.DEFINE_boolean('partition', True,
'If set (default), separates a test split, containing '
'half of the edges. In which case, train graph will be '
'connected.')
flags.DEFINE_integer('num_walks', 5, 'Number of walks per node.')
flags.DEFINE_integer('walk_length', 40,
'Length of each walk. Total number of pairs will be '
'O(walk_length * num_walks * num nodes * context^2)')
flags.DEFINE_integer('context', 3,
'Size of context from each side (right and left). If '
from stable_baselines.common.vec_env import DummyVecEnv
import tensorflow as tf
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
from eval_model import evaluate
import csv
import utils
from baselines.L2SP.model import PPO2L2SP
import ipdb
FLAGS = flags.FLAGS
flags.DEFINE_integer("timesteps", 256000, "# timesteps to train")
flags.DEFINE_string("experiment_dir", "output/updated_gridworld_continuous_L2SP", "Name of experiment")
flags.DEFINE_string("experiment_name", "B0R_B0L_L2SP", "Name of experiment")
flags.DEFINE_boolean("is_save", True, "Saves and logs experiment data if True")
flags.DEFINE_integer("eval_save_period", 1, "how often we save state for eval") # fine
flags.DEFINE_integer("num_envs", 1, "number of envs")
flags.DEFINE_integer("seed", 10, "random seed")
flags.DEFINE_string("target_env", "", "Name of target environment")
flags.DEFINE_string("source_env", "", "Name of source environment")
flags.DEFINE_integer("bs", 1, "barrier_size")
def find_best(dir_name):
def compare(item):
return item[0]
model_list = []
for file_name in os.listdir(dir_name):
if '.pkl' in file_name and ('final' not in file_name) and ('best_model' not in file_name):
"Comma-separated filter sizes (default: '3,4,5')")
flags.DEFINE_integer("num_filters", 64,
"Number of filters per filter size (default: 128)")
flags.DEFINE_float("dropout_keep_prob", 0.5,
"Dropout keep probability (default: 0.5)")
flags.DEFINE_float("l2_reg_lambda", 0.001,
"L2 regularizaion lambda (default: 0.0)")
flags.DEFINE_float("learning_rate", 0.001, "learn rate( default: 0.0)")
flags.DEFINE_integer("max_len_query", 4, "max document length of left input")
flags.DEFINE_integer("max_len_document", 50,
"max document length of right input")
flags.DEFINE_string("loss", "point_wise", "loss function (default:point_wise)")
flags.DEFINE_integer('extend_feature_dim', 10, 'overlap_feature_dim')
# Training parameters
flags.DEFINE_integer("batch_size", 20, "Batch Size (default: 64)")
flags.DEFINE_boolean("trainable", False,
"is embedding trainable? (default: False)")
flags.DEFINE_integer("num_epochs", 100,
"Number of training epochs (default: 200)")
flags.DEFINE_integer(
"evaluate_every", 500,
"Evaluate model on dev set after this many steps (default: 100)")
flags.DEFINE_integer("checkpoint_every", 500,
"Save model after this many steps (default: 100)")
flags.DEFINE_boolean('overlap_needed', True, "is overlap used")
flags.DEFINE_boolean('position_needed', False, 'is position used')
flags.DEFINE_boolean('dns', 'False', 'whether use dns or not')
flags.DEFINE_string('data', 'clueweb09_title', 'data set')
flags.DEFINE_string('CNN_type', 'ircnn', 'data set')
flags.DEFINE_float('sample_train', 1, 'sampe my train data')
flags.DEFINE_boolean(
'fresh', True,
"The directory to load the model files from. "
"The tensorboard metrics files are also saved to this "
"directory.")
flags.DEFINE_string("checkpoint_file", "", "e.g. model.ckpt-170820")
flags.DEFINE_string(
"eval_data_pattern", "",
"File glob defining the evaluation dataset in tensorflow.SequenceExample "
"format. The SequenceExamples are expected to have an 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
# Other flags.
flags.DEFINE_integer("batch_size", 1024,
"How many examples to process per batch.")
flags.DEFINE_integer("num_readers", 8,
"How many threads to use for reading input files.")
flags.DEFINE_boolean("run_once", False, "Whether to run eval only once.")
flags.DEFINE_integer("top_k", 20,
"How many predictions to output per video.")
flags.DEFINE_integer("ensemble_num", 1, "")
flags.DEFINE_multi_integer(
"netvlad_cluster_size", 64,
"Number of units in the NetVLAD cluster layer.")
flags.DEFINE_multi_integer("netvlad_hidden_size", 1024,
"Number of units in the NetVLAD hidden layer.")
flags.DEFINE_multi_float("ensemble_wts", 1, '')
flags.DEFINE_boolean(
"L2 regularization lambda (default: 0.0)")
# Training parameters
flags.DEFINE_integer("batch_size", 32, "Batch Size (default: 64)")
flags.DEFINE_integer("num_epochs", 10,
"Number of training epochs (default: 200)")
flags.DEFINE_integer(
"evaluate_every", 100,
"Evaluate model on dev set after this many steps (default: 100)")
flags.DEFINE_integer("checkpoint_every", 100,
"Save model after this many steps (default: 100)")
flags.DEFINE_integer("num_checkpoints", 5,
"Number of checkpoints to store (default: 5)")
# Misc Parameters
flags.DEFINE_boolean("allow_soft_placement", True,
"Allow device soft device placement")
flags.DEFINE_boolean("log_device_placement", False,
"Log placement of ops on devices")
flags.DEFINE_boolean(
'use_allow_growth', True,
'use Allow Growth flag which will allocate only required amount of GPU memory and prevent full allocation of available GPU memory'
)
none = None
#FLAGS = tf.flags.FLAGS
#FLAGS._parse_flags()
FLAGS = tf.flags.FLAGS
import sys
FLAGS(sys.argv)
print("\nParameters:")
for attr, value in sorted(FLAGS.__flags.items()):
from data import generate
import h5py as h5
import numpy as np
import time
#reproduction of HAN
tf.logging.set_verbosity(tf.logging.INFO)
FLAGS = flags.FLAGS
flags.DEFINE_string("data_dir",'data/yelp-2013', 'directory containing train, val and test h5')
flags.DEFINE_string('checkpoint_dir','checkpoint','directory to save the best model saved as checkpoint_dir/model.chkpt')
flags.DEFINE_string('restore_checkpoint',None,'restore to a state; if None train from scratch default : ')
flags.DEFINE_boolean('gpu',True,'use --nogpu to disable gpu')
flags.DEFINE_integer('epoch',2,'epoch : default 2s')
flags.DEFINE_integer('batchsize',32,'batchsize: default 32')
#network parameters
flags.DEFINE_float('hidden_dim',100 ,'GRU hidden dimension : default 100')
#hyper params
flags.DEFINE_float('lr',1e-3,'Learning rate : default 1e-3')
class Attention():
def __init__(self, input, mask, scope ='A0'):
assert input.get_shape().as_list()[:-1] == mask.get_shape().as_list() and len(mask.get_shape().as_list()) == 2
_, steps, embed_dim = input.get_shape().as_list()
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import importlib
from tensorflow import flags
from data_process import get_data
# model relation
flags.DEFINE_string('classifier', 'bidirectional_lstm.BiLSTM',
"path of the Class for the classifier")
flags.DEFINE_integer('nb_epoch', 50, "number of epoch")
flags.DEFINE_integer('embed_size', 300, "hidden size of embedding layer")
flags.DEFINE_boolean('use_new_vector', False, 'if use new vector')
flags.DEFINE_integer('batch_size', 640, "batch size for train")
flags.DEFINE_string('optimizer', 'adam', "the optimizer for train")
flags.DEFINE_bool('use_pretrained', True,
"if use pretrained vector for embedding layer")
flags.DEFINE_bool(
'trainable', True,
"if the embedding layer is trainable. this param is used only `use_pretrained` is true"
)
# data relation
flags.DEFINE_integer('max_len', 150, "regular sentence to a fixed length")
flags.DEFINE_boolean('is_kfold', False, "is kfold")
flags.DEFINE_integer('kfold', 8, "k when kfold is true")
flags.DEFINE_boolean(
'is_retrain', False,
'if retrain, this will done when embedding is no-trainable')
from tensorflow import flags
import pdb
import tensorflow as tf
import copy
import io
import math
import randomized_telescope_runner as runner
FLAGS = flags.FLAGS
# Almost always leave these fixed
flags.DEFINE_boolean('use_cuda', True, 'use Cuda')
flags.DEFINE_float('meta_lr', None, 'meta-optimization learning rate')
flags.DEFINE_float('exp_decay', 0.9, 'exp decay constant')
flags.DEFINE_float('beta1', 0.9, 'adam beta1')
flags.DEFINE_float('beta2', 0.999, 'adam beta2')
flags.DEFINE_float('adam_eps', 1e-8, 'adam eps')
flags.DEFINE_float('mnist_momentum', 0.9, 'momentum of learner on mnist')
flags.DEFINE_float('warm_start_lr', 0.1, 'warm start learning rate')
flags.DEFINE_integer('warm_start_steps', 50, 'warm start steps')
flags.DEFINE_string('optimizer', 'sgd', 'sgd adam or mom')
"save_first_n_videos",
20,
"save as .png the individual frames of the first X videos",
)
flags.DEFINE_integer("sync_dist", -1,
"maximum distance between the syncronous digits") # 20
flags.DEFINE_integer("noise_parts_size", 8,
"size of the digits parts used as noise")
flags.DEFINE_integer("no_noise_parts", 0, "number of noise parts")
flags.DEFINE_integer("video_frames", 10, "number of frames in a video")
flags.DEFINE_string("split", "train", "train or test")
flags.DEFINE_string("dataset_name", "digits_sync", "name of the dataset")
flags.DEFINE_string("dataset_path", "./datasets/", "path to save the dataset")
flags.DEFINE_boolean(
"predefined", True,
"create random paths (moves) or load from predefined ones")
flags.DEFINE_string("predefined_path",
"./create_datasets/predefined_path_train.pickle",
"file for predefined digits paths (moves)")
# create the set of labels
# a label is defined by a set of two digits
# final label is for the case with all digits moving randomply
# 10 * 9 / 2 + 1 = 45 + 1 labels
def get_labels_dict_sincron():
# create labels
label = 0
labels_dict = {}
#! /usr/bin/env python
from net.build import TFNet
from tensorflow import flags
import os
#os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
flags.DEFINE_string("test", "./test/", "path to testing directory")
flags.DEFINE_boolean(
"slice_frame", False,
"Test by slicing big frame on smaller, because YOLO can't see small objects"
)
flags.DEFINE_boolean(
"from_videos", False,
"Predict from video (that will be found in test dir) frames with video_step"
)
flags.DEFINE_integer("video_step", 120, "Step for next frames in video")
flags.DEFINE_float("slice_step", 0.07, "Step of slicing window")
flags.DEFINE_float(
"slice_max_iou", 0.1,
"Max IOU above which rectangles will be considered as conflicting and will be choosed most probable one"
)
flags.DEFINE_integer("slice_win_size", 100,
"Size of window that will slice frame in pixels")
flags.DEFINE_boolean("save_xml", False, "Saves objects in custom XML file")
flags.DEFINE_boolean("save_image", False, "Saves image with painted objects")
flags.DEFINE_float(
"slice_down_rate", 0.8,
"Downsample rate for frame on each iteration ([0.0 - 1.0], new size will be win_size * rate)"
)
flags.DEFINE_string("binary", "./bin/", "path to .weights directory")
def train():
# # data loading
data_path = 'preprocessed_POSCO.csv' # csv 파일로 불러오기
# 포스코 모델
# data_path = 'repro_45.csv' # csv 파일로 불러오기
contents, points = tool.loading_rdata(data_path) # CSV 읽어오기
vocab_list = tool.cut(contents) # contents 에 모든 기사들을 1개의 리스트에 통합
# transform document to vector
max_document_length = 1400
x, vocabulary, vocab_size = tool.make_vocab(vocab_list,
max_document_length)
tool.save_vocab('news_vocab_sk.txt', contents, max_document_length)
# tool.save_vocab('news_vocab_posco.txt', vocabulary, max_document_length)
# vocab = tool.load_vocab('news_vocab_sk.txt')
print('사전단어수 : %s' % (vocab_size))
y = tool.make_output(points, threshold=0)
# divide dataset into train/test set
x_train, x_test, y_train, y_test = tool.divide(x, y, train_prop=0.9)
# Model Hyperparameters
flags.DEFINE_integer('embedding_dim', 128,
"Dimensionality of embedded vector (default: 128)")
flags.DEFINE_string('filter_sizes', '3,4,5',
"Comma-separated filter sizes (default: '3,4,5')")
flags.DEFINE_integer('num_filters', 128,
"Number of filters per filter size (default: 128)")
flags.DEFINE_float('dropout_keep_prob', 0.5,
"Dropout keep probability (default: 0.5)")
flags.DEFINE_float('l2_reg_lambda', 0.1,
"L2 regularization lambda (default: 0.0)")
# Training parameters
flags.DEFINE_integer("batch_size", 128, "Batch Size (default: 64)")
flags.DEFINE_integer("num_epochs", 200,
"Number of training epochs (default: 200)")
flags.DEFINE_integer(
"evaluate_every", 100,
"Evaluate model on dev set after this many steps (default: 100)")
flags.DEFINE_integer("checkpoint_every", 100,
"Save model after this many steps (default: 100)")
flags.DEFINE_integer("num_checkpoints", 10,
"Number of checkpoints to store (default: 5)")
# Misc Parameters
flags.DEFINE_boolean("allow_soft_placement", True,
"Allow device soft device placement")
flags.DEFINE_boolean("log_device_placement", False,
"Log placement of ops on devices")
FLAGS = flags.FLAGS
# print('\nParameters : ')
# for attr, value in sorted(FLAGS.flag_values_dict()):
# print('{}={}'.format(attr.upper(), value))
# print('')
with tf.Graph().as_default():
sess = tf.Session()
with sess.as_default():
cnn = TextCNN(sequence_length=x_train.shape[1],
num_classes=y_train.shape[1],
vocab_size=vocab_size,
embedding_size=FLAGS.embedding_dim,
filter_sizes=list(
map(int, FLAGS.filter_sizes.split(","))),
num_filters=FLAGS.num_filters,
l2_reg_lambda=FLAGS.l2_reg_lambda)
# cnn = CharCNN()
# Define Training procedure
global_step = tf.Variable(0, name="global_step", trainable=False)
optimizer = tf.train.AdamOptimizer(1e-3)
grads_and_vars = optimizer.compute_gradients(cnn.loss)
train_op = optimizer.apply_gradients(grads_and_vars,
global_step=global_step)
# Keep track of gradient values and sparsity (optional)
grad_summaries = []
for g, v in grads_and_vars:
if g is not None:
grad_hist_summary = tf.summary.histogram(
"{}".format(v.name), g)
sparsity_summary = tf.summary.scalar(
"{}".format(v.name), tf.nn.zero_fraction(g))
grad_summaries.append(grad_hist_summary)
grad_summaries.append(sparsity_summary)
grad_summaries_merged = tf.summary.merge(grad_summaries)
# Output directory for models and summaries
timestamp = str(int(time.time()))
out_dir = os.path.abspath(
os.path.join(os.path.curdir, "runs", timestamp))
print("Writing to {}\n".format(out_dir))
# Summaries for loss and accuracy
loss_summary = tf.summary.scalar("loss", cnn.loss)
acc_summary = tf.summary.scalar("accuracy", cnn.accuracy)
# Train Summaries
train_summary_op = tf.summary.merge(
[loss_summary, acc_summary, grad_summaries_merged])
train_summary_dir = os.path.join(out_dir, "summaries", "train")
train_summary_writer = tf.summary.FileWriter(
train_summary_dir, sess.graph)
# Dev summaries
dev_summary_op = tf.summary.merge([loss_summary, acc_summary])
dev_summary_dir = os.path.join(out_dir, "summaries", "dev")
dev_summary_writer = tf.summary.FileWriter(dev_summary_dir,
sess.graph)
# Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it
checkpoint_dir = os.path.abspath(
os.path.join(out_dir, "checkpoints"))
checkpoint_prefix = os.path.join(checkpoint_dir, "model")
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
saver = tf.train.Saver(tf.global_variables(),
max_to_keep=FLAGS.num_checkpoints)
# Initialize all variables
sess.run(tf.global_variables_initializer())
# initW = tool.load_embedding_vectors(vocabulary)
# sess.run(cnn.W.assign(initW))
def train_step(x_batch, y_batch):
"""
A single training step
"""
feed_dict = {
cnn.input_x: x_batch,
cnn.input_y: y_batch,
cnn.dropout_keep_prob: FLAGS.dropout_keep_prob
}
_, step, summaries, loss, accuracy = sess.run([
train_op, global_step, train_summary_op, cnn.loss,
cnn.accuracy
], feed_dict)
time_str = datetime.datetime.now().isoformat()
print("{}: step {}, loss {:g}, acc {:g}".format(
time_str, step, loss, accuracy))
train_summary_writer.add_summary(summaries, step)
def dev_step(x_batch, y_batch, writer=None):
"""
Evaluates model on a dev set
"""
feed_dict = {
cnn.input_x: x_batch,
cnn.input_y: y_batch,
cnn.dropout_keep_prob: 1.0
}
step, summaries, loss, accuracy = sess.run(
[global_step, dev_summary_op, cnn.loss, cnn.accuracy],
feed_dict)
time_str = datetime.datetime.now().isoformat()
print("{}: step {}, loss {:g}, acc {:g}".format(
time_str, step, loss, accuracy))
if writer:
writer.add_summary(summaries, step)
def batch_iter(data, batch_size, num_epochs, shuffle=True):
"""
Generates a batch iterator for a dataset.
"""
data = np.array(data)
data_size = len(data)
num_batches_per_epoch = int((len(data) - 1) / batch_size) + 1
for epoch in range(num_epochs):
# Shuffle the data at each epoch
if shuffle:
shuffle_indices = np.random.permutation(
np.arange(data_size))
shuffled_data = data[shuffle_indices]
else:
shuffled_data = data
for batch_num in range(num_batches_per_epoch):
start_index = batch_num * batch_size
end_index = min((batch_num + 1) * batch_size,
data_size)
yield shuffled_data[start_index:end_index]
# Generate batches
batches = batch_iter(list(zip(x_train, y_train)), FLAGS.batch_size,
FLAGS.num_epochs)
testpoint = 0
# Training loop. For each batch...
for batch in batches:
x_batch, y_batch = zip(*batch)
train_step(x_batch, y_batch)
current_step = tf.train.global_step(sess, global_step)
if current_step % FLAGS.evaluate_every == 0:
if testpoint + 100 < len(x_test):
testpoint += 100
else:
testpoint = 0
print("\nEvaluation:")
dev_step(x_test[testpoint:testpoint + 100],
y_test[testpoint:testpoint + 100],
writer=dev_summary_writer)
print("")
if current_step % FLAGS.checkpoint_every == 0:
path = saver.save(sess,
checkpoint_prefix,
global_step=current_step)
print("Saved model checkpoint to {}\n".format(path))
"File globs for original model input.")
flags.DEFINE_string("feature_names", "predictions", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "4716",
"Length of the feature vectors.")
# Model flags.
flags.DEFINE_string("model", "LinearRegressionModel",
"Which architecture to use for the model.")
flags.DEFINE_integer("batch_size", 256,
"How many examples to process per batch.")
flags.DEFINE_string("label_loss", "CrossEntropyLoss",
"Loss computed on validation data")
# Other flags.
flags.DEFINE_boolean("run_once", True, "Whether to run eval only once.")
flags.DEFINE_boolean("echo_gap", False, "Whether to echo GAP at the end.")
flags.DEFINE_integer("top_k", 20,
"How many predictions to output per video.")
def find_class_by_name(name, modules):
"""Searches the provided modules for the named class and returns it."""
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def get_input_evaluation_tensors(reader, data_pattern, batch_size=256):
logging.info("Using batch size of " + str(batch_size) + " for evaluation.")
with tf.name_scope("eval_input"):
files = gfile.Glob(data_pattern)
'CallVariantsOutput protos.')
flags.DEFINE_string(
'checkpoint', None,
'Required. Path to the TensorFlow model checkpoint to use to evaluate '
'candidate variant calls.')
flags.DEFINE_integer(
'batch_size', 512,
'Number of candidate variant tensors to batch together during inference. '
'Larger batches use more memory but are more computational efficient.')
flags.DEFINE_integer('max_batches', None,
'Max. batches to evaluate. Defaults to all.')
flags.DEFINE_integer('num_readers', 8,
'Number of parallel readers to create for examples.')
flags.DEFINE_string('model_name', 'inception_v3',
'The name of the model architecture of --checkpoint.')
flags.DEFINE_boolean('include_debug_info', False,
'If true, include extra debug info in the output.')
flags.DEFINE_string(
'execution_hardware', 'auto',
'When in cpu mode, call_variants will not place any ops on the GPU, even '
'if one is available. In accelerator mode call_variants validates that at '
'least some hardware accelerator (GPU/TPU) was available for us. This '
'option is primarily for QA purposes to allow users to validate their '
'accelerator environment is correctly configured. In auto mode, the '
'default, op placement is entirely left up to TensorFlow')
class ExecutionHardwareError(Exception):
pass
def prepare_inputs(source_path, model, batch_size, num_readers=None):
#! /usr/bin/env python
from darknet import *
from tfnet import *
from tensorflow import flags
flags.DEFINE_string("testset", "test", "path to testing directory")
flags.DEFINE_string("dataset", "../pascal/VOCdevkit/IMG/",
"path to dataset directory")
flags.DEFINE_string("annotation", "../pascal/VOCdevkit/ANN/",
"path to annotation directory")
flags.DEFINE_float("threshold", 0.1, "detection threshold")
flags.DEFINE_string("model", "", "configuration of choice")
flags.DEFINE_boolean("train", False, "training mode or not?")
flags.DEFINE_integer("load", 0,
"load a saved backup/checkpoint, -1 for newest")
flags.DEFINE_boolean("savepb", False, "save net and weight to a .pb file")
flags.DEFINE_float("gpu", 0.0, "How much gpu (from 0.0 to 1.0)")
flags.DEFINE_float("lr", 1e-5, "Learning rate")
flags.DEFINE_integer("keep", 20,
"Number of most recent training results to save")
flags.DEFINE_integer("batch", 12, "Batch size")
flags.DEFINE_integer("epoch", 1000, "Number of epoch")
flags.DEFINE_integer("save", 2000, "Save checkpoint every ? training examples")
FLAGS = flags.FLAGS
image = FLAGS.dataset
annot = FLAGS.annotation + 'parsed.bin'
def get_dir(dirs):
import tensorflow.contrib.slim as slim
from tensorflow.python import pywrap_tensorflow
FLAGS = flags.FLAGS
flags.DEFINE_string("Ensemble_Models", "./",
"the directory to store models for ensemble.")
flags.DEFINE_string("ensemble_model_path", None,
"the files to store models for ensemble.")
flags.DEFINE_string("ensemble_output_path", None,
"the files to store ensembled models.")
flags.DEFINE_string("eval_data_pattern", "", "")
flags.DEFINE_integer("num_readers", 8, "")
flags.DEFINE_integer("batch_size", 128, "")
flags.DEFINE_integer("top_k", 20, "")
flags.DEFINE_boolean("run_once", True, "")
flags.DEFINE_boolean("restore_once", False, "restore checkpoint once")
flags.DEFINE_integer("random_seed", 666, "")
flags.DEFINE_integer("tile_num", 10, "the number of sample copies")
tf.set_random_seed(FLAGS.random_seed)
def find_class_by_name(name, modules):
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def combine_models(models, model_input, num_frames, reader, labels_batch,
is_training):
model_nums = len(models)
prob_ratio = [1.0 for i in range(model_nums)]
flags.DEFINE_string(
'input_videos_csv', None,
'CSV file with lines "<video_file>,<labels>", where '
'<video_file> must be a path of a video and <labels> '
'must be an integer list joined with semi-colon ";"')
# Optional flags.
flags.DEFINE_string(
'model_dir', os.path.join(os.getenv('HOME'), 'yt8m'),
'Directory to store model files. It defaults to ~/yt8m')
# The following flags are set to match the YouTube-8M dataset format.
flags.DEFINE_integer('frames_per_second', 12,
'This many frames per second will be processed')
flags.DEFINE_boolean(
'skip_frame_level_features', False,
'If set, frame-level features will not be written: only '
'video-level features will be written with feature '
'names mean_*')
flags.DEFINE_string(
'labels_feature_key', 'labels',
'Labels will be written to context feature with this '
'key, as int64 list feature.')
flags.DEFINE_string(
'image_feature_key', 'rgb',
'Image features will be written to sequence feature with '
'this key, as bytes list feature, with only one entry, '
'containing quantized feature string.')
flags.DEFINE_string(
'video_file_feature_key', 'id',
'Input <video_file> will be written to context feature '
'with this key, as bytes list feature, with only one '
"model", "BasicUNetModel",
"Which architecture to use for the model. Models are defined "
"in models.py.")
# Model flags.
flags.DEFINE_integer(
"batch_size", 8,
"How many examples to process per batch for evaluating.")
flags.DEFINE_string("label_loss", "CrossEntropyLoss",
"Loss computed on validation data")
flags.DEFINE_float(
"prediction_threshold", 0.5,
"Which value to use as a threshold of true and false values")
# Other flags.
flags.DEFINE_boolean("run_once", False, "Whether to run eval only once.")
flags.DEFINE_boolean("half_memory", False,
"Whether to run eval with only 45% GPU memory.")
def find_class_by_name(name, modules):
"""Searches the provided modules for the named class and returns it."""
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def get_input_evaluation_tensors(reader,
data_list,
data_pattern,
batch_size=32,
nfold=5,
FLAGS = flags.FLAGS
flags.DEFINE_string('logging_verbosity', 'INFO', 'Level of logging verbosity '
'(e.g. `INFO` `DEBUG`).')
flags.DEFINE_string('project_id', None, 'GCP project id.')
flags.DEFINE_string('job_name', None, 'Dataflow job name.')
flags.DEFINE_integer('num_workers', None, 'Number of dataflow workers.')
flags.DEFINE_string('worker_machine_type', None, 'Machine types.')
flags.DEFINE_string('region', None, 'GCP region to use.')
flags.DEFINE_string('input_dir', None,
'Path of the directory containing input data.')
flags.DEFINE_string('output_dir', None, 'Path to write output data to.')
flags.DEFINE_float(
'train_size', 0.7, 'Percentage of input data to use for'
' training vs validation.')
flags.DEFINE_boolean('gcp', False, 'Runs on GCP or locally.')
flags.mark_flag_as_required('input_dir')
flags.mark_flag_as_required('output_dir')
def _mark_gcp_flags_as_required(inputs):
if FLAGS.gcp:
return bool(inputs['project_id']) & bool(inputs['job_name'])
return True
flags.register_multi_flags_validator(
['project_id', 'job_name'],
_mark_gcp_flags_as_required,
message=('--project_id and --job_name must be specified if --gcp set to '
'`true`.'))
flags.DEFINE_string('dataset_root_dir', '',
'Root directory to put all the training data.')
flags.DEFINE_string('csv_path', '', 'CSV file path.')
flags.DEFINE_string('checkpoint_dir', './checkpoint', 'Checkpoint directory.')
flags.DEFINE_string('debug_image_dir', './debug_output_image',
'Debug output image directory.')
flags.DEFINE_string('log_dir', './log', 'Log directory for tensorflow.')
flags.DEFINE_integer('batch_size', 4, 'Batch size, default 4.')
flags.DEFINE_integer('iterations', 90000000,
'Number of iterations, default 90000000.')
flags.DEFINE_integer(
'display_step', 20,
'Number of iterations between optimizer print info and save test image, default 20.'
)
flags.DEFINE_float('learning_rate', 0.0005, 'Learning rate, default 0.0005.')
flags.DEFINE_boolean('restore_checkpoint', True,
'If true, restore from latest checkpoint, default True.')
flags.DEFINE_boolean(
'just_set_batch_size_to_one', False,
'If true, just set batch size to one and exit(in order to call python freeze_model.py), default False.'
)
FLAGS = flags.FLAGS
if FLAGS.dataset_root_dir == '':
print('must set dataset_root_dir')
exit()
if FLAGS.csv_path == '':
print('must set csv_path')
exit()
if FLAGS.just_set_batch_size_to_one:
FLAGS.batch_size = 1
flags.DEFINE_string('feature_names', 'mean_audio',
'Features to be used, separated by ,.')
# 1024, 128
flags.DEFINE_string('feature_sizes', '128',
'Dimensions of features to be used, separated by ,.')
# Set by the memory limit. Larger values will reduce data passing times. For debug, use a small value, e.g., 1024.
flags.DEFINE_integer('batch_size', 2048, 'Size of batch processing.')
# For debug, use a single reader.
flags.DEFINE_integer('num_readers', 2,
'Number of readers to form a batch.')
# To find the best k in validate set, set it as True.
# After getting the best k, setting this as False when using train and validate set to re-train the model.
flags.DEFINE_boolean(
'is_tuning_hyper_para', False,
'Boolean variable indicating whether to perform hyper-parameter tuning.'
)
# Separated by ,.
flags.DEFINE_string('ks', '16', 'k nearest neighbors to tune.')
flags.DEFINE_integer('pred_k', 16,
'The k nearest neighbor to make predictions.')
flags.DEFINE_string(
'model_dir', '/tmp/ml-knn',
'The directory to which prior and posterior probabilities should be written.'
)
flags.DEFINE_boolean('is_train', True,
'Boolean variable to indicate training or test.')
def load_flags_config():
pred_flags = flags.FLAGS
# Feature Extractor flags
# Optional flags.
flags.DEFINE_string(
'extractor_model_dir', NLP_PREP_MODEL_PATH,
'Directory to store model files. It defaults to '
'src/data/video_tasks/preprocess_model')
# The following flags are set to match the YouTube-8M dataset format.
flags.DEFINE_integer('frames_per_second', 1,
'This many frames per second will be processed')
flags.DEFINE_string(
'labels_feature_key', 'labels',
'Labels will be written to context feature with this '
'key, as int64 list feature.')
flags.DEFINE_string(
'image_feature_key', 'rgb',
'Image features will be written to sequence feature with '
'this key, as bytes list feature, with only one entry, '
'containing quantized feature string.')
flags.DEFINE_string(
'video_file_key_feature_key', 'id',
'Input <video_file> will be written to context feature '
'with this key, as bytes list feature, with only one '
'entry, containing the file path of the video. This '
'can be used for debugging but not for training or eval.')
flags.DEFINE_boolean(
'insert_zero_audio_features', True,
'If set, inserts features with name "audio" to be 128-D '
'zero vectors. This allows you to use YouTube-8M '
'pre-trained model.')
# Predict Input
flags.DEFINE_string(
"train_dir", NLP_MODEL_PATH,
"The directory to load the model files from. We assume "
"that you have already run eval.py onto this, such that "
"inference_model.* files already exist.")
flags.DEFINE_string(
"vocabulary_file",
os.path.join(os.path.dirname(NLP_MODEL_PATH), "vocabulary.csv"),
"idxmap file")
flags.DEFINE_string(
"input_model_tgz", "",
"If given, must be path to a .tgz file that was written "
"by this binary using flag --output_model_tgz. In this "
"case, the .tgz file will be untarred to "
"--untar_model_dir and the model will be used for "
"inference.")
flags.DEFINE_string(
"untar_model_dir", "",
"If --input_model_tgz is given, then this directory will "
"be created and the contents of the .tgz file will be "
"untarred here.")
# Predict Output
# flags.DEFINE_string("output_file", "",
# "The file to save the predictions to.")
flags.DEFINE_string(
"output_model_tgz", "",
"If given, should be a filename with a .tgz extension, "
"the model graph and checkpoint will be bundled in this "
"gzip tar. This file can be uploaded to Kaggle for the "
"top 10 participants.")
flags.DEFINE_integer("top_k", 10,
"How many predictions to output per video.")
# Other flags.
flags.DEFINE_integer("batch_size", 1,
"How many examples to process per batch.")
flags.DEFINE_integer("num_readers", 1,
"How many threads to use for reading input files.")
flags.DEFINE_boolean("use_gpu", False,
"Use device of cpu or gpu. Default False(Use cpu)")
return pred_flags
"L2 regularization lambda (default: 0.0)")
# Training parameters
flags.DEFINE_integer("batch_size", 64, "Batch Size (default: 64)")
flags.DEFINE_integer("num_epochs", 50,
"Number of training epochs (default: 200)")
flags.DEFINE_integer(
"evaluate_every", 100,
"Evaluate model on dev set after this many steps (default: 100)")
flags.DEFINE_integer("checkpoint_every", 100,
"Save model after this many steps (default: 100)")
flags.DEFINE_integer("num_checkpoints", 5,
"Number of checkpoints to store (default: 5)")
# Misc Parameters
flags.DEFINE_boolean("allow_soft_placement", True,
"Allow device soft device placement")
flags.DEFINE_boolean("log_device_placement", False,
"Log placement of ops on devices")
FLAGS = tf.flags.FLAGS
FLAGS._parse_flags()
print("\nParameters:")
for attr, value in sorted(FLAGS.__flags.items()):
print("{}={}".format(attr.upper(), value))
print("")
# 3. train the model and test
with tf.Graph().as_default():
sess = tf.Session()
with sess.as_default():
cnn = TextCNN(sequence_length=200,
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
flags.DEFINE_integer(
"moe_num_mixtures", 4,
"The number of mixtures (excluding the dummy 'expert') used for MoeModel.")
flags.DEFINE_float(
"l2_penalty", 1e-8,
"the l2 penalty of classifier weights and bias"
)
flags.DEFINE_boolean("video_trainable", True, "video level model trainable parameter")
class LogisticModel(models.BaseModel):
"""Logistic model with L2 regularization."""
def create_model(self, model_input, vocab_size, l2_penalty=None, **unused_params):
"""Creates a logistic model.
Args:
model_input: 'batch' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes."""
l2_penalty = l2_penalty or FLAGS.l2_penalty
logits = slim.fully_connected(
