def run():
import importlib
from tensorflow import flags
flags.DEFINE_string("config_model", "configs.config_model",
"The model config.")
flags.DEFINE_string("config_data", "configs.config_giga",
"The dataset config.")
flags.DEFINE_float(
'decay_factor', 500.,
'The hyperparameter controling the speed of increasing '
'the probability of sampling from model')
flags.DEFINE_integer('n_samples', 10,
'number of samples for every target sentence')
flags.DEFINE_float('tau', 0.4, 'the temperature in RAML algorithm')
flags.DEFINE_string('output_dir', '.', 'where to keep training logs')
flags.DEFINE_bool('cpu', False, 'whether to use cpu')
flags.DEFINE_string('gpu', '0', 'use which gpu(s)')
flags.DEFINE_bool('debug', False,
'if debug, skip the training process after one step')
flags.DEFINE_bool('load', False, 'Whether to load existing checkpoint')
flags.DEFINE_string('script', 'python_test', 'which script to use')
flags.DEFINE_bool('infer', False, 'infer (use pretrained model)')
FLAGS = flags.FLAGS
# print(FLAGS.load)
module_name = FLAGS.script
# from python_test import main
module = importlib.import_module(module_name)
module.main(FLAGS)
def define_cli_args():
tf_flags.DEFINE_integer(consts.BATCH_SIZE, None, DESC)
tf_flags.DEFINE_string(consts.OPTIMIZER, None, DESC)
tf_flags.DEFINE_float(consts.LEARNING_RATE, None, DESC)
tf_flags.DEFINE_integer(consts.TRAIN_STEPS, None, DESC)
tf_flags.DEFINE_integer(consts.EVAL_STEPS_INTERVAL, None, DESC)
tf_flags.DEFINE_list(consts.EXCLUDED_KEYS, None, DESC)
def parse_args():
flags.DEFINE_string(
'train_path',
'/home/pierre/riken/riken/rnn/records/train_riken_data.tfrecords',
'Path to training records')
flags.DEFINE_string(
'val_path',
'/home/pierre/riken/riken/rnn/records/test_riken_data.tfrecords',
'Path to training records')
flags.DEFINE_integer('n_classes', 590, 'Number of classes')
flags.DEFINE_string('log_dir', './results', 'Path to training records')
flags.DEFINE_integer('epochs', 10,
'Number of epochs to train the model on')
flags.DEFINE_integer('batch_size', 128,
'Number of epochs to train the model on')
flags.DEFINE_float('lr', 1e-3, 'Maximum sequence lenght')
flags.DEFINE_integer('max_size', 500, 'max size')
flags.DEFINE_integer('lstm_size', 4, 'max size')
flags.DEFINE_float('margin', 1.0, 'Maximum sequence lenght')
return flags.FLAGS
flags.DEFINE_string('feature_sizes', '1024,128',
'Dimensions of features to be used, separated by ,.')
flags.DEFINE_integer('batch_size', 1024, 'Size of batch processing.')
flags.DEFINE_integer('num_readers', 2,
'Number of readers to form a batch.')
flags.DEFINE_bool('is_bootstrap', False,
'Boolean variable indicating using bootstrap or not.')
flags.DEFINE_boolean(
'init_with_linear_clf', True,
'Boolean variable indicating whether to init logistic regression with linear classifier.'
)
flags.DEFINE_float('init_learning_rate', 0.01,
'Float variable to indicate initial learning rate.')
flags.DEFINE_integer(
'decay_steps', NUM_TRAIN_EXAMPLES,
'Float variable indicating no. of examples to decay learning rate once.'
)
flags.DEFINE_float('decay_rate', 0.95,
'Float variable indicating how much to decay.')
flags.DEFINE_float('l2_reg_rate', 0.001, 'l2 regularization rate.')
flags.DEFINE_integer(
'train_epochs', 20,
'Training epochs, one epoch means passing all training data once.')
flags.DEFINE_integer("netvlad_hidden_size_audio", 64,
"Number of units in the NetVLAD audio hidden layer.")
flags.DEFINE_bool("netvlad_add_batch_norm", True,
"Adds batch normalization to the DBoF model.")
flags.DEFINE_integer("fv_cluster_size", 64,
"Number of units in the NetVLAD cluster layer.")
flags.DEFINE_integer("fv_hidden_size", 2048,
"Number of units in the NetVLAD hidden layer.")
flags.DEFINE_bool("fv_relu", True, "ReLU after the NetFV hidden layer.")
flags.DEFINE_bool("fv_couple_weights", True, "Coupling cluster weights or not")
flags.DEFINE_float("fv_coupling_factor", 0.01, "Coupling factor")
flags.DEFINE_string(
"dbof_pooling_method", "max",
"The pooling method used in the DBoF cluster layer. "
"Choices are 'average' and 'max'.")
flags.DEFINE_string(
"video_level_classifier_model", "MoeModel_CG",
"Some Frame-Level models can be decomposed into a "
"generalized pooling operation followed by a "
"classifier layer")
flags.DEFINE_integer("lstm_cells", 1024, "Number of LSTM cells.")
flags.DEFINE_integer("lstm_layers", 2, "Number of LSTM layers.")
flags.DEFINE_integer("lstm_cells_video", 1024, "Number of LSTM cells (video).")
flags.DEFINE_integer("lstm_cells_audio", 128, "Number of LSTM cells (audio).")
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from .util import *
from .hed_net import *
import os
from tensorflow import flags
flags.DEFINE_string('input_img', 'test_image/test2.jpg',
'Image path to run hed, must be jpg image.')
flags.DEFINE_string('checkpoint_dir', './checkpoint', 'Checkpoint directory.')
flags.DEFINE_string('output_img', 'test_image/result.jpg',
'Output image path.')
flags.DEFINE_float('output_threshold', 0.0, 'output threshold, default: 0.0')
FLAGS = flags.FLAGS
if not os.path.exists(FLAGS.input_img):
print('--input_img invalid')
exit()
if FLAGS.output_img == '':
print('--output_img invalid')
exit()
if __name__ == "__main__":
image_path_placeholder = tf.placeholder(tf.string)
feed_dict_to_use = {image_path_placeholder: FLAGS.input_img}
"""Contains model definitions."""
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
flags.DEFINE_integer(
"moe_num_mixtures", 2,
"The number of mixtures (excluding the dummy 'expert') used for MoeModel.")
flags.DEFINE_float(
"moe_l2", 1e-8,
"L2 penalty for MoeModel.")
flags.DEFINE_integer(
"moe_low_rank_gating", -1,
"Low rank gating for MoeModel.")
flags.DEFINE_bool(
"moe_prob_gating", False,
"Prob gating for MoeModel.")
flags.DEFINE_string(
"moe_prob_gating_input", "prob",
"input Prob gating for MoeModel.")
class LogisticModel(models.BaseModel):
"""Logistic model with L2 regularization."""
def create_model(self, model_input, vocab_size, l2_penalty=1e-8, **unused_params):
'still be run on the CPU if they have no GPU kernel.')
tf.app.flags.DEFINE_boolean('norm_input', True, 'norm input [-1:1].')
tf.app.flags.DEFINE_boolean('random_erase', False, 'random_erase input')
tf.app.flags.DEFINE_boolean('random_crop', False, 'random_crop')
tf.app.flags.DEFINE_boolean('random_rotate', False, 'random_rotate')
tf.app.flags.DEFINE_boolean('random_flip', True, 'random_flip')
tf.app.flags.DEFINE_integer('worker_replicas', 1, 'Number of worker+trainer '
'replicas.')
tf.app.flags.DEFINE_integer(
'ps_tasks', 0, 'Number of parameter server tasks. If None, does not use '
'a parameter server.')
tf.app.flags.DEFINE_float('moving_average_decay', None, 'moving_average_decay')
flags.DEFINE_float(
'last_layer_gradient_multiplier', 1,
'The gradient multiplier for last layers, which is used to '
'boost the gradient of last layers if the value > 1.')
tf.app.flags.DEFINE_integer('top_k', 10, '')
tf.app.flags.DEFINE_integer('num_images', 10, '')
FLAGS = tf.app.flags.FLAGS
os.environ["CUDA_VISIBLE_DEVICES"] = FLAGS.gpu_id
_FILE_PATTERN = '%s-*'
_SPLITS_TO_SIZES = {
'train': 125220,
}
_ITEMS_TO_DESCRIPTIONS = {
max_document_length = 200
x, vocabulary, vocab_size = tool.make_input(contents, max_document_length)
print('사전단어수 : %s' % (vocab_size))
y = tool.make_output(points, threshold=0.5)
# divide dataset into train/test set
x_train, x_test, y_train, y_test = tool.divide(x, y, train_prop=0.8)
# Model Hyperparameters
flags.DEFINE_integer("embedding_dim", 128,
"Dimensionality of embedded vector (default: 128)")
flags.DEFINE_string("filter_sizes", "3,4,5",
"Comma-separated filter sizes (default: '3,4,5')")
flags.DEFINE_integer("num_filters", 128,
"Number of filters per filter size (default: 128)")
flags.DEFINE_float("dropout_keep_prob", 0.5,
"Dropout keep probability (default: 0.5)")
flags.DEFINE_float("l2_reg_lambda", 0.1,
"L2 regularization lambda (default: 0.0)")
# Training parameters
flags.DEFINE_integer("batch_size", 64, "Batch Size (default: 64)")
flags.DEFINE_integer("num_epochs", 50,
"Number of training epochs (default: 200)")
flags.DEFINE_integer(
"evaluate_every", 100,
"Evaluate model on dev set after this many steps (default: 100)")
flags.DEFINE_integer("checkpoint_every", 100,
"Save model after this many steps (default: 100)")
flags.DEFINE_integer("num_checkpoints", 5,
"Number of checkpoints to store (default: 5)")
from tensorflow import flags
FLAGS = flags.FLAGS
# Image related flags
flags.DEFINE_integer("img_size_ori", 101, "Size of original images")
flags.DEFINE_integer("img_size_target", 128, "Size of input images")
# Training flags
flags.DEFINE_integer("num_epochs", 200, "Maximum of epochs")
flags.DEFINE_integer("batch_size", 32, "Number of batch size")
flags.DEFINE_integer("start_channels", 16,
"Number of the start channels of the newwork")
flags.DEFINE_integer("random_seed", 1234, "Number of random seed")
flags.DEFINE_float("dropout", 0.2, "The ratio of dropout")
flags.DEFINE_float("val_ratio", 0.1, "The ratio of train/validation split")
flags.DEFINE_bool("shuffle", True, "Shuffle the dataset every epoch")
flags.DEFINE_bool("augmentation", True, "Do data augmentation")
flags.DEFINE_bool("random_crop", True, "Do data random cropping")
# Network flags
flags.DEFINE_integer("encoder_type", 1,
"1 - simResnet, 2 - Resnet18, 3 - Resnet34")
flags.DEFINE_integer("decoder_type", 1,
"1 - simResnet, 2 - Resnet18, 3 - Resnet34")
# Optimizer flags
flags.DEFINE_string("optimizer", "Adam", "Type of used optimizer")
flags.DEFINE_float("min_lr", 0.00001, "Minimum of learning rate")
flags.DEFINE_float("factor", 0.4, "Value of recuding factor")
flags.DEFINE_integer("factor_patience", 4,
"D:/yt8m_data/models/video/predictions.csv", "结果存放的位置")
flags.DEFINE_string(
"output_model_tgz", "",
"If given, should be a filename with a .tgz extension, "
"the model graph and checkpoint will be bundled in this "
"gzip tar. This file can be uploaded to Kaggle for the "
"top 10 participants.")
flags.DEFINE_integer("top_k", 20,
"How many predictions to output per video.")
# Other flags.
flags.DEFINE_integer("batch_size", 256,
"How many examples to process per batch.")
flags.DEFINE_integer("num_readers", 6,
"How many threads to use for reading input files.")
flags.DEFINE_float("gpu_ratio", "0.7", "The ratio of gpu needed to use")
def format_lines(video_ids, predictions, top_k):
batch_size = len(video_ids)
for video_index in range(batch_size):
top_indices = numpy.argpartition(predictions[video_index],
-top_k)[-top_k:]
line = [(class_index, predictions[video_index][class_index])
for class_index in top_indices]
line = sorted(line, key=lambda p: -p[1])
yield video_ids[video_index].decode('utf-8') + "," + " ".join(
"%i %g" % (label, score) for (label, score) in line) + "\n"
def get_input_data_tensors(reader, data_pattern, batch_size, num_readers=1):
from tensorflow import flags
flags.DEFINE_integer("embedding_dim", 300,
"Dimensionality of character embedding (default: 128)")
flags.DEFINE_string("filter_sizes", "20",
"Comma-separated filter sizes (default: '3,4,5')")
flags.DEFINE_integer("num_filters", 64,
"Number of filters per filter size (default: 128)")
flags.DEFINE_float("dropout_keep_prob", 0.5,
"Dropout keep probability (default: 0.5)")
flags.DEFINE_float("l2_reg_lambda", 0.001,
"L2 regularizaion lambda (default: 0.0)")
flags.DEFINE_float("learning_rate", 0.001, "learn rate( default: 0.0)")
flags.DEFINE_integer("max_len_query", 4, "max document length of left input")
flags.DEFINE_integer("max_len_document", 50,
"max document length of right input")
flags.DEFINE_string("loss", "point_wise", "loss function (default:point_wise)")
flags.DEFINE_integer('extend_feature_dim', 10, 'overlap_feature_dim')
# Training parameters
flags.DEFINE_integer("batch_size", 20, "Batch Size (default: 64)")
flags.DEFINE_boolean("trainable", False,
"is embedding trainable? (default: False)")
flags.DEFINE_integer("num_epochs", 100,
"Number of training epochs (default: 200)")
flags.DEFINE_integer(
"evaluate_every", 500,
"Evaluate model on dev set after this many steps (default: 100)")
flags.DEFINE_integer("checkpoint_every", 500,
"Save model after this many steps (default: 100)")
flags.DEFINE_boolean('overlap_needed', True, "is overlap used")
flags.DEFINE_boolean('position_needed', False, 'is position used')
flags.DEFINE_string('eval_dir', '/tmp/deepvariant/',
'Directory where the results are saved to.')
flags.DEFINE_integer('max_evaluations', None,
'Max number of batches to evaluate')
flags.DEFINE_integer('max_examples', 64 * 1024 * 4,
'Maximum number of examples to evaluate.')
flags.DEFINE_string('model_name', 'inception_v3',
'The name of the model to use for predictions.')
flags.DEFINE_string('dataset_config_pbtxt', None,
'The path to the dataset config file.')
flags.DEFINE_float('moving_average_decay', 0.9999,
'The decay to use for the moving average.')
def select_variants_weights(variant_p_func, encoded_variants, name=None):
"""Creates a Tensor with 1.0 values anywhere variant_p_func returns True.
Creates a TensorFlow operation with tf.py_func that calls variant_p_func on
each Variant proto in encoded_variants (after decoding it), returning a 1.0
for each variant where variant_p_func returns True and 0.0 where it returns
value. For example, if is_snp returns True when a Variant is a SNP, then:
weights = select_variants_weights(is_snp, encoded_variants)
produces a weights tensor with 1.0 for each SNP in encoded_variants and 0.0
for any non-SNP variants.
flags.DEFINE_integer("file_size", 4096,
"Number of frames per batch for DBoF.")
flags.DEFINE_string(
"model", "YouShouldSpecifyAModel",
"Which architecture to use for the model. Models are defined "
"in models.py.")
# Other flags.
flags.DEFINE_integer("num_readers", 1,
"How many threads to use for reading input files.")
flags.DEFINE_integer("top_k", 20,
"How many predictions to output per video.")
flags.DEFINE_bool("dropout", False, "Whether to consider dropout")
flags.DEFINE_float(
"keep_prob", 1.0,
"probability to keep output (used in dropout, keep it unchanged in validationg and test)"
)
flags.DEFINE_float("noise_level", 0.0,
"standard deviation of noise (added to hidden nodes)")
def find_class_by_name(name, modules):
"""Searches the provided modules for the named class and returns it."""
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def get_input_data_tensors(reader, data_pattern, batch_size, num_readers=1):
"""Creates the section of the graph which reads the input data.
Args:
tf.logging.set_verbosity(tf.logging.INFO)
FLAGS = flags.FLAGS
flags.DEFINE_string("data_dir",'data/yelp-2013', 'directory containing train, val and test h5')
flags.DEFINE_string('checkpoint_dir','checkpoint','directory to save the best model saved as checkpoint_dir/model.chkpt')
flags.DEFINE_string('restore_checkpoint',None,'restore to a state; if None train from scratch default : ')
flags.DEFINE_boolean('gpu',True,'use --nogpu to disable gpu')
flags.DEFINE_integer('epoch',2,'epoch : default 2s')
flags.DEFINE_integer('batchsize',32,'batchsize: default 32')
#network parameters
flags.DEFINE_float('hidden_dim',100 ,'GRU hidden dimension : default 100')
#hyper params
flags.DEFINE_float('lr',1e-3,'Learning rate : default 1e-3')
class Attention():
def __init__(self, input, mask, scope ='A0'):
assert input.get_shape().as_list()[:-1] == mask.get_shape().as_list() and len(mask.get_shape().as_list()) == 2
_, steps, embed_dim = input.get_shape().as_list()
print steps, embed_dim
#trainable variales
self.u_w = tf.Variable(tf.truncated_normal([1, embed_dim], stddev=0.1), name='%s_query' %scope, dtype=tf.float32)
weights = tf.Variable(tf.truncated_normal([embed_dim, embed_dim], stddev=0.1), name='%s_Weight' %scope, dtype=tf.float32)
bias = tf.Variable(tf.truncated_normal([1, embed_dim], stddev=0.1), name='%s_bias' %scope, dtype=tf.float32)
#equations
"The directory to save the 24 top-level verticals in, used for 'calculate_loss_mix'"
)
flags.DEFINE_string(
"frequent_file", "./resources/labels_frequent.out",
"The directory to save the frequency of 4716 labels in, used only in early experiment."
)
flags.DEFINE_string("autoencoder_dir", "./resources/",
"The directory to save the autoencoder model layers in.")
flags.DEFINE_string(
"support_type", None,
"The support type for mix models, options are None, class, frequent and encoder,"
"used for 'calculate_loss_mix'.")
flags.DEFINE_string("loss_function", None,
"different loss funtions used in CrossEntropyLoss.")
flags.DEFINE_integer("encoder_layers", 2, "The number of autoencoder layers.")
flags.DEFINE_float("jsd_pi", 0.5, "wight used when loss function is loss_jsd.")
flags.DEFINE_float("threshold", 0.5, "used only in early experiment.")
class BaseLoss(object):
"""Inherit from this class when implementing new losses."""
def calculate_loss(self, unused_predictions, unused_labels,
**unused_params):
"""Calculates the average loss of the examples in a mini-batch.
Args:
unused_predictions: a 2-d tensor storing the prediction scores, in which
each row represents a sample in the mini-batch and each column
represents a class.
unused_labels: a 2-d tensor storing the labels, which has the same shape
as the unused_predictions. The labels must be in the range of 0 and 1.
'dbg_max_num_paths', 256,
'Maximum number of paths within a graph to consider for realignment. '
'Set max_num_paths to 0 to have unlimited number of paths.')
flags.DEFINE_integer('aln_match', 4,
'Match score (expected to be a non-negative score).')
flags.DEFINE_integer('aln_mismatch', 6,
'Mismatch score (expected to be a non-negative score).')
flags.DEFINE_integer(
'aln_gap_open', 8, 'Gap open score (expected to be a non-negative score). '
'Score for a gap of length g is -(gap_open + (g - 1) * gap_extend).')
flags.DEFINE_integer(
'aln_gap_extend', 1,
'Gap extend score (expected to be a non-negative score). '
'Score for a gap of length g is -(gap_open + (g - 1) * gap_extend).')
flags.DEFINE_integer('aln_k', 23, 'k-mer size used to index target sequence.')
flags.DEFINE_float('aln_error_rate', .01, 'Estimated sequencing error rate.')
flags.DEFINE_string(
'realigner_diagnostics', '',
'Root directory where the realigner should place diagnostic output (such as'
' a dump of the DeBruijn graph, and a log of metrics reflecting the graph '
'and realignment to the haplotypes). If empty, no diagnostics are output.'
)
flags.DEFINE_bool(
'emit_realigned_reads', False,
'If True, we will emit realigned reads if our realigner_diagnostics are '
'also enabled.')
# Margin added to the reference sequence for the aligner module.
_REF_ALIGN_MARGIN = 20
# ---------------------------------------------------------------------------
import tensorflow as tf
from tensorflow import flags
import datasets
flags.DEFINE_integer(
'epochs', -1, 'Epochs of training data, or -1 to continue indefinitely.')
flags.DEFINE_integer('image_height', 256, 'Image height in pixels.')
flags.DEFINE_integer('image_width', 256, 'Image width in pixels.')
flags.DEFINE_integer('sequence_length', 10,
'Sequence length for each example.')
flags.DEFINE_integer('min_stride', 3, 'Minimum stride for sequence.')
flags.DEFINE_integer('max_stride', 10, 'Maximum stride for sequence.')
flags.DEFINE_float('augment_min_scale', 1.0,
'Minimum scale for data augmentation.')
flags.DEFINE_float('augment_max_scale', 1.15,
'Maximum scale for data augmentation.')
flags.DEFINE_integer('batch_size', 8, 'The size of a sample batch.')
FLAGS = flags.FLAGS
# The lambdas in this file are only long because of clear argument names.
# pylint: disable=g-long-lambda
class Loader(object):
"""Process video sequences into a dataset for use in training or testing."""
def __init__(
self,
"model", "LogisticModel",
"Which architecture to use for the model. Models are defined "
"in models.py.")
flags.DEFINE_bool(
"start_new_model", False,
"If set, this will not resume from a checkpoint and will instead create a"
" new model instance.")
# Training flags.
flags.DEFINE_integer(
"batch_size", 1024,
"How many examples to process per batch for training.")
flags.DEFINE_string("label_loss", "CrossEntropyLoss",
"Which loss function to use for training the model.")
flags.DEFINE_float(
"regularization_penalty", 1e-3,
"How much weight to give to the regularization loss (the label loss has "
"a weight of 1).")
flags.DEFINE_float("base_learning_rate", 0.01,
"Which learning rate to start with.")
# Other flags.
flags.DEFINE_integer("num_readers", 8,
"How many threads to use for reading input files.")
flags.DEFINE_string("master", "", "TensorFlow master to use.")
flags.DEFINE_integer(
"task", 0, "Task id of the replica running the training."
" 0 implies chief Supervisor."
"")
flags.DEFINE_integer(
"ps_tasks", 0, """Number of tasks in the ps job.
If 0 no ps job is used.""")
from tensorflow import flags
from tensorflow import logging
from google.protobuf import text_format
from nltk.stem.porter import PorterStemmer
from sklearn.metrics import average_precision_score
import eval_utils
from protos import pipeline_pb2
from protos import baseline_model_pb2
from reader import ads_examples
from models import builder
from utils import train_utils
from sklearn.metrics import accuracy_score
flags.DEFINE_float('per_process_gpu_memory_fraction', 0.5,
'GPU usage limitation.')
flags.DEFINE_string('pipeline_proto', '', 'Path to the pipeline proto file.')
flags.DEFINE_string(
'train_log_dir', '',
'The directory where the graph and checkpoints are saved.')
flags.DEFINE_string(
'eval_log_dir', '',
'The directory where the graph and checkpoints are saved.')
flags.DEFINE_integer('number_of_steps', 1000000, 'Maximum number of steps.')
flags.DEFINE_string('saved_ckpts_dir', '',
'The directory to backup checkpoint')
"""Contains model definitions."""
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
flags.DEFINE_integer(
"moe_num_mixtures", 2,#2
"The number of mixtures (excluding the dummy 'expert') used for MoeModel.")
flags.DEFINE_float(
"moe_l2", 1e-6, #1e-8
"L2 penalty for MoeModel.")
flags.DEFINE_integer(
"moe_low_rank_gating", -1,
"Low rank gating for MoeModel.")
flags.DEFINE_bool(
"moe_prob_gating", True, #False
"Prob gating for MoeModel.")
flags.DEFINE_string(
"moe_prob_gating_input", "prob",
"input Prob gating for MoeModel.")
flags.DEFINE_bool(
"gating_remove_diag", False,
"input Prob gating for MoeModel.")
flags.DEFINE_float(
"l2_penalty", 1e-8,
from preprocessing import preprocess
FLAGS = flags.FLAGS
flags.DEFINE_string('logging_verbosity', 'INFO', 'Level of logging verbosity '
'(e.g. `INFO` `DEBUG`).')
flags.DEFINE_string('project_id', None, 'GCP project id.')
flags.DEFINE_string('job_name', None, 'Dataflow job name.')
flags.DEFINE_integer('num_workers', None, 'Number of dataflow workers.')
flags.DEFINE_string('worker_machine_type', None, 'Machine types.')
flags.DEFINE_string('region', None, 'GCP region to use.')
flags.DEFINE_string('input_dir', None,
'Path of the directory containing input data.')
flags.DEFINE_string('output_dir', None, 'Path to write output data to.')
flags.DEFINE_float(
'train_size', 0.7, 'Percentage of input data to use for'
' training vs validation.')
flags.DEFINE_boolean('gcp', False, 'Runs on GCP or locally.')
flags.mark_flag_as_required('input_dir')
flags.mark_flag_as_required('output_dir')
def _mark_gcp_flags_as_required(inputs):
if FLAGS.gcp:
return bool(inputs['project_id']) & bool(inputs['job_name'])
return True
flags.register_multi_flags_validator(
['project_id', 'job_name'],
_mark_gcp_flags_as_required,
flags.DEFINE_bool("use_lstm_output", False,
"Use LSTM output instead of state for classification")
flags.DEFINE_string("pooling_method", "average",
"The type of pooling of frame level features to use.")
flags.DEFINE_bool("use_attention", False, "Apple attention to RNN models")
flags.DEFINE_integer("attention_len", 10, "The size of the attention window")
flags.DEFINE_integer("rhn_cells", 512, "Number of RHN cells.")
flags.DEFINE_integer("rhn_layers", 1, "Number of RHN layers.")
flags.DEFINE_integer("rhn_depth", 5, "Depth of the RHN cells")
flags.DEFINE_integer("num_filters", 32, "Number of 1D convolution filters")
flags.DEFINE_integer("filter_size", 5, "size of the 1D convolution filters")
flags.DEFINE_integer("time_skip", 2, "Number of time skips in each layer")
flags.DEFINE_float("dropout_keep_prob", 0.9,
"Dropout keep prob for layer norm LSTM")
flags.DEFINE_bool("use_residuals", False,
"Whether to use residual lstm wrapper")
class FrameLevelLogisticModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames,
**unused_params):
"""Creates a model which uses a logistic classifier over the average of the
frame-level features.
This class is intended to be an example for implementors of frame level
models. If you want to train a model over averaged features it is more
efficient to average them beforehand rather than on the fly.
Args:
# limitations under the License.
"""Contains model definitions."""
import math
import models
import tensorflow as tf
import utils
import numpy as np
from hickle import load
from tensorflow import flags
import tensorflow.contrib.slim as slim
import tensorflow.contrib.layers as layers
FLAGS = flags.FLAGS
flags.DEFINE_integer("hidden_size", 4096, "NN hidden size")
flags.DEFINE_float("sharpening", 1.0,
"Corelation matrix sharpening parameters")
flags.DEFINE_float("corelation_gamma", 0.1, "corelation matrix strength")
flags.DEFINE_integer(
"moe_num_mixtures", 2,
"The number of mixtures (excluding the dummy 'expert') used for MoeModel.")
flags.DEFINE_integer("dim_gate", 2048, "2-layer moe model dimension")
flags.DEFINE_integer("bottleneck_size", 2048, "bottleneck feature dimension")
class LogisticModel(models.BaseModel):
"""Logistic model with L2 regularization."""
def create_model(self,
model_input,
vocab_size,
# Input data
flags.DEFINE_string('Vocab_Processor_PATH',
'./Ch01_Data_load/data/VocabularyProcessor',
'VocabularyProcessor object file path')
flags.DEFINE_integer('VOCAB_SIZE', 72844, 'The number of terms in vocabulary')
flags.DEFINE_integer('EMBEDDING_SIZE', 256, 'Dimension of embedded terms')
flags.DEFINE_integer('MAXLEN', 22, 'max length of document')
# Class
flags.DEFINE_integer('NUM_OF_CLASS', 2, 'positive, negative')
# Parameter
flags.DEFINE_string('RNN_CELL', 'LSTM', 'RNN cell default LSTM')
flags.DEFINE_integer('RNN_HIDDEN_DIMENSION', 256, 'RNN hidden dimension')
flags.DEFINE_integer('FC_HIDDEN_DIMENSION', 256, 'FC hidden dimension')
flags.DEFINE_float('Dropout_Rate1', 0.7, 'Dropout_Rate1')
flags.DEFINE_float('Dropout_Rate2', 0.7, 'Dropout_Rate2')
flags.DEFINE_integer('N_LAYERS', 3, 'The number of layers')
# Save
flags.DEFINE_string('WRITER', 'Text_RNN_word', 'saver name')
flags.DEFINE_boolean('WRITER_generate', True, 'saver generate')
# Train
flags.DEFINE_integer('BATCH_SIZE', 128, 'batch size')
flags.DEFINE_integer('TEST_BATCH', 128, 'test batch size')
flags.DEFINE_integer('NUM_OF_EPOCH', 3, 'number of epoch')
flags.DEFINE_float('lr_value', 0.01, 'initial learning rate')
flags.DEFINE_float('lr_decay', 0.7, 'learning rate decay')
flags.DEFINE_multi_integer('Check_Loss', [5] * 20, 'loss decay')
"in models.py.")
flags.DEFINE_bool("multitask", False,
"Whether to consider support_predictions")
flags.DEFINE_bool(
"start_new_model", False,
"If set, this will not resume from a checkpoint and will instead create a"
" new model instance.")
# Training flags.
flags.DEFINE_integer(
"batch_size", 32,
"How many examples to process per batch for training.")
flags.DEFINE_string("label_loss", "CrossEntropyLoss",
"Which loss function to use for training the model.")
flags.DEFINE_float(
"regularization_penalty", 1,
"How much weight to give to the regularization loss (the label loss has "
"a weight of 1).")
flags.DEFINE_float("base_learning_rate", 0.01,
"Which learning rate to start with.")
flags.DEFINE_float(
"learning_rate_decay", 0.95,
"Learning rate decay factor to be applied every "
"learning_rate_decay_examples.")
flags.DEFINE_float(
"learning_rate_decay_examples", 1000000,
"Multiply current learning rate by learning_rate_decay "
"every learning_rate_decay_examples.")
flags.DEFINE_integer(
"num_epochs", 10, "How many passes to make over the dataset before "
"halting training.")
flags.DEFINE_float("keep_checkpoint_every_n_hours", 0.1,
flags.DEFINE_string(
'input_videos_csv', None,
'CSV file with lines "<video_file>,<labels>", where '
'<video_file> must be a path of a video and <labels> '
'must be an integer list joined with semi-colon ";"')
flags.DEFINE_string('frame_iterator', 'VideoCaptureFrameIterator',
'The name of the frame iterator.')
flags.DEFINE_string('clip_iterator', 'DenseOpticalFlowClipIterator',
'The name of the clip iterator.')
flags.DEFINE_integer("top_k", 10,
"How many predictions to output per video clip.")
flags.DEFINE_float('distance_threshold', 0.8,
'The threshold used to split video clips.')
flags.DEFINE_string("vocab_file", "",
"Path to the vocabulary CSV file from youtube-8m.")
flags.DEFINE_string("export_path", "",
"The path used to store prediction results.")
flags.DEFINE_string("tmp_dir", "",
"The directory used to store visualization results.")
flags.DEFINE_string("partition", "0/1", "")
flags.DEFINE_string("task", "visualize", "")
import tensorflow as tf
from tensorflow import flags
FLAGS = flags.FLAGS
flags.DEFINE_float("alpha", "0.5", "")
class BaseLoss(object):
def calculate_loss(self, unused_predictions, unused_labels,
**unused_params):
raise NotImplementedError()
class CrossEntropyLoss(BaseLoss):
def calculate_loss(self, predictions, labels, **unused_params):
with tf.name_scope("loss_xent"):
epsilon = 10e-6
alpha = FLAGS.alpha
float_labels = tf.cast(labels, tf.float32)
cross_entropy_loss = 2*(\
alpha * float_labels * tf.log(predictions+epsilon) +
(1-alpha)*(1-float_labels) * tf.log(1-predictions+epsilon))
cross_entropy_loss = tf.negative(cross_entropy_loss)
return tf.reduce_mean(tf.reduce_sum(cross_entropy_loss, 1))
flags.DEFINE_bool(
"start_new_model", False,
"If set, this will not resume from a checkpoint and will instead create a"
" new model instance.")
# Training flags.
flags.DEFINE_integer(
"num_gpu", 1, "The maximum number of GPU devices to use for training. "
"Flag only applies if GPUs are installed")
flags.DEFINE_integer(
"batch_size", 1024,
"How many examples to process per batch for training.")
flags.DEFINE_string("label_loss", "CrossEntropyLoss",
"Which loss function to use for training the model.")
flags.DEFINE_float(
"regularization_penalty", 1.0,
"How much weight to give to the regularization loss (the label loss has "
"a weight of 1).")
flags.DEFINE_float("base_learning_rate", 0.01,
"Which learning rate to start with.")
flags.DEFINE_float(
"learning_rate_decay", 0.95,
"Learning rate decay factor to be applied every "
"learning_rate_decay_examples.")
flags.DEFINE_float(
"learning_rate_decay_examples", 4000000,
"Multiply current learning rate by learning_rate_decay "
"every learning_rate_decay_examples.")
flags.DEFINE_integer(
"num_epochs", 100, "How many passes to make over the dataset before "
"halting training.")
flags.DEFINE_integer(
flags.DEFINE_integer(
'partition_size', 1000,
'The maximum number of basepairs we will allow in a region before splitting'
'it into multiple smaller subregions.')
flags.DEFINE_integer(
'max_reads_per_partition', 1500,
'The maximum number of reads per partition that we consider before '
'following processing such as sampling and realigner.')
flags.DEFINE_string(
'multi_allelic_mode', '',
'How to handle multi-allelic candidate variants. For DEBUGGING')
flags.DEFINE_bool('realign_reads', True,
'If True, locally realign reads before calling variants.')
flags.DEFINE_float(
'downsample_fraction', NO_DOWNSAMPLING,
'If not ' + str(NO_DOWNSAMPLING) + ' must be a value between 0.0 and 1.0. '
'Reads will be kept (randomly) with a probability of downsample_fraction '
'from the input BAM. This argument makes it easy to create examples as '
'though the input BAM had less coverage.')
flags.DEFINE_string(
'sample_name', '', 'Sample name to use for our sample_name in the output '
'Variant/DeepVariantCall protos. If not specified, will be inferred from '
'the header information from --reads.')
flags.DEFINE_string('hts_logging_level',
hts_verbose.htsLogLevel.HTS_LOG_WARNING.name,
'Sets the htslib logging threshold.')
flags.DEFINE_integer(
'hts_block_size', _DEFAULT_HTS_BLOCK_SIZE,
'Sets the htslib block size. Zero or negative uses default htslib setting; '
'larger values (e.g. 1M) may be beneficial for using remote files. '
'Currently only applies to SAM/BAM reading.')
flags.DEFINE_integer('vsc_min_count_snps', 2,
