def run():
import importlib
from tensorflow import flags
flags.DEFINE_string("config_model", "configs.config_model",
"The model config.")
flags.DEFINE_string("config_data", "configs.config_giga",
"The dataset config.")
flags.DEFINE_float(
'decay_factor', 500.,
'The hyperparameter controling the speed of increasing '
'the probability of sampling from model')
flags.DEFINE_integer('n_samples', 10,
'number of samples for every target sentence')
flags.DEFINE_float('tau', 0.4, 'the temperature in RAML algorithm')
flags.DEFINE_string('output_dir', '.', 'where to keep training logs')
flags.DEFINE_bool('cpu', False, 'whether to use cpu')
flags.DEFINE_string('gpu', '0', 'use which gpu(s)')
flags.DEFINE_bool('debug', False,
'if debug, skip the training process after one step')
flags.DEFINE_bool('load', False, 'Whether to load existing checkpoint')
flags.DEFINE_string('script', 'python_test', 'which script to use')
flags.DEFINE_bool('infer', False, 'infer (use pretrained model)')
FLAGS = flags.FLAGS
# print(FLAGS.load)
module_name = FLAGS.script
# from python_test import main
module = importlib.import_module(module_name)
module.main(FLAGS)
def setFlags(template_staging_bucket, library_staging_bucket, project,
candidate_name, include_hidden):
flags.DEFINE_string(
'template_prod_bucket', '', 'Bucket in which to store '
'production templates. Should begin with "gs://".')
flags.DEFINE_string(
'template_staging_bucket', template_staging_bucket,
'Bucket in which to store '
'staged templates. Should begin with "gs://".')
flags.DEFINE_string(
'library_staging_bucket', library_staging_bucket,
'Bucket in which to store staged jar files.'
' Should begin with "gs://".')
flags.DEFINE_string(
'project', project,
'GCP Project in which to create the template pipelines.')
flags.DEFINE_string('candidate_name', candidate_name,
'Name of release candidate.')
flags.DEFINE_string('release_name', '',
'Name of release -- used to version templates.')
flags.DEFINE_string('metadata_dir_override', '',
'Directory for metadata files')
flags.DEFINE_bool('include_hidden', include_hidden,
'Include hidden templates in the release.')
def init():
# Input
flags.DEFINE_string(
"train_dir", "/home/ubuntu/EE542_final_project/Cloud-Enabled-Smart-Speaker/audio_analyzer/classifier/models/frame/sample_model", "The directory to load the model files from. We assume "
"that you have already run eval.py onto this, such that "
"inference_model.* files already exist.")
flags.DEFINE_string(
"input_data_pattern", "/home/ubuntu/EE542_final_project/Cloud-Enabled-Smart-Speaker/audio_analyzer/extracted_features.tfrecord",
"File glob defining the evaluation dataset in tensorflow.SequenceExample "
"format. The SequenceExamples are expected to have an 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
flags.DEFINE_string(
"input_model_tgz", "",
"If given, must be path to a .tgz file that was written "
"by this binary using flag --output_model_tgz. In this "
"case, the .tgz file will be untarred to "
"--untar_model_dir and the model will be used for "
"inference.")
flags.DEFINE_string(
"untar_model_dir", "/tmp/yt8m-model",
"If --input_model_tgz is given, then this directory will "
"be created and the contents of the .tgz file will be "
"untarred here.")
flags.DEFINE_bool(
"segment_labels", False,
"If set, then --input_data_pattern must be frame-level features (but with"
" segment_labels). Otherwise, --input_data_pattern must be aggregated "
"video-level features. The model must also be set appropriately (i.e. to "
"read 3D batches VS 4D batches.")
flags.DEFINE_integer("segment_max_pred", 100000,
"Limit total number of segment outputs per entity.")
flags.DEFINE_string(
"segment_label_ids_file",
"https://raw.githubusercontent.com/google/youtube-8m/master/segment_label_ids.csv",
"The file that contains the segment label ids.")
# Output
flags.DEFINE_string("output_file", "/home/ubuntu/EE542_final_project/Cloud-Enabled-Smart-Speaker/audio_analyzer/solution.csv", "The file to save the predictions to.")
flags.DEFINE_string(
"output_model_tgz", "",
"If given, should be a filename with a .tgz extension, "
"the model graph and checkpoint will be bundled in this "
"gzip tar. This file can be uploaded to Kaggle for the "
"top 10 participants.")
flags.DEFINE_integer("top_k", 20, "How many predictions to output per video.")
# Other flags.
flags.DEFINE_integer("batch_size", 512,
"How many examples to process per batch.")
flags.DEFINE_integer("num_readers", 1,
"How many threads to use for reading input files.")
"The directory to save the model files in.")
flags.DEFINE_string(
"train_data_pattern", "",
"File glob for the training dataset. If the files refer to Frame Level "
"features (i.e. tensorflow.SequenceExample), then set --reader_type "
"format. The (Sequence)Examples are expected to have 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
flags.DEFINE_string("feature_names", "mean_rgb", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "1024",
"Length of the feature vectors.")
# Model flags.
flags.DEFINE_bool(
"frame_features", False,
"If set, then --train_data_pattern must be frame-level features. "
"Otherwise, --train_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_string(
"model", "LogisticModel",
"Which architecture to use for the model. Models are defined "
"in models.py.")
flags.DEFINE_bool(
"start_new_model", False,
"If set, this will not resume from a checkpoint and will instead create a"
" new model instance.")
# Training flags.
flags.DEFINE_integer(
"num_gpu", 1, "The maximum number of GPU devices to use for training. "
"Flag only applies if GPUs are installed")
# limitations under the License.
"""Contains model definitions."""
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
flags.DEFINE_integer(
"moe_num_mixtures", 2,
"The number of mixtures (excluding the dummy 'expert') used for MoeModel.")
flags.DEFINE_bool("is_training", True, "Is training or not")
class LogisticModel(models.BaseModel):
"""Logistic model with L2 regularization."""
def create_model(self,
model_input,
vocab_size,
l2_penalty=1e-8,
**unused_params):
"""Creates a logistic model.
Args:
model_input: 'batch' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
import math
import models
import video_level_models
import tensorflow as tf
import model_utils as utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
flags.DEFINE_integer("iterations", 30,
"Number of frames per batch for DBoF.")
flags.DEFINE_bool("dbof_add_batch_norm", True,
"Adds batch normalization to the DBoF model.")
flags.DEFINE_bool(
"sample_random_frames", True,
"If true samples random frames (for frame level models). If false, a random"
"sequence of frames is sampled instead.")
flags.DEFINE_string("video_level_classifier_model", "MoeModel",
"Some Frame-Level models can be decomposed into a "
"generalized pooling operation followed by a "
"classifier layer")
flags.DEFINE_integer("lstm_cells", 1024, "Number of LSTM cells.")
flags.DEFINE_integer("lstm_layers", 2, "Number of LSTM layers.")
class FrameLevelLogisticModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a logistic classifier over the average of the
frame-level features.
from __future__ import division
from __future__ import print_function
import copy
import itertools
from tensorflow import flags
import numpy as np
from absl import logging
from third_party.nucleus.util import genomics_math
from third_party.nucleus.util import variant_utils
FLAGS = flags.FLAGS
flags.DEFINE_bool(
'disable_haplotype_resolution', False,
'If True, makes `maybe_resolve_conflicting_variants` a no-op.')
# The maximum number of overlapping variants to try to resolve into compatible
# haplotypes. This corresponds to generating 3^12 (= 531,441) possible variant
# configurations for diploid individuals.
_MAX_OVERLAPPING_VARIANTS_TO_RESOLVE = 12
def maybe_resolve_conflicting_variants(sorted_variants):
"""Yields Variant protos in sorted order after fixing conflicting haplotypes.
The input is an iterable of Variants in chromosome and position sorted order,
with potential incompatibilies as described in this module's docstring. This
function tries to resolve variants into valid haplotypes, though is not
guaranteed to do so if the variant composition is not amenable to this or it
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
flags.DEFINE_integer(
"moe_num_mixtures", 2,
"The number of mixtures (excluding the dummy 'expert') used for MoeModel.")
flags.DEFINE_float(
"moe_l2", 1e-8,
"L2 penalty for MoeModel.")
flags.DEFINE_integer(
"moe_low_rank_gating", -1,
"Low rank gating for MoeModel.")
flags.DEFINE_bool(
"moe_prob_gating", False,
"Prob gating for MoeModel.")
flags.DEFINE_string(
"moe_prob_gating_input", "prob",
"input Prob gating for MoeModel.")
class LogisticModel(models.BaseModel):
"""Logistic model with L2 regularization."""
def create_model(self, model_input, vocab_size, l2_penalty=1e-8, **unused_params):
"""Creates a logistic model.
Args:
model_input: 'batch' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
flags.DEFINE_integer(
"moe_num_mixtures", 2,#2
"The number of mixtures (excluding the dummy 'expert') used for MoeModel.")
flags.DEFINE_float(
"moe_l2", 1e-6, #1e-8
"L2 penalty for MoeModel.")
flags.DEFINE_integer(
"moe_low_rank_gating", -1,
"Low rank gating for MoeModel.")
flags.DEFINE_bool(
"moe_prob_gating", True, #False
"Prob gating for MoeModel.")
flags.DEFINE_string(
"moe_prob_gating_input", "prob",
"input Prob gating for MoeModel.")
flags.DEFINE_bool(
"gating_remove_diag", False,
"input Prob gating for MoeModel.")
flags.DEFINE_float(
"l2_penalty", 1e-8,
"the l2 penalty of classifier weights and bias"
)
class LogisticModel(models.BaseModel):
"""Logistic model with L2 regularization."""
def create_model(self, model_input, vocab_size, l2_penalty=None, **unused_params):
FLAGS = flags.FLAGS
if __name__ == "__main__":
# Dataset flags.
flags.DEFINE_string("train_dir", "/tmp/mnist_model/",
"The directory to save the model files in.")
flags.DEFINE_string("train_data_pattern", "",
"File glob for the training dataset.")
flags.DEFINE_string(
"model", "LogisticModel",
"Which architecture to use for the model. Models are defined "
"in models.py.")
flags.DEFINE_bool(
"start_new_model", False,
"If set, this will not resume from a checkpoint and will instead create a"
" new model instance.")
# Training flags.
flags.DEFINE_integer(
"batch_size", 512,
"How many examples to process per batch for training.")
flags.DEFINE_string("label_loss", "CrossEntropyLoss",
"Which loss function to use for training the model.")
flags.DEFINE_float(
"regularization_penalty", 1.0,
"How much weight to give to the regularization loss (the label loss has "
"a weight of 1).")
flags.DEFINE_float("base_learning_rate", 0.01,
"Which learning rate to start with.")
"D:/yt8m_data/video_level/test/test*.tfrecord",
"测试集存放路径，用于生成提交到官方的结果")
flags.DEFINE_string(
"input_model_tgz", "",
"If given, must be path to a .tgz file that was written "
"by this binary using flag --output_model_tgz. In this "
"case, the .tgz file will be untarred to "
"--untar_model_dir and the model will be used for "
"inference.")
flags.DEFINE_string(
"untar_model_dir", "/tmp/yt8m-model",
"If --input_model_tgz is given, then this directory will "
"be created and the contents of the .tgz file will be "
"untarred here.")
flags.DEFINE_bool(
"multitask_loss", True, "如果训练使用了MultiTaskCrossEntropyLoss，这里就是True"
"如果使用的是其他的函数，这里就是False"
"这个选项非常重要！！！！！！！！！！")
# Output
flags.DEFINE_string("output_file",
"D:/yt8m_data/models/video/predictions.csv", "结果存放的位置")
flags.DEFINE_string(
"output_model_tgz", "",
"If given, should be a filename with a .tgz extension, "
"the model graph and checkpoint will be bundled in this "
"gzip tar. This file can be uploaded to Kaggle for the "
"top 10 participants.")
flags.DEFINE_integer("top_k", 20,
"How many predictions to output per video.")
# Other flags.
flags.DEFINE_integer("batch_size", 256,
flags.DEFINE_string(
"binary_tfrecord_data_dir", "",
"Output directory to store binary labeled data in tfrecord record format. "
"Each file will contain 256 tfrecords.")
flags.DEFINE_string("feature_names", "audio_embedding",
"Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "128",
"Length of the feature vectors.")
flags.DEFINE_integer("num_classes", 527, "Number of classes in dataset.")
flags.DEFINE_integer("water_samples", -1,
"Number of water samples in dataset.")
flags.DEFINE_integer("non_water_samples", -1,
"Number of non-water samples in dataset.")
flags.DEFINE_bool(
"create_water_tfrecords", False, "Boolean to specify if water sample "
"tfrecord files need to be created or not.")
flags.DEFINE_bool(
"create_non_water_tfrecords", False,
"Boolean to specify if non-water sample tfrecord files "
"need to be created or not")
flags.DEFINE_integer("start_file_index", 145,
"Starting file suffix for the tfrecord file.")
flags.DEFINE_integer("end_file_index", 106,
"Ending file suffix for the tfrecord file.")
main(FLAGS.input_tfrecord_data_path, FLAGS.binary_tfrecord_data_dir,
FLAGS.feature_names, FLAGS.feature_sizes, FLAGS.num_classes,
FLAGS.water_samples, FLAGS.non_water_samples,
FLAGS.create_water_tfrecords, FLAGS.create_non_water_tfrecords,
FLAGS.start_file_index, FLAGS.end_file_index)
# @ Author: Dagui Chen/ Xin Su # @ Email: [email protected]/[email protected] # @ Date: 2017-05-08/2017-05-19 # ===================================== from tensorflow import flags FLAGS = flags.FLAGS flags.DEFINE_string("workspace", "/home/dagui/.keras/datasets/", "The workspace dir path") flags.DEFINE_string("feature_path", "image_vgg19_block5_pool_feature.h5", "The path of feature file") flags.DEFINE_string("caption_file_path", "{}.txt", "The path of Caption file") flags.DEFINE_bool("ifpool", False, "If use the pooling feature.") class Config(object): def __init__(self): self.workspace = FLAGS.workspace self.feature_path = self.workspace + FLAGS.feature_path self.caption = self.workspace + FLAGS.caption_file_path print FLAGS.ifpool self.ifpool = FLAGS.ifpool # Default feature is fc / if use the pooling feature, it needs to be True
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains a collection of models which operate on variable-length sequences.
"""
from tensorflow import flags
FLAGS = flags.FLAGS
flags.DEFINE_integer("iterations", 30, "Number of frames per batch for DBoF.")
flags.DEFINE_bool("dbof_add_batch_norm", True,
"Adds batch normalization to the DBoF model.")
flags.DEFINE_bool(
"sample_random_frames", True,
"If true samples random frames (for frame level models). If false, a random"
"sequence of frames is sampled instead.")
flags.DEFINE_integer("dbof_cluster_size", 8192,
"Number of units in the DBoF cluster layer.")
flags.DEFINE_integer("dbof_hidden_size", 1024,
"Number of units in the DBoF hidden layer.")
flags.DEFINE_string(
"dbof_pooling_method", "max",
"The pooling method used in the DBoF cluster layer. "
"Choices are 'average' and 'max'.")
flags.DEFINE_string(
"video_level_classifier_model", "MoeModel",
"Some Frame-Level models can be decomposed into a "
"Which discriminator odel to use: see models.py")
flags.DEFINE_integer("batch_size", 64,
"How many examples to process per batch.")
flags.DEFINE_string("label_loss", "CrossEntropyLoss",
"Loss computed on validation data")
# Other flags.
flags.DEFINE_integer("num_readers", 8,
"How many threads to use for reading input files.")
flags.DEFINE_boolean("run_once", True, "Whether to run eval only once.")
flags.DEFINE_bool(
"use_mnist", False,
"Whether to use MNIST dataset for easy validation of "
"GAN model.")
def find_class_by_name(name, modules):
"""Searches the provided modules for the named class and returns it."""
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def get_input_evaluation_tensors(reader,
data_pattern,
batch_size=64,
num_readers=1):
"""Creates the section of the graph which reads the evaluation data.
'aln_gap_open', 8, 'Gap open score (expected to be a non-negative score). '
'Score for a gap of length g is -(gap_open + (g - 1) * gap_extend).')
flags.DEFINE_integer(
'aln_gap_extend', 1,
'Gap extend score (expected to be a non-negative score). '
'Score for a gap of length g is -(gap_open + (g - 1) * gap_extend).')
flags.DEFINE_integer('aln_k', 23, 'k-mer size used to index target sequence.')
flags.DEFINE_float('aln_error_rate', .01, 'Estimated sequencing error rate.')
flags.DEFINE_string(
'realigner_diagnostics', '',
'Root directory where the realigner should place diagnostic output (such as'
' a dump of the DeBruijn graph, and a log of metrics reflecting the graph '
'and realignment to the haplotypes). If empty, no diagnostics are output.'
)
flags.DEFINE_bool(
'emit_realigned_reads', False,
'If True, we will emit realigned reads if our realigner_diagnostics are '
'also enabled.')
# Margin added to the reference sequence for the aligner module.
_REF_ALIGN_MARGIN = 20
# ---------------------------------------------------------------------------
# Set configuration settings.
# ---------------------------------------------------------------------------
def realigner_config(flags_obj):
"""Creates a RealignerOptions proto based on input and default settings.
Args:
flags_obj: configuration FLAGS.
"The directory to save the model files in.")
flags.DEFINE_string(
"train_data_pattern", "",
"File glob for the training dataset. If the files refer to Frame Level "
"features (i.e. tensorflow.SequenceExample), then set --reader_type "
"format. The (Sequence)Examples are expected to have 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
flags.DEFINE_string("feature_names", "mean_rgb", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "1024",
"Length of the feature vectors.")
# Model flags.
flags.DEFINE_bool(
"frame_features", False,
"If set, then --train_data_pattern must be frame-level features. "
"Otherwise, --train_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_string(
"model", "LogisticModel",
"Which architecture to use for the model. Models are defined "
"in models.py.")
flags.DEFINE_bool(
"start_new_model", False,
"If set, this will not resume from a checkpoint and will instead create a"
" new model instance.")
# Training flags.
flags.DEFINE_integer(
"batch_size", 1024,
"How many examples to process per batch for training.")
FLAGS = flags.FLAGS
# Image related flags
flags.DEFINE_integer("img_size_ori", 101, "Size of original images")
flags.DEFINE_integer("img_size_target", 128, "Size of input images")
# Training flags
flags.DEFINE_integer("num_epochs", 200, "Maximum of epochs")
flags.DEFINE_integer("batch_size", 32, "Number of batch size")
flags.DEFINE_integer("start_channels", 16,
"Number of the start channels of the newwork")
flags.DEFINE_integer("random_seed", 1234, "Number of random seed")
flags.DEFINE_float("dropout", 0.2, "The ratio of dropout")
flags.DEFINE_float("val_ratio", 0.1, "The ratio of train/validation split")
flags.DEFINE_bool("shuffle", True, "Shuffle the dataset every epoch")
flags.DEFINE_bool("augmentation", True, "Do data augmentation")
flags.DEFINE_bool("random_crop", True, "Do data random cropping")
# Network flags
flags.DEFINE_integer("encoder_type", 1,
"1 - simResnet, 2 - Resnet18, 3 - Resnet34")
flags.DEFINE_integer("decoder_type", 1,
"1 - simResnet, 2 - Resnet18, 3 - Resnet34")
# Optimizer flags
flags.DEFINE_string("optimizer", "Adam", "Type of used optimizer")
flags.DEFINE_float("min_lr", 0.00001, "Minimum of learning rate")
flags.DEFINE_float("factor", 0.4, "Value of recuding factor")
flags.DEFINE_integer("factor_patience", 4,
"Multiply factor in every factor patience")
FLAGS = flags.FLAGS
if __name__ == '__main__':
flags.DEFINE_string("train_dir", "/tmp/yt8m_model/",
"The directory to load the model files from.")
flags.DEFINE_string("check_point", "", "if blank, use lastest check point")
flags.DEFINE_string("output_file", "",
"The file to save the predictions to.")
flags.DEFINE_string(
"input_data_pattern", "",
"File glob defining the evaluation dataset in tensorflow.SequenceExample "
"format. The SequenceExamples are expected to have an 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
# Model flags.
flags.DEFINE_bool("use_ema_var", False, "Use the EMA version of variable.")
flags.DEFINE_bool(
"frame_features", False,
"If set, then --eval_data_pattern must be frame-level features. "
"Otherwise, --eval_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_integer("batch_size", 8192,
"How many examples to process per batch.")
flags.DEFINE_string("feature_names", "mean_rgb", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "1024",
"Length of the feature vectors.")
flags.DEFINE_string(
"c_vars", "",
"A list of computed variables. Available transformations sq_a log_a inv_a abs_a sqrt_a diff_a_b interaction__a_b"
'V': 99.06841,
'Y': 163.06333,
'M(ox)': 147.035405,
'groupCH3': 14.01565,
'groupOH': 17.00274,
'groupH': 1.007825,
'groupH2O': 18.01057,
'groupCH3CO': 42.01057,
'groupO': 15.994915,
'groupNH3': 17.02655
}
FLAGS = flags.FLAGS
flags.DEFINE_string('input_data', '', 'Input data filepath.')
flags.DEFINE_string('output_data_dir', '', 'Input data filepath.')
flags.DEFINE_bool('clean_peptides', False,
'True if peptide modifications are in [x] format.')
flags.DEFINE_string('sequence_col', _MOD_SEQUENCE,
'Modified sequence column name in the input file.')
flags.DEFINE_string('charge_col', _CHARGE,
'Charge column name in the input file.')
flags.DEFINE_string('fragmentation_col', _FRAGMENTATION,
'Fragmentation column name in the input file.')
flags.DEFINE_string('analyzer_col', _MASS_ANALYZER,
'Mass analyzer column name in the input file.')
def generate_json_inputs(data, encoding):
"""Generates inputs to-be stored into a JSON file.
Args:
data: A pandas dataframe with modified sequence and metadata features.
"""Contains a collection of models which operate on variable-length sequences.
"""
import math
import models
import video_level_models
import tensorflow as tf
import model_utils as utils
from rhn import *
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
flags.DEFINE_integer("iterations", 30, "Number of frames per batch for DBoF.")
flags.DEFINE_bool("dbof_add_batch_norm", True,
"Adds batch normalization to the DBoF model.")
flags.DEFINE_bool(
"sample_random_frames", True,
"If true samples random frames (for frame level models). If false, a random"
"sequence of frames is sampled instead.")
flags.DEFINE_integer("dbof_cluster_size", 8192,
"Number of units in the DBoF cluster layer.")
flags.DEFINE_integer("dbof_hidden_size", 1024,
"Number of units in the DBoF hidden layer.")
flags.DEFINE_string(
"dbof_pooling_method", "max",
"The pooling method used in the DBoF cluster layer. "
"Choices are 'average' and 'max'.")
flags.DEFINE_string(
"video_level_classifier_model", "MoeModel",
"Some Frame-Level models can be decomposed into a "
flags.DEFINE_string("train_dir", "/tmp/yt8m_model/",
"The directory to save the model files in.")
flags.DEFINE_string(
"train_data_pattern", "",
"File glob for the training dataset. If the files refer to Frame Level "
"features (i.e. tensorflow.SequenceExample), then set --reader_type "
"format. The (Sequence)Examples are expected to have 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
flags.DEFINE_string("feature_names", "mean_rgb", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "1024",
"Length of the feature vectors.")
# Distillation flags
flags.DEFINE_bool(
"distillation_features", False,
"If set, *DistillationFeatureReader will be used, the feature must contains"
"prediction features (shape = [4716]).")
flags.DEFINE_integer("distillation_type", 0,
"Type of distillation, options are 1 and 2.")
flags.DEFINE_bool(
"distillation_as_input", False,
"If set true, distillation_predictions will be given to model.")
flags.DEFINE_bool(
"distillation_as_boosting", False,
"If set true, distillation_predictions will be used in computation of weighted loss."
)
flags.DEFINE_float(
"distillation_percent", 0.0,
"If larger than 0, final_loss = distillation_loss * percent + normal_loss * (1.0 - percent)."
)
"""Contains a collection of models which operate on variable-length sequences.
"""
import math
import numpy as np
import models
import video_level_models
import tensorflow as tf
import model_utils as utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
flags.DEFINE_bool("gating_remove_diag", False, "Remove diag for self gating")
flags.DEFINE_bool("lightvlad", False, "Light or full NetVLAD")
flags.DEFINE_bool("vlagd", False, "vlagd of vlad")
flags.DEFINE_integer("iterations", 30, "Number of frames per batch for DBoF.")
flags.DEFINE_bool("dbof_add_batch_norm", True,
"Adds batch normalization to the DBoF model.")
flags.DEFINE_bool(
"sample_random_frames", True,
"If true samples random frames (for frame level models). If false, a random"
"sequence of frames is sampled instead.")
flags.DEFINE_integer("dbof_cluster_size", 16384,
"Number of units in the DBoF cluster layer.")
flags.DEFINE_integer("dbof_hidden_size", 2048,
"Number of units in the DBoF hidden layer.")
"The directory to save the model files in.")
flags.DEFINE_string("train_data_patterns", "",
"Multiple file globs for the training dataset.")
flags.DEFINE_string("input_data_pattern", None,
"File globs for original model input.")
flags.DEFINE_string("feature_names", "predictions", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "3862",
"Length of the feature vectors.")
# Model flags.
flags.DEFINE_string(
"model", "LogisticModel",
"Which architecture to use for the model. Models are defined "
"in models.py.")
flags.DEFINE_bool("multitask", False,
"Whether to consider support_predictions")
flags.DEFINE_bool(
"start_new_model", False,
"If set, this will not resume from a checkpoint and will instead create a"
" new model instance.")
# Training flags.
flags.DEFINE_integer(
"batch_size", 32,
"How many examples to process per batch for training.")
flags.DEFINE_string("label_loss", "CrossEntropyLoss",
"Which loss function to use for training the model.")
flags.DEFINE_float(
"regularization_penalty", 1,
"How much weight to give to the regularization loss (the label loss has "
"a weight of 1).")
if __name__ == '__main__':
flags.DEFINE_string("train_dir", "/tmp/yt8m_model/",
"The directory to load the model files from.")
flags.DEFINE_string("output_file", "",
"The file to save the predictions to.")
flags.DEFINE_string(
"input_data_pattern", "",
"File glob defining the evaluation dataset in tensorflow.SequenceExample "
"format. The SequenceExamples are expected to have an 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
# Model flags.
flags.DEFINE_bool(
"frame_features", False,
"If set, then --eval_data_pattern must be frame-level features. "
"Otherwise, --eval_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_integer("batch_size", 8192,
"How many examples to process per batch.")
flags.DEFINE_string("feature_names", "mean_rgb", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "1024",
"Length of the feature vectors.")
# Other flags.
flags.DEFINE_integer("num_readers", 1,
"How many threads to use for reading input files.")
flags.DEFINE_integer("top_k", 20,
"How many predictions to output per video.")
"The directory to load the generator model from.")
flags.DEFINE_string("output_dir", "result/",
"The file to save the generated images to.")
flags.DEFINE_string(
"input_data_pattern", "",
"File glob defining the evaluation dataset in tensorflow.SequenceExample "
"format. This will be used for real images that will be mixed with "
"generated images.")
flags.DEFINE_integer(
"num_generate", 15,
"How many generated images among total output images.")
flags.DEFINE_integer("num_total_images", 30,
"How many images will be used for each round.")
# Other flags.
flags.DEFINE_bool("use_mnist", False, "Whether to use MNIST dataset.")
flags.DEFINE_integer("num_readers", 1,
"How many threads to use for reading input files.")
def get_image_data_tensors(reader, data_pattern, batch_size, num_readers=1):
"""Creates the section of the graph which reads the input data.
Args:
reader: A class which parses the input data.
data_pattern: A 'glob' style path to the data files.
batch_size: How many examples to process at a time.
num_readers: How many I/O threads to use.
Returns:
A image tensor.
Raises:
IOError: If no files matching the given pattern were found.
'validate_data_pattern',
'/Users/Sophie/Documents/youtube-8m-data/train_validate/validate*.tfrecord',
'Validate data pattern, to be specified when doing hyper-parameter tuning.'
)
flags.DEFINE_string('feature_names', 'mean_rgb,mean_audio',
'Features to be used, separated by ,.')
flags.DEFINE_string('feature_sizes', '1024,128',
'Dimensions of features to be used, separated by ,.')
flags.DEFINE_integer('batch_size', 1024, 'Size of batch processing.')
flags.DEFINE_integer('num_readers', 2,
'Number of readers to form a batch.')
flags.DEFINE_bool('is_bootstrap', False,
'Boolean variable indicating using bootstrap or not.')
flags.DEFINE_boolean(
'init_with_linear_clf', True,
'Boolean variable indicating whether to init logistic regression with linear classifier.'
)
flags.DEFINE_float('init_learning_rate', 0.01,
'Float variable to indicate initial learning rate.')
flags.DEFINE_integer(
'decay_steps', NUM_TRAIN_EXAMPLES,
'Float variable indicating no. of examples to decay learning rate once.'
)
flags.DEFINE_float('decay_rate', 0.95,
'truth_variants', '',
'Tabix-indexed VCF file containing the truth variant calls for this labels '
'which we use to label our examples.')
flags.DEFINE_integer('task', 0, 'Task ID of this task')
flags.DEFINE_integer(
'partition_size', 1000,
'The maximum number of basepairs we will allow in a region before splitting'
'it into multiple smaller subregions.')
flags.DEFINE_integer(
'max_reads_per_partition', 1500,
'The maximum number of reads per partition that we consider before '
'following processing such as sampling and realigner.')
flags.DEFINE_string(
'multi_allelic_mode', '',
'How to handle multi-allelic candidate variants. For DEBUGGING')
flags.DEFINE_bool('realign_reads', True,
'If True, locally realign reads before calling variants.')
flags.DEFINE_float(
'downsample_fraction', NO_DOWNSAMPLING,
'If not ' + str(NO_DOWNSAMPLING) + ' must be a value between 0.0 and 1.0. '
'Reads will be kept (randomly) with a probability of downsample_fraction '
'from the input BAM. This argument makes it easy to create examples as '
'though the input BAM had less coverage.')
flags.DEFINE_string(
'sample_name', '', 'Sample name to use for our sample_name in the output '
'Variant/DeepVariantCall protos. If not specified, will be inferred from '
'the header information from --reads.')
flags.DEFINE_string('hts_logging_level',
hts_verbose.htsLogLevel.HTS_LOG_WARNING.name,
'Sets the htslib logging threshold.')
flags.DEFINE_integer(
'hts_block_size', _DEFAULT_HTS_BLOCK_SIZE,
"files are also saved to this directory.")
flags.DEFINE_string("board_dir", "",
"The directory to save the tensorboard metric files.")
flags.DEFINE_string(
"eval_data_pattern", "",
"File glob defining the evaluation dataset in tensorflow.SequenceExample "
"format. The SequenceExamples are expected to have an 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
flags.DEFINE_string("feature_names", "mean_rgb", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "1024", "Length of the feature vectors.")
# Model flags.
flags.DEFINE_bool(
"frame_features", False,
"If set, then --eval_data_pattern must be frame-level features. "
"Otherwise, --eval_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_bool(
"image_server", False,
"Si esta ficat a True vol dir que s'esta executant en el servidor del "
" d'imatge, si no, al google (predeterminat)")
flags.DEFINE_string(
"model", "LogisticModel",
"Which architecture to use for the model. Options include 'Logistic', "
"'SingleMixtureMoe', and 'TwoLayerSigmoid'. See aggregated_models.py and "
"frame_level_models.py for the model definitions.")
flags.DEFINE_integer("batch_size", 1024,
"How many examples to process per batch.")
flags.DEFINE_bool("select_randomly", False,
"How to do the selection of features.")
if __name__ == "__main__":
# Dataset flags.
flags.DEFINE_string(
"train_dir", "/tmp/yt8m_model/",
"The directory to load the model files from. "
"The tensorboard metrics files are also saved to this "
"directory.")
flags.DEFINE_string(
"eval_data_pattern", "",
"File glob defining the evaluation dataset in tensorflow.SequenceExample "
"format. The SequenceExamples are expected to have an 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
flags.DEFINE_bool(
"segment_labels", False,
"If set, then --eval_data_pattern must be frame-level features (but with"
" segment_labels). Otherwise, --eval_data_pattern must be aggregated "
"video-level features. The model must also be set appropriately (i.e. to "
"read 3D batches VS 4D batches.")
#flags.DEFINE_string("checkpoint_file", "",
# "e.g. model.ckpt-170820")
flags.DEFINE_integer("checkpoint_file", 0, "170820")
flags.DEFINE_bool(
"google_cloud", False,
"If set, it means we use eval data for google cloud running.")
# Other flags
flags.DEFINE_integer("batch_size", 1024,
"How many examples to process per batch.")
flags.DEFINE_integer("num_readers", 8,
"How many threads to use for reading input files.")
flags.DEFINE_boolean("run_once", False, "Whether to run eval only once.")
flags.DEFINE_integer("top_k", 20,
