def __init__(self, name, flag_values, **kwargs):
super(UploadCommand, self).__init__(name, flag_values, **kwargs)
# Command-specific flags.
flags.DEFINE_string('root_dir',
os.path.curdir,
'Local root dir of all files.',
flag_values=flag_values)
flags.DEFINE_string('target_path',
'/',
'Remote root target path for uploaded documents.',
flag_values=flag_values)
# Mixin-specific flags:
flags.DEFINE_string(
'changeset',
None,
'If versions mixin is enabled, the changeset number or "new".',
flag_values=flag_values)
flags.DEFINE_bool(
'commit',
False,
'Whether or not to commit the changeset. This option requires that '
'--changeset=new is given.',
flag_values=flag_values)
flags.DEFINE_bool(
'confirm_manifest',
False,
'Whether or not the current list of files given to upload can be '
'trusted as a complete manifest of the files in the changeset. This '
'allows you to combine --changeset=<num> with --commit.',
flag_values=flag_values)
def main(argv):
"""Implement a simple demo for computing error CDFs."""
# Input/output flags.
gflags.DEFINE_string('input_file', None, 'Full path to wing HDF5 log file.')
gflags.MarkFlagAsRequired('input_file')
gflags.DEFINE_string('output_file', None, 'Full path to output MAT file.')
gflags.MarkFlagAsRequired('output_file')
# Segment processing flags.
gflags.DEFINE_integer('increment', 100,
'Integer number of messages between segments.')
gflags.DEFINE_integer('seg_length', 1000,
'Integer number of messages in each segment.')
# Evaluate segments over a specific time interval.
gflags.DEFINE_float('start_time', -float('inf'),
'Start time to evaluate segment errors.')
gflags.DEFINE_float('end_time', float('inf'),
'End time to evaluate segment errors.')
# Override default parameters.
gflags.DEFINE_list('params', [],
'A comma-separated list of param=value tokens, where '
'each param describes the dot path to a parameter in '
'EstimatorParams.')
gflags.RegisterValidator('params',
lambda l: all(len(s.split('=')) == 2 for s in l),
message='Invalid key=value parameter syntax.')
# Scenarios to process.
gflags.DEFINE_bool('scenario_pure_inertial', False,
'Process pure inertial scenario.')
gflags.DEFINE_bool('scenario_gps_dropout', False,
'Process GPS dropout scenario.')
# Common faults to introduce.
gflags.DEFINE_bool('fault_weather', False,
'Fault weather subsystems to avoid an assert when '
'reprocessing historical data.')
gflags.DEFINE_bool('fault_glas', False, 'Fault GLAS subsystems.')
# Specify flight for special handling.
gflags.DEFINE_string('flight', None,
'Fix known issues associated with the given flight.')
try:
argv = gflags.FLAGS(argv)
except gflags.FlagsError, e:
print '{}\nUsage: {} ARGS\n{}'.format(e, sys.argv[0], gflags.FLAGS)
sys.exit(1)
def __init__(self, name, flag_values, **kwargs):
super(CommitCommand, self).__init__(name, flag_values, **kwargs)
flags.DEFINE_bool('force_commit',
False,
'Must be given to ensure commit consistency.',
flag_values=flag_values)
flags.DEFINE_string('changeset',
None,
'The changeset number to commit.',
flag_values=flag_values)
flags.DEFINE_bool('save_manifest',
True,
'Whether or not to save a filesystem manifest.',
flag_values=flag_values)
def __init__(self, name, flag_values):
super(SetCommonInstanceMetadata, self).__init__(name, flag_values)
flags.DEFINE_bool('force',
False,
'Set the metadata even if it erases existing keys',
flag_values=flag_values,
short_name='f')
self._metadata_flags_processor = metadata.MetadataFlagsProcessor(
flag_values)
def DefineFlags(flag_values):
flags.DEFINE_string('internal_revision',
'',
'The internal revision to sync to. Should be the most '
'recent green revision.',
flag_values=flag_values)
flags.DEFINE_string('public_revision',
'',
'The public revision in equivalence with the given '
'internal revision.',
flag_values=flag_values)
flags.DEFINE_bool('install_on_db_only', False,
'Install project on MOE db and then quit.')
def __init__(self, name, flag_values):
super(OperationCommand, self).__init__(name, flag_values)
flags.DEFINE_bool('global',
None,
'List global operations.',
flag_values=flag_values)
flags.DEFINE_string(
'region',
None,
'Specifies the region for which to list operations.',
flag_values=flag_values)
flags.DEFINE_string('zone',
None,
'Specifies the zone for which to list operations.',
flag_values=flag_values)
def testIsAnyScopeFlagSpecified(self):
flag_values = flags.FlagValues()
flags.DEFINE_string('zone', None, 'zone', flag_values)
flags.DEFINE_string('region', None, 'region', flag_values)
flags.DEFINE_bool('global', False, 'global', flag_values)
def Set(name, value):
flag_values[name].present = 1
flag_values[name].value = value
def Clear(name):
flag_values[name].present = 0
flag_values[name].value = flag_values[name].default
#
# The order of Set/Clear below is important to test all combinations.
#
# Nothing is set
self.assertFalse(utils.IsAnyScopeFlagSpecified(flag_values))
# Zone
Set('zone', 'my-test-zone')
self.assertTrue(utils.IsAnyScopeFlagSpecified(flag_values))
# Zone + Global
Set('global', True)
self.assertTrue(utils.IsAnyScopeFlagSpecified(flag_values))
# Global
Clear('zone')
self.assertTrue(utils.IsAnyScopeFlagSpecified(flag_values))
# Region + Global
Set('region', 'my-test-region')
self.assertTrue(utils.IsAnyScopeFlagSpecified(flag_values))
# Region
Clear('global')
self.assertTrue(utils.IsAnyScopeFlagSpecified(flag_values))
# Region + Zone
Set('zone', 'my-test-zone')
self.assertTrue(utils.IsAnyScopeFlagSpecified(flag_values))
# Zone + Region + Global
Set('global', True)
self.assertTrue(utils.IsAnyScopeFlagSpecified(flag_values))
def __init__(self, name, flag_values, **kwargs):
super(DownloadCommand, self).__init__(name, flag_values, **kwargs)
# Command-specific flags.
flags.DEFINE_multistring('file_path', [],
'Remote file to download.',
flag_values=flag_values)
flags.DEFINE_string('dir_path',
None,
'Remote directory to download.',
flag_values=flag_values)
flags.DEFINE_bool('recursive',
False,
'Downloads from a directory recursively',
flag_values=flag_values)
flags.DEFINE_integer(
'depth',
None,
'Specifies recusion depth if "recursive" is specified',
flag_values=flag_values)
def main():
gflags.DEFINE_bool("reverse", False, "De anonymize?")
gflags.DEFINE_string("nlfile", "val_nl.txt", "file to be anonymised")
gflags.DEFINE_string("sqlfile", "val_sql.txt",
"sql file to be deanonymised")
gflags.DEFINE_string("anonfile", "val_nl_anon.txt", "anonymized file")
gflags.DEFINE_string("stop_file", "../data/ppdb/stopwords.txt",
"stopwords file")
gflags.DEFINE_string("mapfile", "val_nl_map.txt", "mapping file")
gflags.DEFINE_string("db", "geo", "")
gflags.DEFINE_string("schema", "geo.schema", "grammar file")
FLAGS(sys.argv)
anony = Anonymizer()
if FLAGS.reverse: # De anonymization
anony.deanonymizeFile(FLAGS.mapfile, FLAGS.sqlfile, FLAGS.schema,
FLAGS.stop_file)
else: # anonymization
anony.anonymizeFile(FLAGS.nlfile, FLAGS.anonfile, FLAGS.mapfile,
FLAGS.schema, FLAGS.db, FLAGS.stop_file)
from titan.files.mixins import versions_client
from titan.files.utils import titanfiles_runners
FLAGS = flags.FLAGS
flags.DEFINE_string('host', '', 'Hostname of the App Engine app.')
flags.DEFINE_integer('port', None,
'Port number; only useful for dev_appserver.')
flags.DEFINE_string('api_base_path', files_client.FILE_API_PATH_BASE,
'Base path of Titan Files API.')
flags.DEFINE_integer('threads', 100, 'Number of threads to use.')
flags.DEFINE_bool('force', False, 'Whether or not to confirm action.')
flags.DEFINE_bool('insecure', False, 'Whether or not to disable SSL.')
flags.DEFINE_bool('quiet', False, 'Whether or not to show logging.')
class BaseCommand(appcommands.Cmd):
"""Base class for commands."""
def __init__(self, *args, **kwargs):
super(BaseCommand, self).__init__(*args, **kwargs)
# Cannot access FLAGS until after initialized.
self._remote_file_factory = None
self._vcs_factory = None
@property
import gflags as flags
from rst_evaluation import const2rst, evaltrees, compare_disc_constituency
from tree import Tree
sys.setrecursionlimit(20000)
FLAGS = flags.FLAGS
flags.DEFINE_string("train", "data/joint/shuffle.train.txt",
"training data")
flags.DEFINE_string("dev", "data/joint/shuffle.dev.txt", "development data")
flags.DEFINE_string("save", None, "model saving path")
flags.DEFINE_integer("epoch", 30, "number of training epochs")
flags.DEFINE_integer("batch", 1, "Minibatch size")
flags.DEFINE_bool("extlabelfeatures", True,
"extend label features to include two children of s0")
# arguments for span-based parser
flags.DEFINE_integer("dynet_mem", 10000,
"Memory allocation for Dynet. (DEFAULT=10000)")
flags.DEFINE_float("dynet_l2", 0, "L2 regularization parmeter. (DEFAULT=0)")
flags.DEFINE_integer("dynet_seed", 123, "Seed for PNG (0: generate)")
flags.DEFINE_integer("word_dims", 50,
"Embedding dimensions for word forms. (DEFAULT=50)")
flags.DEFINE_integer("tag_dims", 20,
"Embedding dimensions of POS tags. (DEFAULT=2-)")
flags.DEFINE_integer("lstm_units", 200,
"Number of LSTM units in each layer/direction. "
"(DEFAULT=200)")
flags.DEFINE_integer("hidden_units", 200,
"Number of hidden units for each FC ReLU layer. "
gflags.DEFINE_string('server',
'localhost:2181',
"Server:Port string eg. 'localhost:2181'",
short_name='s')
gflags.DEFINE_string('username',
None,
'Your Zookeeper username',
short_name='u')
gflags.DEFINE_string('password',
None,
'Your Zookeeper password',
short_name='p')
gflags.DEFINE_bool('quiet',
False,
"When set, ndsr will not print out any useful status \
messages, but will only output the results of the command.",
short_name='q')
gflags.DEFINE_integer(
'loglevel', logging.INFO,
"The python logging level in integer form. 50-0 in \
increments of 10 descending from CRITICAL to NOTSET")
gflags.DEFINE_string(
'outputformat', 'yaml', 'The desired output format for queries. One of \
(yaml,json)')
gflags.DEFINE_bool('data',
False,
"Show data associated with each node listed (if exists)",
short_name='d')
gflags.DEFINE_bool('recursive',
False,
#!/usr/bin/python
import driver
import gflags
import sys
FLAGS = gflags.FLAGS
gflags.DEFINE_integer('zone', None, 'ID of the zone')
gflags.DEFINE_bool('state', None, 'Desired state of the zone')
gflags.MarkFlagAsRequired('zone')
gflags.MarkFlagAsRequired('state')
def main():
argv = sys.argv
try:
positional_args = gflags.FLAGS(argv) # parse flags
except gflags.FlagsError, e:
print '%s\nUsage: %s ARGS\n%s' % (e, sys.argv[0],
gflags.FLAGS.MainModuleHelp())
sys.exit(1)
sprink = driver.SprinklerDriver()
sprink.set_zone(FLAGS.zone, FLAGS.state)
print sprink.get_zone_states()
return
if __name__ == "__main__":
main()
import gflags
import os
import sys
import numpy as np
FLAGS = gflags.FLAGS
gflags.DEFINE_string('input_dir', "", "Input dir")
gflags.DEFINE_string('output_dir', "", "Output dir")
gflags.DEFINE_bool("is_grouped", False, "Whether distance is grouped")
def write_output(output_filename, distance, fun):
with open(output_filename, 'w') as f:
for key in distance:
f.write("%s,%s\n" % (key, fun(distance[key])))
def group_distances(filename):
distance = dict()
with open(filename) as f:
lines = map(lambda line: line[:-1], f.readlines())
heading = map(lambda x: x.split("_")[0], lines[1].split(","))
for i in xrange(2, len(lines)):
fields = lines[i].split(",")
class_name = fields[1].split("_")[0]
for j in xrange(2, len(fields)):
key = "%s,%s" % (class_name, heading[j])
if key not in distance:
distance[key] = []
distance[key].append(float(fields[j]))
import gflags
from main_loop_tf import gflags_ext
# Summaries and samples
gflags.DEFINE_bool('group_summaries', True, 'If True, groups the scalar '
'summaries by `layer_sublayer` rather than just by '
'`layer`. The total number of summaries remains unchanged')
gflags.DEFINE_integer('train_summary_freq', 10,
'How frequent save train summaries (in steps)')
gflags_ext.DEFINE_multidict('hyperparams_summaries',
{'1-Dataset': ['dataset',
'batch_size',
'crop_size',
'seq_length',
'of',
'remove_mean',
'divide_by_std',
'remove_per_img_mean',
'divide_by_per_img_std'],
'2-Optimization': ['optimizer',
'lr',
'lr_decay',
'decay_steps',
'decay_rate',
'staircase',
'lr_boundaries',
'lr_values',
'power',
'end_lr'],
'3-GradientProcessing': ['max_grad_norm',
from torch.utils.data import DataLoader
from torch.autograd import Variable
import matplotlib.pyplot as plt
from model import Siamese
import time
import numpy as np
import gflags
import sys
from collections import deque
import os
if __name__ == '__main__':
Flags = gflags.FLAGS
gflags.DEFINE_bool("cuda", True, "use cuda")
gflags.DEFINE_string("train_path", "/home/data/pin/data/omniglot/images_background", "training folder")
gflags.DEFINE_string("test_path", "/home/data/pin/data/omniglot/images_evaluation", 'path of testing folder')
gflags.DEFINE_integer("way", 20, "how much way one-shot learning")
gflags.DEFINE_string("times", 400, "number of samples to test accuracy")
gflags.DEFINE_integer("workers", 4, "number of dataLoader workers")
gflags.DEFINE_integer("batch_size", 128, "number of batch size")
gflags.DEFINE_float("lr", 0.00006, "learning rate")
gflags.DEFINE_integer("show_every", 10, "show result after each show_every iter.")
gflags.DEFINE_integer("save_every", 100, "save model after each save_every iter.")
gflags.DEFINE_integer("test_every", 100, "test model after each test_every iter.")
gflags.DEFINE_integer("max_iter", 50000, "number of iterations before stopping")
gflags.DEFINE_string("model_path", "/home/data/pin/model/siamese", "path to store model")
gflags.DEFINE_string("gpu_ids", "0,1,2,3", "gpu ids used to train")
Flags(sys.argv)
import cPickle as pickle
from iceberk import mpi, classifier, mathutil
from iceberk.experimental.meu_logistic_loss import loss_meu_logistic
import numpy as np
import logging
import os, sys
from sklearn.cross_validation import StratifiedKFold
import time
import gflags
########
# Settings
########
FEATDIR = "/tscratch/tmp/jiayq/imagenet-sbow/"
gflags.DEFINE_bool("load", False, "If set, load the data and then stop.")
gflags.DEFINE_float("reg", 0.01, "The reg term")
gflags.DEFINE_integer("minibatch", 100000, "The minibatch size")
gflags.DEFINE_bool("svm", False, "If set, run SVM")
gflags.DEFINE_bool("hier", False, "If set, use hierarchical loss")
FLAGS = gflags.FLAGS
FLAGS(sys.argv)
########
# Main script
########
if mpi.SIZE > 1:
raise RuntimeError, "This script runs on single machines only."
np.random.seed(42 + mpi.RANK)
mpi.root_log_level(level=logging.DEBUG)
gflags.DEFINE_integer("get_entries_retry_delay", 1, "Number of seconds after "
"which get-entries will be retried if it encountered "
"an error.")
gflags.DEFINE_integer("entries_buffer", 100000, "Size of buffer which stores "
"fetched entries before async log client is able to "
"return them. 100000 entries shouldn't take more "
"than 600 Mb of memory.")
gflags.DEFINE_integer("response_buffer_size_bytes", 50 * 1000 * 1000, "Maximum "
"size of a single response buffer. Should be set such "
"that a get_entries response comfortably fits in the "
"the buffer. A typical log entry is expected to be < "
"10kB.")
gflags.DEFINE_bool("persist_entries", True, "Cache entries on disk.")
class HTTPConnectionError(log_client.HTTPError):
"""Connection failed."""
pass
class HTTPResponseSizeExceededError(log_client.HTTPError):
"""HTTP response exceeded maximum permitted size."""
pass
###############################################################################
# The asynchronous twisted log client. #
###############################################################################
def get_flags():
# Debug settings.
gflags.DEFINE_bool(
"debug",
False,
"Set to True to disable debug_mode and type_checking.")
gflags.DEFINE_bool(
"show_progress_bar",
True,
"Turn this off when running experiments on HPC.")
gflags.DEFINE_string("git_branch_name", "", "Set automatically.")
gflags.DEFINE_string("slurm_job_id", "", "Set automatically.")
gflags.DEFINE_integer(
"deque_length",
100,
"Max trailing examples to use when computing average training statistics.")
gflags.DEFINE_string("git_sha", "", "Set automatically.")
gflags.DEFINE_string("experiment_name", "", "")
gflags.DEFINE_string("load_experiment_name", None, "")
# Data types.
gflags.DEFINE_enum("data_type",
"bl",
["bl",
"sst",
"sst-binary",
"nli",
"arithmetic",
"listops",
"sign",
"eq",
"relational"],
"Which data handler and classifier to use.")
# Choose Genre.
# 'fiction', 'government', 'slate', 'telephone', 'travel'
# 'facetoface', 'letters', 'nineeleven', 'oup', 'verbatim'
gflags.DEFINE_string("train_genre", None, "Filter MultiNLI data by genre.")
gflags.DEFINE_string("eval_genre", None, "Filter MultiNLI data by genre.")
# Where to store checkpoints
gflags.DEFINE_string(
"log_path",
"./logs",
"A directory in which to write logs.")
gflags.DEFINE_string(
"load_log_path",
None,
"A directory from which to read logs.")
gflags.DEFINE_boolean(
"write_proto_to_log",
False,
"Write logs in a protocol buffer format.")
gflags.DEFINE_string(
"ckpt_path", None, "Where to save/load checkpoints. Can be either "
"a filename or a directory. In the latter case, the experiment name serves as the "
"base for the filename.")
gflags.DEFINE_integer(
"ckpt_step",
1000,
"Steps to run before considering saving checkpoint.")
gflags.DEFINE_boolean(
"load_best",
False,
"If True, attempt to load 'best' checkpoint.")
# Data settings.
gflags.DEFINE_string("training_data_path", None, "")
gflags.DEFINE_string(
"eval_data_path", None, "Can contain multiple file paths, separated "
"using ':' tokens. The first file should be the dev set, and is used for determining "
"when to save the early stopping 'best' checkpoints.")
gflags.DEFINE_integer("seq_length", 200, "")
gflags.DEFINE_boolean(
"allow_cropping",
False,
"Trim overly long training examples to fit. If not set, skip them.")
gflags.DEFINE_integer("eval_seq_length", None, "")
gflags.DEFINE_boolean(
"allow_eval_cropping",
False,
"Trim overly long evaluation examples to fit. If not set, crash on overly long examples.")
gflags.DEFINE_boolean(
"smart_batching",
True,
"Organize batches using sequence length.")
gflags.DEFINE_boolean("use_peano", True, "A mind-blowing sorting key.")
gflags.DEFINE_integer(
"eval_data_limit",
None,
"Truncate evaluation set to this many batches. -1 indicates no truncation.")
gflags.DEFINE_boolean(
"bucket_eval",
True,
"Bucket evaluation data for speed improvement.")
gflags.DEFINE_boolean("shuffle_eval", False, "Shuffle evaluation data.")
gflags.DEFINE_integer(
"shuffle_eval_seed",
123,
"Seed shuffling of eval data.")
gflags.DEFINE_string("embedding_data_path", None,
"If set, load GloVe-formatted embeddings from here.")
gflags.DEFINE_boolean("fine_tune_loaded_embeddings", False,
"If set, backpropagate into embeddings even when initializing from pretrained.")
# Model architecture settings.
gflags.DEFINE_enum(
"model_type", "RNN", [
"CBOW", "RNN", "SPINN", "RLSPINN", "ChoiPyramid", "Maillard", "LMS"], "")
gflags.DEFINE_integer("gpu", -1, "")
gflags.DEFINE_integer("model_dim", 8, "")
gflags.DEFINE_integer("word_embedding_dim", 8, "")
gflags.DEFINE_boolean("lowercase", False, "When True, ignore case.")
gflags.DEFINE_boolean("use_internal_parser", False, "Use predicted parse.")
gflags.DEFINE_boolean(
"validate_transitions",
True,
"Constrain predicted transitions to ones that give a valid parse tree.")
gflags.DEFINE_float(
"embedding_keep_rate",
1.0,
"Used for dropout on transformed embeddings and in the encoder RNN.")
gflags.DEFINE_boolean("use_difference_feature", True, "")
gflags.DEFINE_boolean("use_product_feature", True, "")
# SPINN tracking LSTM settings.
gflags.DEFINE_integer(
"tracking_lstm_hidden_dim",
None,
"Set to none to avoid using tracker.")
gflags.DEFINE_boolean(
"tracking_ln",
False,
"When True, layer normalization is used in tracking.")
gflags.DEFINE_float(
"transition_weight",
None,
"Set to none to avoid predicting transitions.")
gflags.DEFINE_boolean("lateral_tracking", True,
"Use previous tracker state as input for new state.")
gflags.DEFINE_boolean(
"use_tracking_in_composition",
True,
"Use tracking lstm output as input for the reduce function.")
gflags.DEFINE_boolean(
"composition_ln",
True,
"When True, layer normalization is used in TreeLSTM composition.")
gflags.DEFINE_boolean("predict_use_cell", True,
"Use cell output as feature for transition net.")
# SPINN composition function settings.
gflags.DEFINE_enum(
"reduce", "treelstm", [
"treelstm", "treegru", "tanh", "lms"], "Specify composition function.")
# ChoiPyramid/ST-Gumbel model settings
gflags.DEFINE_boolean(
"pyramid_trainable_temperature",
None,
"If set, add a scalar trained temperature parameter.")
gflags.DEFINE_float("pyramid_temperature_decay_per_10k_steps",
0.5, "What it says on the box. Does not impact SparseAdam (for word embedding fine-tuning).")
gflags.DEFINE_float(
"pyramid_temperature_cycle_length",
0.0,
"For wake-sleep-style experiments. 0.0 disables this feature.")
# Embedding preprocessing settings.
gflags.DEFINE_enum("encode",
"projection",
["pass",
"projection",
"gru",
"attn"],
"Encode embeddings with sequential context.")
gflags.DEFINE_boolean("encode_reverse", False, "Encode in reverse order.")
gflags.DEFINE_boolean(
"encode_bidirectional",
False,
"Encode in both directions.")
gflags.DEFINE_integer(
"encode_num_layers",
1,
"RNN layers in encoding net.")
# RL settings.
gflags.DEFINE_float(
"rl_mu",
0.1,
"Use in exponential moving average baseline.")
gflags.DEFINE_enum("rl_baseline",
"ema",
["ema",
"pass",
"greedy",
"value"],
"Different configurations to approximate reward function.")
gflags.DEFINE_enum("rl_reward", "standard", ["standard", "xent"],
"Different reward functions to use.")
gflags.DEFINE_float("rl_weight", 1.0, "Hyperparam for REINFORCE loss.")
gflags.DEFINE_boolean("rl_whiten", False, "Reduce variance in advantage.")
gflags.DEFINE_boolean(
"rl_valid",
True,
"Only consider non-validated actions.")
gflags.DEFINE_float(
"rl_epsilon",
1.0,
"Percent of sampled actions during train time.")
gflags.DEFINE_float(
"rl_epsilon_decay",
50000,
"Step constant in epsilon delay equation.")
gflags.DEFINE_float(
"rl_confidence_interval",
1000,
"Penalize probabilities of transitions.")
gflags.DEFINE_float(
"rl_confidence_penalty",
None,
"Penalize probabilities of transitions.")
gflags.DEFINE_boolean(
"rl_catalan",
False,
"Sample over a uniform distribution of binary trees.")
gflags.DEFINE_boolean(
"rl_catalan_backprop",
False,
"Sample over a uniform distribution of binary trees.")
gflags.DEFINE_boolean(
"rl_wake_sleep",
False,
"Inverse relationship between temperature and rl_weight.")
gflags.DEFINE_boolean(
"rl_transition_acc_as_reward",
False,
"Use the transition accuracy as the reward. For debugging only.")
# MLP settings.
gflags.DEFINE_integer(
"mlp_dim",
256,
"Dimension of intermediate MLP layers.")
gflags.DEFINE_integer("num_mlp_layers", 1, "Number of MLP layers.")
gflags.DEFINE_boolean(
"mlp_ln",
True,
"When True, layer normalization is used between MLP layers.")
gflags.DEFINE_float("semantic_classifier_keep_rate", 0.9,
"Used for dropout in the semantic task classifier.")
# Optimization settings.
gflags.DEFINE_enum("optimizer_type", "SGD", ["Adam", "SGD"], "")
gflags.DEFINE_integer(
"training_steps",
1000000,
"Stop training after this point.")
gflags.DEFINE_integer("batch_size", 32, "Minibatch size.")
gflags.DEFINE_float("learning_rate", 0.5, "Used in optimizer.") # https://twitter.com/karpathy/status/801621764144971776
gflags.DEFINE_float("learning_rate_decay_when_no_progress", 0.5,
"Used in optimizer. Decay the LR by this much every epoch steps if a new best has not been set in the last epoch.")
gflags.DEFINE_float("clipping_max_value", 5.0, "")
gflags.DEFINE_float("l2_lambda", 1e-5, "")
# Display settings.
gflags.DEFINE_integer(
"statistics_interval_steps",
100,
"Log training set performance statistics at this interval.")
gflags.DEFINE_integer(
"eval_interval_steps",
100,
"Evaluate at this interval.")
gflags.DEFINE_integer(
"sample_interval_steps",
None,
"Sample transitions at this interval.")
gflags.DEFINE_integer("ckpt_interval_steps", 5000,
"Update the checkpoint on disk at this interval.")
gflags.DEFINE_boolean(
"ckpt_on_best_dev_error",
True,
"If error on the first eval set (the dev set) is "
"at most 0.99 of error at the previous checkpoint, save a special 'best' checkpoint.")
gflags.DEFINE_integer(
"early_stopping_steps_to_wait",
50000,
"If development set error doesn't improve significantly in this many steps, cease training.")
gflags.DEFINE_boolean("evalb", False, "Print transition statistics.")
gflags.DEFINE_integer("num_samples", 0, "Print sampled transitions.")
# Evaluation settings
gflags.DEFINE_boolean(
"expanded_eval_only_mode",
False,
"If set, a checkpoint is loaded and a forward pass is done to get the predicted "
"transitions. The inferred parses are written to the supplied file(s) along with example-"
"by-example accuracy information. Requirements: Must specify checkpoint path.") # TODO: Rename.
gflags.DEFINE_boolean(
"expanded_eval_only_mode_use_best_checkpoint",
True,
"When in expanded_eval_only_mode, load the ckpt_best checkpoint.")
gflags.DEFINE_boolean("write_eval_report", False, "")
gflags.DEFINE_boolean(
"eval_report_use_preds", True, "If False, use the given transitions in the report, "
"otherwise use predicted transitions. Note that when predicting transitions but not using them, the "
"reported predictions will look very odd / not valid.") # TODO: Remove.
# Maillard Pyramid
gflags.DEFINE_boolean(
"cosine",
False,
"If true, use cosine similarity instead of dot product in measuring score for compositions.")
gflags.DEFINE_enum(
"parent_selection",
"gumbel",
["gumbel",
"st-gumbel",
"softmax"],
"Which function to use to select or calculate the parent.")
gflags.DEFINE_boolean(
"right_branching",
False,
"Force right-branching composition.")
gflags.DEFINE_boolean(
"debug_branching",
False,
"use alternative style of right-branching composition.")
gflags.DEFINE_boolean(
"uniform_branching",
False,
"Uniform distribution instead of gumble softmax weighting during chart parsing.")
gflags.DEFINE_boolean(
"random_branching",
False,
"Random weighting instead of gumble softmax weighting during chart parsing.")
gflags.DEFINE_boolean(
"st_gumbel",
False,
"ST-gumble softmax weighting during chart parsing.")
import subprocess
from cdp.client.stargate.client import StargateClient
from cdp.client.stargate.stargate_interface.stargate_interface_pb2 import (
IscsiGetRedirectedSessionsArg, IscsiMigrateSessionsArg,
IscsiGetPreferredSVMArg)
from util.misc.protobuf import pb2json
from zeus.zookeeper_session import ZookeeperSession
from zeus.configuration import Configuration
FLAGS = gflags.FLAGS
gflags.DEFINE_string("vg_name", None,
("Name of the VG for which the seeions have be migrated"))
gflags.DEFINE_bool("dry_run", False,
("When set to true, print the intended distribution"))
def migrate_iscsi_target(target_name, src_svm_id, dst_svm_id,
stargate_master_ip):
"""
Calls the IscsiMigrateSessions rpc to migrate a given target to some other
SVM.
Args:
target_name(str): Name of the target to migrate
src_svm_id(str): SVM id of the source SVM where the target currently
resides
dst_svm_id(str): SVM id of the destination SVM
stargate_master_ip(str): IP of the stargate master
Returns:
Empty dict.
'rgb or grayscale')
# Training
gflags.DEFINE_integer('batch_size', 32,
'Batch size in training and evaluation')
gflags.DEFINE_integer('epochs', 100, 'Number of epochs for training')
gflags.DEFINE_integer('log_rate', 10, 'Logging rate for full model (epochs)')
gflags.DEFINE_integer('initial_epoch', 0, 'Initial epoch to start training')
# Files
gflags.DEFINE_string(
'experiment_rootdir', "./model/trained/", 'Folder '
' containing all the logs, model weights and results')
gflags.DEFINE_string('train_dir', "./data/extracted/training",
'Folder containing'
' training experiments')
gflags.DEFINE_string('val_dir', "./data/extracted/validation",
'Folder containing'
' validation experiments')
gflags.DEFINE_string('test_dir', "./data/extracted/testing",
'Folder containing'
' testing experiments')
# Model
gflags.DEFINE_bool('restore_model', False, 'Whether to restore a trained'
' model for training')
gflags.DEFINE_string('weights_fname', "best_weights.h5", '(Relative) '
'filename of model weights')
gflags.DEFINE_string('json_model_fname', "model_struct.json",
'Model struct json serialization, filename')
import csv
import os
import sys
import gflags
import hyou
import hyou.client
import oauth2client.client
CREDENTIAL_PATH = os.path.join(os.environ['HOME'], '.hyou.credential.json')
TEST_CLIENT_ID = '958069810280-th697if59r9scrf1qh0sg6gd9d9u0kts.apps.googleusercontent.com'
TEST_CLIENT_SECRET = '5nlcvd54WycOd8h8w7HD0avT'
FLAGS = gflags.FLAGS
gflags.DEFINE_bool('authenticate', False, '')
gflags.DEFINE_string('client_id', TEST_CLIENT_ID, '')
gflags.DEFINE_string('client_secret', TEST_CLIENT_SECRET, '')
gflags.MarkFlagAsRequired('client_id')
gflags.MarkFlagAsRequired('client_secret')
def load_sheet(path):
with open(path, 'rb') as f:
dialect = csv.Sniffer().sniff(f.read(1024))
f.seek(0)
reader = csv.reader(f, dialect)
return list(reader)
def upload_main(argv):
BOOT_DISK_TYPE_FLAG = ' --gce_boot_disk_type='
MACHINE_TYPE_FLAG = ' --machine_type='
MYSQL_SVC_DB_CORES_FLAG = ' --mysql_svc_db_instance_cores='
MYSQL_SVC_DB_TABLES_COUNT_FLAG = ' --mysql_svc_oltp_tables_count='
MYSQL_SVC_OLTP_TABLE_SIZE_FLAG = ' --mysql_svc_oltp_table_size='
MYSQL_INSTANCE_STORAGE_SIZE_FLAG = ' --mysql_instance_storage_size='
PROVISION = 'provision'
PREPARE = 'prepare'
RUN = 'run'
CLEANUP = 'cleanup'
TEARDOWN = 'teardown'
FLAGS = gflags.FLAGS
gflags.DEFINE_bool(
PER_SECOND_GRAPHS, False, 'Indicator for using per second data collection.'
'To enable set True.')
gflags.DEFINE_integer(
SYSBENCH_RUN_SECONDS, 480,
'The duration, in seconds, of each run phase with varying'
'thread count.')
gflags.DEFINE_integer(
SYSBENCH_WARMUP_SECONDS, 0,
'The duration, in seconds, of the warmup run in which '
'results are discarded.')
gflags.DEFINE_list(THREAD_COUNT_LIST, [1, 2, 4, 8, 16, 32, 64, 128, 256, 512],
'The number of test threads on the client side.')
gflags.DEFINE_integer(
SYSBENCH_REPORT_INTERVAL, 1,
'The interval, in seconds, we ask sysbench to report '
'results.')
flags.DEFINE_boolean('ignore_apk_installation_failures', False,
'Ignore errors if we fail to install a apk')
flags.DEFINE_list(
'accounts', None, '[START ONLY] a list of strings in format '
'username:password. These accounts will be added to the '
'emulator when it is fully booted and after all apks '
'have been installed.')
# TODO: Change default to true after all teams migrate.
flags.DEFINE_boolean('enable_console_auth', False,
'Enable console port auth for security reason')
flags.DEFINE_boolean('enable_g3_monitor', True,
'Enable g3 monitor for android_test')
flags.DEFINE_boolean('enable_gps', True, 'Enable emulator gps simulation')
flags.DEFINE_integer('retry_attempts', 4,
'The retry count when transient failure happens.')
flags.DEFINE_bool('enable_gms_usage_reporting', False, 'Whether to show gms '
'usage reporting dialog')
flags.DEFINE_string('android_sdk_path', None,
'The path where android_sdk resides on the system.')
flags.DEFINE_list(
'platform_apks', None, '[BOOT ONLY] Platform apks are '
'installed once at boot time as opposed to --apks which are '
'installed every time the emulator starts.')
_METADATA_FILE_NAME = 'emulator-meta-data.pb'
_USERDATA_IMAGES_NAME = 'userdata_images.dat'
_ADD_ACCOUNT_BROADCAST_ACTION = ('com.google.android.apps.auth.test.support'
'.action.ADD_ACCOUNT_WITH_PASSWORD')
_ADD_ACCOUNT_BOOLEAN_EXTRAS = {'syncable': True, 'auto_sync': True}
@flags.validator('window_scale')
import sys
import types
import gflags
from . import usb_exceptions
gflags.DEFINE_integer('timeout_ms', 10000, 'Timeout in milliseconds.')
gflags.DEFINE_list('port_path', [], 'USB port path integers (eg 1,2 or 2,1,1)')
gflags.DEFINE_string('serial',
None,
'Device serial to look for (host:port or USB serial)',
short_name='s')
gflags.DEFINE_bool(
'output_port_path', False,
'Affects the devices command only, outputs the port_path '
'alongside the serial if true.')
FLAGS = gflags.FLAGS
_BLACKLIST = {
'Connect',
'Close',
'ConnectDevice',
'DeviceIsAvailable',
}
def Uncamelcase(name):
parts = re.split(r'([A-Z][a-z]+)', name)[1:-1:2]
return ('-'.join(parts)).lower()
from pysc2.lib import app
import gflags as flags
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
import tensorflow.contrib.slim as slim
import scipy.signal
import os
from random import choice
from time import sleep
from time import time
FLAGS = flags.FLAGS
flags.DEFINE_bool("render", True, "Whether to render with pygame.")
flags.DEFINE_integer("screen_resolution", 84,
"Resolution for screen feature layers.")
flags.DEFINE_integer("minimap_resolution", 64,
"Resolution for minimap feature layers.")
flags.DEFINE_integer("max_agent_steps", 2500, "Total agent steps.")
flags.DEFINE_integer("game_steps_per_episode", 0, "Game steps per episode.")
flags.DEFINE_integer("step_mul", 8, "Game steps per agent step.")
flags.DEFINE_string("agent", "pysc2.agents.a3cAgent.py", "Which agent to run")
flags.DEFINE_enum("agent_race", None, sc2_env.races.keys(), "Agent's race.")
flags.DEFINE_enum("bot_race", None, sc2_env.races.keys(), "Bot's race.")
flags.DEFINE_enum("difficulty", None, sc2_env.difficulties.keys(),
"Bot's strength.")
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import gflags
import inspect
import logging
import os
import sys
FLAGS = gflags.FLAGS
gflags.DEFINE_integer('default_log_level', None, 'default log level')
gflags.DEFINE_bool('verbose', False, 'show debug output')
gflags.DEFINE_bool('use_stderr', True, 'log to standard error')
gflags.DEFINE_string('use_syslog', False, 'output to syslog')
gflags.DEFINE_string('log_dir', None, 'log file directory')
gflags.DEFINE_string('log_file', None, 'log file name')
gflags.DEFINE_string('log_file_mode', '0644', 'default log file permission')
_early_log_handler = None
def earlyInitLog(level=None):
global _early_log_handler
_early_log_handler = logging.StreamHandler(sys.stderr)
log = logging.getLogger()
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Common module for dpxdt client and server pieces."""
import logging
import sys
logging.basicConfig(
format='%(levelname)s %(filename)s:%(lineno)s] %(message)s')
# Local Libraries
sys.path.append(".")
import gflags
FLAGS = gflags.FLAGS
gflags.DEFINE_bool('verbose', False, 'When set, do verbose logging output.')
gflags.DEFINE_bool('verbose_queries', False,
'When set, do verbose logging of SQL queries.')
gflags.DEFINE_bool(
'verbose_workers', False,
'When set, do verbose logging of background Worker progress.')
import gflags
FLAGS = gflags.FLAGS
# Random seed
gflags.DEFINE_bool('random_seed', True, 'Random seed')
# Input
gflags.DEFINE_integer('target_level', -14, 'Target Level dB')
gflags.DEFINE_float('target_nu', 0.1995262315, 'Target Level nu')
gflags.DEFINE_integer('sr', 44100, 'Sample Rate')
gflags.DEFINE_integer('img_width', 862, 'Target Image Width')
gflags.DEFINE_integer('img_height', 96, 'Target Image Height')
gflags.DEFINE_string('img_mode', "rgb",
'Load mode for images, either rgb or grayscale')
gflags.DEFINE_string('h_grid', 50, 'Horizontal size of the grid classifiers')
gflags.DEFINE_string('v_grid', 25, 'Vertical size of the grid classifiers')
# Training parameters
gflags.DEFINE_integer('batch_size', 128,
'Batch size in training and evaluation')
gflags.DEFINE_integer('epochs', 100, 'Number of epochs for training')
gflags.DEFINE_integer('verbose', 1, 'Type of verbose for training')
gflags.DEFINE_integer('initial_epoch', 0, 'Initial epoch to start training')
gflags.DEFINE_float('initial_lr', 1e-3, 'Initial learning rate for adam')
gflags.DEFINE_string('f_output', 'softmax', 'Output function')
# Testing parameters
gflags.DEFINE_float('IOU', 0.5, 'Threshold for the IoU')
gflags.DEFINE_float('NMS', 0.3, 'Threshold for the NMS')
gflags.DEFINE_float(
def get_flags():
gflags.DEFINE_enum("model_type", "transup", ["transup", "bprmf", "fm",
"transe", "transh", "transr", "transd",
"cfkg", "cke", "cofm", "jtransup"], "")
gflags.DEFINE_enum("dataset", "ml1m", ['kktix', "ml1m", "dbbook2014", "amazon-book", "last-fm", "yelp2018"], "including ratings.csv, r2kg.tsv and a kg dictionary containing kg_hop[0-9].dat")
gflags.DEFINE_bool(
"filter_wrong_corrupted",
True,
"If set to True, filter test samples from train and validations.")
gflags.DEFINE_bool("share_embeddings", False, "")
gflags.DEFINE_bool("use_st_gumbel", False, "")
gflags.DEFINE_integer("max_queue", 10, ".")
gflags.DEFINE_integer("num_processes", 40, ".")
gflags.DEFINE_float("learning_rate", 0.001, "Used in optimizer.")
gflags.DEFINE_float("norm_lambda", 1.0, "decay of joint model.")
gflags.DEFINE_float("kg_lambda", 1.0, "decay of kg model.")
gflags.DEFINE_integer(
"early_stopping_steps_to_wait",
70000,
"How many times will lr decrease? If set to 0, it remains constant.")
gflags.DEFINE_bool(
"L1_flag",
False,
"If set to True, use L1 distance as dissimilarity; else, use L2.")
gflags.DEFINE_bool(
"is_report",
False,
"If set to True, use L1 distance as dissimilarity; else, use L2.")
gflags.DEFINE_float("l2_lambda", 1e-5, "")
gflags.DEFINE_integer("embedding_size", 64, ".")
gflags.DEFINE_integer("negtive_samples", 1, ".")
gflags.DEFINE_integer("batch_size", 512, "Minibatch size.")
gflags.DEFINE_enum("optimizer_type", "Adagrad", ["Adam", "SGD", "Adagrad", "Rmsprop"], "")
gflags.DEFINE_float("learning_rate_decay_when_no_progress", 0.5,
"Used in optimizer. Decay the LR by this much every epoch steps if a new best has not been set in the last epoch.")
gflags.DEFINE_integer(
"eval_interval_steps",
14000,
"Evaluate at this interval in each epoch.")
gflags.DEFINE_integer(
"training_steps",
1400000,
"Stop training after this point.")
gflags.DEFINE_float("clipping_max_value", 5.0, "")
gflags.DEFINE_float("margin", 1.0, "Used in margin loss.")
gflags.DEFINE_float("momentum", 0.9, "The momentum of the optimizer.")
gflags.DEFINE_integer("seed", 0, "Fix the random seed. Except for 0, which means no setting of random seed.")
gflags.DEFINE_integer("topn", 10, "")
gflags.DEFINE_integer("num_preferences", 4, "")
gflags.DEFINE_float("joint_ratio", 0.5, "(0 - 1). The train ratio of recommendation, kg is 1 - joint_ratio.")
gflags.DEFINE_string("experiment_name", None, "")
gflags.DEFINE_string("data_path", None, "")
gflags.DEFINE_string("rec_test_files", None, "multiple filenames separated by ':'.")
gflags.DEFINE_string("kg_test_files", None, "multiple filenames separated by ':'.")
gflags.DEFINE_string("log_path", None, "")
gflags.DEFINE_enum("log_level", "debug", ["debug", "info"], "")
gflags.DEFINE_string(
"ckpt_path", None, "Where to save/load checkpoints. If not set, the same as log_path")
gflags.DEFINE_string(
"load_ckpt_file", None, "Where to load pretrained checkpoints under log path. multiple filenames separated by ':'.")
gflags.DEFINE_boolean(
"has_visualization",
True,
"if set True, use visdom for visualization.")
gflags.DEFINE_integer("visualization_port", 8097, "")
# todo: only eval when no train.dat when load data
gflags.DEFINE_boolean(
"eval_only_mode",
False,
"If set, a checkpoint is loaded and a forward pass is done to get the predicted candidates."
"Requirements: Must specify load_experiment_name.")
gflags.DEFINE_string("load_experiment_name", None, "")
