import gflags
import sys
NYU_NON_PBS = False
NAME = "listops20s_07_28"
SWEEP_RUNS = 20
LIN = "LIN"
EXP = "EXP"
BOOL = "BOOL"
CHOICE = "CHOICE"
SS_BASE = "SS_BASE"
FLAGS = gflags.FLAGS
gflags.DEFINE_string("training_data_path", "spinn/data/listops/train_d20s.tsv",
"")
gflags.DEFINE_string("eval_data_path", "spinn/data/listops/test_d20s.tsv", "")
gflags.DEFINE_string("log_path", "/scratch/nn1119/spinn/sweep", "")
FLAGS(sys.argv)
# Instructions: Configure the variables in this block, then run
# the following on a machine with qsub access:
# python make_sweep.py > my_sweep.sh
# bash my_sweep.sh
# - #
# Non-tunable flags that must be passed in.
FIXED_PARAMETERS = {
import serial
import os
import time
from kegbot.util import app
from kegbot.util import util
from kegbot.pycore import common_defs
from kegbot.pycore import kegnet
from kegbot.kegboard import kegboard
FLAGS = gflags.FLAGS
gflags.DEFINE_string(
'kegboard_name', 'kegboard',
'Name of this kegboard that will be used when talking to the core. '
'If you are using multiple kegboards, you will want to run different '
'daemons with different names. Otherwise, the default is fine.')
gflags.DEFINE_boolean(
'show_messages', True,
'Print all messages going to and from the kegboard. Useful for '
'debugging.')
gflags.DEFINE_integer(
'required_firmware_version', 7,
'Minimum firmware version required. If the device has an older '
'firmware version, the daemon will refuse to service it. This '
'value should probably not be changed.')
FLAGS.SetDefault('tap_name', common_defs.ALIAS_ALL_TAPS)
# -*- coding:utf-8 -*-
import sys
import predict_one as po
import create_features as cf
from collections import defaultdict
import gflags
FLAGS = gflags.FLAGS
gflags.DEFINE_string('model_file',
'model.txt',
'Use model file path.')
def predict_all(model_file):
weight = defaultdict(lambda : 0.0)
# load model_file
fin = open(model_file)
for line in iter(fin.readline, ""):
parts = line.rstrip("\n").split()
value = parts.pop()
name = " ".join(parts)
weight[name] = float(value)
fin.close()
# predict all
for line in iter(sys.stdin.readline, ""):
phi = cf.create_features(line)
y = po.predict_one(weight, phi)
print int(y)
# 获取一个batch的数据
images_batch, labels_batch = ds.next_batch(flags.bsize)
loss, train_accuracy = train(images_batch, labels_batch, 0.5)
# 每100次迭代输出训练交叉熵损失以batch上的accuracy
if i % 100 == 0:
print('Loop {}:\tloss={}\ttrain accuracy={}'.format(i, loss, train_accuracy))
# 输出在测试集上的accuracy
accuracy = predict(ds.test_images, ds.test_labels, 1.0)
print('Accuracy on test set: {}'.format(accuracy))
return 0
if __name__ == '__main__':
global_flags = gflags.FLAGS
gflags.DEFINE_boolean('help', False, 'Show this help.')
gflags.DEFINE_string('gpu', '0', 'Which GPU to use.')
gflags.DEFINE_string('directory', './examples', 'Folder to save the origin data.')
gflags.DEFINE_integer('height', 28, 'Height of image.')
gflags.DEFINE_integer('width', 28, 'Width of image.')
gflags.DEFINE_integer('depth', 1, 'Depth of image.')
gflags.DEFINE_integer('feature_size', 1024, 'Output dimension of fully-connected layer .')
gflags.DEFINE_integer('num_classes', 10, 'Number of classes.')
gflags.DEFINE_integer('nloop', 20000, 'Number of loops.')
gflags.DEFINE_integer('bsize', 50, 'Batch size.')
global_flags(sys.argv)
if global_flags.help:
print(global_flags.main_module_help())
exit(0)
os.environ['CUDA_VISIBLE_DEVICES'] = global_flags.gpu
exit(main(global_flags))
# Concatenate, droppping the padding rows.
# --------------------
df = pd.concat(dfs_with_feats).dropna(subset=['filename'])
# --------------------
# Save images
# --------------------
t = time.time()
save_results(df, FLAGS)
print("Done. Saving to {:d} images took {:.3f} s.".format(
len(inputs),
time.time() - t))
if __name__ == "__main__":
# Parse cmdline options
gflags.DEFINE_string("model_def", "", "Model definition file.")
gflags.DEFINE_string("pretrained_model", "",
"Pretrained model weights file.")
gflags.DEFINE_boolean("gpu", False, "Switch for gpu computation.")
gflags.DEFINE_string("crop_mode", "center_only",
"Crop mode, from {}".format(CROP_MODES))
gflags.DEFINE_string("input_file", "", "Input txt/csv filename.")
gflags.DEFINE_string("output_file", "", "Output h5/csv filename.")
gflags.DEFINE_string("images_dim", 256,
"Canonical dimension of (square) images.")
gflags.DEFINE_string(
"images_mean_file",
os.path.join(os.path.dirname(__file__),
'../imagenet/ilsvrc_2012_mean.npy'),
"Data set image mean (numpy array).")
# AJ:
"""
import os
import sys
import time
import gflags
import glog
from cyber.python.cyber_py3 import cyber
from cyber.python.cyber_py3 import record
from modules.drivers.gnss.proto.gnss_pb2 import RawData
# Requried flags.
gflags.DEFINE_string('input_file', None, 'Input record file path.')
# Optional flags.
gflags.DEFINE_string('output_dir', './', 'Output directory path.')
# Stable flags which rarely change.
gflags.DEFINE_string('gps_raw_data_channel',
'/apollo/sensor/gnss/raw_data',
'gps raw data channel.')
def process_record_file(args):
"""Read record file and extract the message with specified channels"""
freader = record.RecordReader(args.input_file)
glog.info('#processing record file {}'.format(args.input_file))
time.sleep(1)
The color atom overlap features are loaded as a sanity check for color atom
index order.
"""
import cPickle as pickle
import gflags as flags
import gzip
import logging
import numpy as np
from oe_utils import chem
from oe_utils.utils import h5_utils
from openeye.oeshape import *
import sys
flags.DEFINE_string('model', None, 'Trained model.')
flags.DEFINE_string('query', None, 'Query molecule.')
flags.DEFINE_string('prefix', None, 'Prefix for output files.')
flags.DEFINE_string('color_atom_overlaps', None,
'Color atom overlap features.')
flags.DEFINE_integer('ref_index', 0,
'Index for query in color atom overlap features.')
FLAGS = flags.FLAGS
logging.getLogger().setLevel(logging.INFO)
def check_features(mol, color_ff, weights):
"""Check that the query matches the features."""
features = h5_utils.load(FLAGS.color_atom_overlaps)
# Compare molecule titles.
You need to specify "--root=/path/to/cifar-data" to run the code. For other
optional flags, run the script with --help or --helpshort.
@author: jiayq
'''
import cPickle as pickle
import cProfile
import gflags
import logging
from iceberk import mpi, visiondata, pipeline, classifier
import numpy as np
import os
import sys
gflags.DEFINE_string("root", "",
"The root to the cifar dataset (python format)")
gflags.RegisterValidator('root',
lambda x: x != "",
message='--root must be provided.')
gflags.DEFINE_string(
"output_dir", ".", "The output directory that stores dumped feature and"
"classifier files.")
gflags.DEFINE_string("model_file", "conv.pickle",
"The filename to output the CNN model.")
gflags.DEFINE_string("feature_file", "cnn_features",
"The filename to output the features.")
gflags.DEFINE_string("svm_file", "svm.pickle",
"The filename to output the trained svm parameters.")
gflags.DEFINE_string(
"profile_file", "", "If set, do cProfile analysis, and output the result"
"to the filename given by the flag.")
import re
import stat
import subprocess
import sys
import tarfile
import tempfile
import pytz
from google.apputils import datelib
import gflags as flags
import logging
FLAGS = flags.FLAGS
flags.DEFINE_string('project', '', 'Name of the MOE project')
flags.DEFINE_string('project_config_file', '',
'Path to the MOE project configuration file')
flags.DEFINE_string('public_username', '',
'Username for public repository client')
flags.DEFINE_string('public_password', '',
'Password for public repository client')
flags.DEFINE_boolean(
'debug_dry_run', False,
"Show what would be done, but don't make any changes to "
'the DB or any VCS. NOTE: this is only for debugging; do '
'not use if you just want to avoid automatically pushing '
'to your external repo')
# Commit strategy enumeration
import gflags
import MySQLdb
from sqlalchemy import create_engine
FLAGS = gflags.FLAGS
gflags.DEFINE_boolean('echosql', False, "is echo sql?")
gflags.DEFINE_string('dbuser', "guang", "mysql user")
gflags.DEFINE_string('dbpasswd', "guang", "mysql passwd")
gflags.DEFINE_string('db', "guang", "mysql database")
gflags.DEFINE_string('dbhost', "127.0.0.1", "mysql host")
gflags.DEFINE_integer('dbport', 3306, "mysql port")
gflags.DEFINE_integer('dbtimeout', 3600 * 4, "db timeout")
gflags.DEFINE_list('dbconnstrs', [
"guang:[email protected]:3306/gunag?charset=utf8",
], "database connstrs")
def get_db_engines(**kwargs):
dbconnstrs = kwargs.get('dbconnstrs') if kwargs.has_key(
'dbconnstrs') else FLAGS.dbconnstrs
dbtimeout = kwargs.get('dbtimeout') if kwargs.has_key(
'dbtimeout') else FLAGS.dbtimeout
echosql = kwargs.get('echosql') if kwargs.has_key(
'echosql') else FLAGS.echosql
dbs = []
for dbconnstr in dbconnstrs:
conn = create_engine('mysql://%s' % dbconnstr,
def get_flags():
# Debug settings.
gflags.DEFINE_bool("debug", False,
"Set to True to disable debug_mode and type_checking.")
gflags.DEFINE_bool("show_progress_bar", True,
"Turn this off when running experiments on HPC.")
gflags.DEFINE_enum( # change to enum
"transition_mode", "gt", ["gt", "lb", "rb", "bal"],
"Transition sequence (left-branching (lb), right-branching (rb), balanced (bal), ground-truth (gt)"
)
gflags.DEFINE_string("git_branch_name", "", "Set automatically.")
gflags.DEFINE_string("slurm_job_id", "", "Set automatically.")
gflags.DEFINE_integer(
"deque_length", 100,
"Max trailing examples to use when computing average training statistics."
)
gflags.DEFINE_string("git_sha", "", "Set automatically.")
gflags.DEFINE_string("experiment_name", "", "")
gflags.DEFINE_string("load_experiment_name", None, "")
gflags.DEFINE_string("target_vocabulary", None, "MT Target Vocabulary")
gflags.DEFINE_string("target_language", "de", "de/ar/zh")
# Data types.
gflags.DEFINE_enum("data_type", "bl", [
"bl", "sst", "sst-binary", "nli", "arithmetic", "listops", "sign",
"eq", "relational", "mt"
], "Which data handler and classifier to use.")
# Choose Genre.
# 'fiction', 'government', 'slate', 'telephone', 'travel'
# 'facetoface', 'letters', 'nineeleven', 'oup', 'verbatim'
gflags.DEFINE_string("train_genre", None, "Filter MultiNLI data by genre.")
gflags.DEFINE_string("eval_genre", None, "Filter MultiNLI data by genre.")
# Where to store checkpoints
gflags.DEFINE_string("log_path", "./logs",
"A directory in which to write logs.")
gflags.DEFINE_string("load_log_path", None,
"A directory from which to read logs.")
gflags.DEFINE_boolean("write_proto_to_log", False,
"Write logs in a protocol buffer format.")
gflags.DEFINE_string(
"ckpt_path", None, "Where to save/load checkpoints. Can be either "
"a filename or a directory. In the latter case, the experiment name serves as the "
"base for the filename.")
gflags.DEFINE_integer(
"ckpt_step", 1000,
"Steps to run before considering saving checkpoint.")
gflags.DEFINE_boolean("load_best", False,
"If True, attempt to load 'best' checkpoint.")
gflags.DEFINE_boolean(
"with_attention", False,
"If True, add all tree internal/leaf nodes to the output of SPINN.")
gflags.DEFINE_boolean(
"rl_alternate", False,
"If True, alternate between RL and MT loss backprop. (only supported for the MT classifier right now)"
)
# Data settings.
gflags.DEFINE_string("training_data_path", None, "")
gflags.DEFINE_string("source_training_path", None,
"MT source(SNLI format)")
gflags.DEFINE_string("target_training_path", None, "MT target")
gflags.DEFINE_string("source_eval_path", None, "MT source(SNLI format)")
gflags.DEFINE_string("target_eval_path", None, "MT target")
gflags.DEFINE_string("onmt_file_path", None, "onmt path")
gflags.DEFINE_string(
"eval_data_path", None, "Can contain multiple file paths, separated "
"using ':' tokens. The first file should be the dev set, and is used for determining "
"when to save the early stopping 'best' checkpoints.")
gflags.DEFINE_integer("seq_length", 200, "")
gflags.DEFINE_boolean(
"allow_cropping", False,
"Trim overly long training examples to fit. If not set, skip them.")
gflags.DEFINE_integer("eval_seq_length", None, "")
gflags.DEFINE_boolean(
"allow_eval_cropping", False,
"Trim overly long evaluation examples to fit. If not set, crash on overly long examples."
)
gflags.DEFINE_boolean("smart_batching", True,
"Organize batches using sequence length.")
gflags.DEFINE_boolean("use_peano", True, "A mind-blowing sorting key.")
gflags.DEFINE_integer(
"eval_data_limit", None,
"Truncate evaluation set to this many batches. -1 indicates no truncation."
)
gflags.DEFINE_boolean("bucket_eval", True,
"Bucket evaluation data for speed improvement.")
gflags.DEFINE_boolean("shuffle_eval", False, "Shuffle evaluation data.")
gflags.DEFINE_integer("shuffle_eval_seed", 123,
"Seed shuffling of eval data.")
gflags.DEFINE_string("embedding_data_path", None,
"If set, load GloVe-formatted embeddings from here.")
gflags.DEFINE_boolean(
"fine_tune_loaded_embeddings", False,
"If set, backpropagate into embeddings even when initializing from pretrained."
)
# Model architecture settings.
gflags.DEFINE_enum("model_type", "RNN", [
"CBOW", "RNN", "SPINN", "RLSPINN", "ChoiPyramid", "Maillard", "LMS",
"CatalanPyramid"
], "")
gflags.DEFINE_integer("gpu", -1, "")
gflags.DEFINE_integer("model_dim", 8, "")
gflags.DEFINE_integer("word_embedding_dim", 8, "")
gflags.DEFINE_boolean("lowercase", False, "When True, ignore case.")
gflags.DEFINE_boolean("use_internal_parser", False, "Use predicted parse.")
gflags.DEFINE_boolean(
"validate_transitions", True,
"Constrain predicted transitions to ones that give a valid parse tree."
)
gflags.DEFINE_float(
"embedding_keep_rate", 1.0,
"Used for dropout on transformed embeddings and in the encoder RNN.")
gflags.DEFINE_boolean("use_difference_feature", True, "")
gflags.DEFINE_boolean("use_product_feature", True, "")
# SPINN tracking LSTM settings.
gflags.DEFINE_integer("tracking_lstm_hidden_dim", None,
"Set to none to avoid using tracker.")
gflags.DEFINE_boolean(
"tracking_ln", False,
"When True, layer normalization is used in tracking.")
gflags.DEFINE_float("transition_weight", None,
"Set to none to avoid predicting transitions.")
gflags.DEFINE_boolean(
"lateral_tracking", True,
"Use previous tracker state as input for new state.")
gflags.DEFINE_boolean(
"use_tracking_in_composition", True,
"Use tracking lstm output as input for the reduce function.")
gflags.DEFINE_boolean(
"composition_ln", True,
"When True, layer normalization is used in TreeLSTM composition.")
gflags.DEFINE_boolean("predict_use_cell", True,
"Use cell output as feature for transition net.")
# SPINN composition function settings.
gflags.DEFINE_enum("reduce", "treelstm",
["treelstm", "treegru", "tanh", "lms"],
"Specify composition function.")
# ChoiPyramid/ST-Gumbel model settings
gflags.DEFINE_boolean(
"pyramid_trainable_temperature", None,
"If set, add a scalar trained temperature parameter.")
gflags.DEFINE_float(
"pyramid_temperature_decay_per_10k_steps", 0.5,
"What it says on the box. Does not impact SparseAdam (for word embedding fine-tuning)."
)
gflags.DEFINE_float(
"pyramid_temperature_cycle_length", 0.0,
"For wake-sleep-style experiments. 0.0 disables this feature.")
# Embedding preprocessing settings.
gflags.DEFINE_enum("encode", "projection",
["pass", "projection", "gru", "attn", "lstm"],
"Encode embeddings with sequential context.")
gflags.DEFINE_boolean("encode_reverse", False, "Encode in reverse order.")
gflags.DEFINE_boolean("encode_bidirectional", False,
"Encode in both directions.")
gflags.DEFINE_integer("encode_num_layers", 1,
"RNN layers in encoding net.")
# RL settings.
gflags.DEFINE_float("rl_mu", 0.1,
"Use in exponential moving average baseline.")
gflags.DEFINE_enum(
"rl_baseline", "value",
["ema", "pass", "greedy", "value", "shared", "lbtree"],
"Different configurations to approximate reward function.")
gflags.DEFINE_integer("rl_value_size", 128,
"Size of MLP used in rl baseline \"value\"")
gflags.DEFINE_integer("rl_value_lstm", 100,
"Size of LSTM used in rl basline \"value\"")
gflags.DEFINE_enum("rl_reward", "xent",
["standard", "xent", "sum", "mean"],
"Different reward functions to use.")
gflags.DEFINE_enum(
"rl_value_reward", "mse", ["bce", "mse"],
"Different loss function for state value function which estamated reward."
)
gflags.DEFINE_float("rl_weight", 1.0, "Hyperparam for REINFORCE loss.")
gflags.DEFINE_boolean("rl_whiten", False, "Reduce variance in advantage.")
gflags.DEFINE_boolean("rl_valid", True,
"Only consider non-validated actions.")
gflags.DEFINE_float("rl_epsilon", 1.0,
"Percent of sampled actions during train time.")
gflags.DEFINE_float("rl_epsilon_decay", 50000,
"Step constant in epsilon delay equation.")
gflags.DEFINE_float("rl_confidence_interval", 1000,
"Penalize probabilities of transitions.")
gflags.DEFINE_float("rl_confidence_penalty", None,
"Penalize probabilities of transitions.")
gflags.DEFINE_boolean(
"rl_catalan", False,
"Sample over a uniform distribution of binary trees.")
gflags.DEFINE_boolean(
"rl_catalan_backprop", False,
"Sample over a uniform distribution of binary trees.")
gflags.DEFINE_boolean(
"rl_wake_sleep", False,
"Inverse relationship between temperature and rl_weight.")
gflags.DEFINE_boolean(
"rl_transition_acc_as_reward", False,
"Use the transition accuracy as the reward. For debugging only.")
gflags.DEFINE_boolean(
"rl_lbtree_detach", False,
"Don't backpropogate into the treeLSTM when using it to compute the left branching tree for the rl baseline."
)
# MLP settings.
gflags.DEFINE_integer("mlp_dim", 256,
"Dimension of intermediate MLP layers.")
gflags.DEFINE_integer("num_mlp_layers", 1, "Number of MLP layers.")
gflags.DEFINE_boolean(
"mlp_ln", True,
"When True, layer normalization is used between MLP layers.")
gflags.DEFINE_float("semantic_classifier_keep_rate", 0.9,
"Used for dropout in the semantic task classifier.")
# Optimization settings.
gflags.DEFINE_enum("optimizer_type", "SGD", ["Adam", "SGD"], "")
gflags.DEFINE_integer("training_steps", 1000000,
"Stop training after this point.")
gflags.DEFINE_integer("batch_size", 32, "Minibatch size.")
gflags.DEFINE_float(
"learning_rate", 0.5, "Used in optimizer."
) # https://twitter.com/karpathy/status/801621764144971776
gflags.DEFINE_float(
"learning_rate_decay_when_no_progress", 0.5,
"Used in optimizer. Decay the LR by this much every epoch steps if a new best has not been set in the last epoch."
)
gflags.DEFINE_float("clipping_max_value", 5.0, "")
gflags.DEFINE_float("l2_lambda", 1e-5, "")
# Display settings.
gflags.DEFINE_integer(
"statistics_interval_steps", 100,
"Log training set performance statistics at this interval.")
gflags.DEFINE_integer("eval_interval_steps", 100,
"Evaluate at this interval.")
gflags.DEFINE_integer("sample_interval_steps", None,
"Sample transitions at this interval.")
gflags.DEFINE_integer("ckpt_interval_steps", 5000,
"Update the checkpoint on disk at this interval.")
gflags.DEFINE_boolean(
"ckpt_on_best_dev_error", True,
"If error on the first eval set (the dev set) is "
"at most 0.99 of error at the previous checkpoint, save a special 'best' checkpoint."
)
gflags.DEFINE_integer(
"early_stopping_steps_to_wait", 50000,
"If development set error doesn't improve significantly in this many steps, cease training."
)
gflags.DEFINE_boolean("evalb", False, "Print transition statistics.")
gflags.DEFINE_integer("num_samples", 0, "Print sampled transitions.")
# Evaluation settings
gflags.DEFINE_boolean(
"expanded_eval_only_mode", False,
"If set, a checkpoint is loaded and a forward pass is done to get the predicted "
"transitions. The inferred parses are written to the supplied file(s) along with example-"
"by-example accuracy information. Requirements: Must specify checkpoint path."
) # TODO: Rename.
gflags.DEFINE_boolean(
"expanded_eval_only_mode_use_best_checkpoint", True,
"When in expanded_eval_only_mode, load the ckpt_best checkpoint.")
gflags.DEFINE_boolean("write_eval_report", False, "")
gflags.DEFINE_boolean(
"eval_report_use_preds", True,
"If False, use the given transitions in the report, "
"otherwise use predicted transitions. Note that when predicting transitions but not using them, the "
"reported predictions will look very odd / not valid."
) # TODO: Remove.
gflags.DEFINE_boolean(
"cp_metric", False,
"Get evaluation results on consituents parsed in ListOps.")
# Chart-Parsing Variables
gflags.DEFINE_boolean(
"cosine", False,
"If true, use cosine similarity instead of dot product in measuring score for compositions."
)
gflags.DEFINE_enum(
"parent_selection", "st_gumbel", ["gumbel", "st_gumbel", "softmax"],
"Which function to use to select or calculate the parent.")
gflags.DEFINE_integer(
"low_dim", 10,
"The size of representation in chart-parser for choosing binary parses."
)
gflags.DEFINE_integer(
"cp_num", None,
"The number of times to run the chart-parser at low dim.")
gflags.DEFINE_integer(
"topk", 5,
"The number of parses to choose from chart-parser to use with SPINN.")
gflags.DEFINE_boolean(
"enforce_right", False,
"Use right branching trees, don't run through Chart-Parser.")
gflags.DEFINE_boolean("save_all_ckpts", False, "Save all checkpoints")
# Temporary flags
gflags.DEFINE_boolean("mp", False, "Use multiprocessing with chart-parser")
"""The sparse model for the parser."""
__author__ = 'kzhao'
import sys
import tempfile
from wvector import WVector
import gflags as flags
FLAGS = flags.FLAGS
flags.DEFINE_string("feats",
"feats/budget7.notag.feats",
"feature template file",
short_name="f")
class Model(object):
""" templates and weights """
names = {"SHIFT": 0, "REDUCE": 1, "SKIP": 2}
start_sym = "<s>"
end_sym = "</s>"
none_sym = "<NONE>"
eval_module = None # will be loaded on the fly
def __init__(self):
assert FLAGS.feats, "please specify feature templates"
WVector.setup(Model.names.values())
from google.cloud.security.common.data_access.sql_queries import snapshot_cycles_sql
from google.cloud.security.common.util.email_util import EmailUtil
from google.cloud.security.common.util.errors import EmailSendError
from google.cloud.security.common.util.log_util import LogUtil
from google.cloud.security.inventory.errors import LoadDataPipelineError
from google.cloud.security.inventory.pipelines import load_org_iam_policies_pipeline
from google.cloud.security.inventory.pipelines import load_projects_iam_policies_pipeline
from google.cloud.security.inventory.pipelines import load_projects_pipeline
FLAGS = flags.FLAGS
flags.DEFINE_integer('max_crm_api_calls_per_100_seconds', 400,
'Cloud Resource Manager queries per 100 seconds.')
flags.DEFINE_string('organization_id', None, 'Organization ID.')
flags.mark_flag_as_required('organization_id')
# YYYYMMDDTHHMMSSZ, e.g. 20170130T192053Z
CYCLE_TIMESTAMP_FORMAT = '%Y%m%dT%H%M%SZ'
LOGGER = LogUtil.setup_logging(__name__)
def _exists_snapshot_cycles_table(dao):
"""Whether the snapshot_cycles table exists.
Args:
dao: Data access object.
from mincepie import mapreducer, launcher
import gflags
import glob
import numpy as np
import os
from skimage import io
import uuid
import logging
from decaf.scripts.jeffnet import JeffNet
# constant value
FEATURE_DTYPE = np.float32
gflags.DEFINE_string("input_folder", "",
"The folder that contains all input tar files")
gflags.DEFINE_string("output_folder", "",
"The folder that we write output features to")
gflags.DEFINE_boolean("single", False,
"If True, the input is single image names.")
gflags.DEFINE_string("feature_name", "fc6_cudanet_out", "The feature name.")
gflags.DEFINE_integer(
"randprojection", 0, "If positive, perform random projection down to the "
" given dimension.")
FLAGS = gflags.FLAGS
class DecafnetMapper(mapreducer.BasicMapper):
"""The ImageNet Compute mapper. The input value would be a synset name.
"""
def set_up(self):
logging.info('Loading jeffnet...')
#!/usr/bin/env python
# coding=utf-8
import gflags
import time
import unittest
import sys
from ct.crypto import cert, error
from ct.crypto.asn1 import oid
from ct.crypto.asn1 import x509_extension as x509_ext
from ct.crypto.asn1 import x509_name
FLAGS = gflags.FLAGS
gflags.DEFINE_string("testdata_dir", "ct/crypto/testdata",
"Location of test certs")
class CertificateTest(unittest.TestCase):
_PEM_FILE = "google_cert.pem"
# Contains 3 certificates
# C=US/ST=California/L=Mountain View/O=Google Inc/CN=www.google.com
# C=US/O=Google Inc/CN=Google Internet Authority
# C=US/O=Equifax/OU=Equifax Secure Certificate Authority
_PEM_CHAIN_FILE = "google_chain.pem"
_DER_FILE = "google_cert.der"
# An X509v1 certificate
_V1_PEM_FILE = "v1_cert.pem"
# A old but common (0.5% of all certs as of 2013-10-01) SSL
# cert that uses a different or older DER format for Boolean
#!/usr/bin/env python
if __name__ != '__main__':
raise ImportError(u"%s may only be run as a script" % __file__)
import gflags
gflags.DEFINE_string('address', u"localhost:1984",
u"Address clients will connect to")
gflags.DEFINE_string('proxyaddress', u"2000", u"")
gflags.DEFINE_string('dbaddress', u"localhost:1984",
u"Network address backend database")
gflags.DEFINE_string('dbtype', u"dummy", u"Backend type for MinQL server")
gflags.DEFINE_string('dbname', u"dbname", u"Backend db name for MinQL server")
gflags.DEFINE_string('dbuser', u"dbuser", u"Backend db user for MinQL server")
gflags.DEFINE_string('dbpass', u"dbpass",
u"Backend db password for MinQL server")
gflags.DEFINE_integer('workers',
10,
u"Number of proxy Workers",
short_name='w')
gflags.RegisterValidator(
'workers',
lambda workers: 1 <= workers <= 50,
message=u"Number of workers must be between 1 and 50.")
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import logging
import subprocess
import sys
import gflags as flags
FLAGS = flags.FLAGS
flags.DEFINE_integer('num_streams', 1, 'Number of netperf processes to run')
flags.DEFINE_string('netperf_cmd', None,
'netperf command to run')
flags.DEFINE_integer('port_start', None,
'Starting port for netperf command and data ports')
def Main(argv=sys.argv):
# Parse command-line flags
try:
argv = FLAGS(argv)
except flags.FlagsError as e:
logging.error('%s\nUsage: %s ARGS\n%s', e, sys.argv[0], FLAGS)
sys.exit(1)
netperf_cmd = FLAGS.netperf_cmd
num_streams = FLAGS.num_streams
#!/usr/bin/env python
# coding=utf8
import os
import gflags
import sys
FLAGS = gflags.FLAGS
gflags.DEFINE_string('srcpath', "", "Source path", short_name='s')
gflags.DEFINE_string('dstpath', "", "Destination path", short_name='d')
gflags.DEFINE_string('srcpre',
"hftp://job4:50070",
"Source path",
short_name='p')
gflags.DEFINE_string('dstpre',
"hdfs://sdl-job1",
"Destination path",
short_name='q')
class PathItem:
is_file = False
length = 0
ts = 0
path = ""
def __init__(self):
self.subdir = []
import logging
import os
import subprocess
import sys
import time
import gflags
from frontend import frontend_util
from util import api
from util import stdlog
FLAGS = gflags.FLAGS
gflags.DEFINE_string('cluster_solver', None, 'Path to cluster solver binary.')
gflags.MarkFlagAsRequired('cluster_solver')
gflags.DEFINE_string('batch_evaluate_solver', None,
'Path to batch evaluate solver binary.')
gflags.MarkFlagAsRequired('batch_evaluate_solver')
gflags.DEFINE_integer(
'max_cluster_size', None,
'Maximum size of a cluster allowed to proceed before starting to solve '
'a problem. Specify 0 for no threshold.')
gflags.MarkFlagAsRequired('max_cluster_size')
gflags.DEFINE_boolean(
'counterexample_filter', None,
'Call batch_evaluate to filter candidates with counterexamples.')
import sys
from apiclient.discovery import build
from apiclient.http import MediaFileUpload
import gdata.apps.groups.client
import gflags
import httplib2
from oauth2client.client import flow_from_clientsecrets
from oauth2client.file import Storage
from oauth2client.tools import run
FLAGS = gflags.FLAGS
gflags.DEFINE_string('domain',
None,
"""Primary domain of Apps account.
Will prompt at the command line if not provided.""",
short_name='d')
gflags.DEFINE_string('group_id',
None,
'The group to migrate the sample message to.',
short_name='g')
gflags.DEFINE_string('message_file',
None,
'The file containing the RFC2822 formatted message.',
short_name='m')
gflags.DEFINE_string(
'oauth2_conf_file', 'client_secrets.json',
"""The configuration file containing the credentials
in JSON format.""")
gflags.DEFINE_string('scopes',
from provider_localization import LocalizationProvider
from provider_routing import RoutingProvider
from obstacle_decider import ObstacleDecider
from ad_vehicle import ADVehicle
gflags.DEFINE_integer('max_cruise_speed', 20,
'max speed for cruising in meter per second')
gflags.DEFINE_boolean('enable_follow', False,
'enable follow function.')
gflags.DEFINE_boolean('enable_nudge', True,
'enable nudge function.')
gflags.DEFINE_boolean('enable_change_lane', False,
'enable change lane function.')
gflags.DEFINE_boolean('enable_routing_aid', True,
'enable planning leveraging routing information.')
gflags.DEFINE_string('navigation_planning_node_name', 'navigation_planning',
'node name for navigation planning.')
gflags.DEFINE_string('navigation_planning_topic', '/apollo/planning',
'navigation planning publish topic.')
planning_pub = None
log_file = None
path_decider = None
speed_decider = None
traj_generator = None
mobileye_provider = None
routing_provider = None
obs_decider = None
adv = None
def routing_callback(routing_str):
"""
import os
import sys
import gflags
import makani
from makani.analysis.hover_disturbances import batch_sim_params
from makani.config import overrides_util
from makani.lib.python import flag_types
from makani.lib.python.batch_sim import client as client_base
from makani.lib.python.batch_sim import parameter_tables
import numpy
gflags.DEFINE_string(
'output_dir', os.path.join(makani.HOME, 'logs', 'hover_disturbances'),
'Directory in which to output .html files. '
'This will be created if it does not exist.')
gflags.DEFINE_string('azimuth', 0.0, 'Azimuth [deg] of wing.')
gflags.DEFINE_string('elevation', 10.0, 'Elevation [deg] of wing.')
gflags.DEFINE_string('payout', 20.0, 'Tether payout [m].')
gflags.DEFINE_integer('num_amplitudes', 5, 'Number of amplitudes to apply.')
flag_types.DEFINE_linspace('wind_speeds',
'0.0, 15.0, 7',
'Linspace range of wind speed values.',
nonempty=True)
#!/usr/bin/env python
from mincepie import mapreducer, launcher
import gflags
import os
import cv2
from PIL import Image
# gflags
gflags.DEFINE_string(
'image_lib', 'opencv',
'OpenCV or PIL, case insensitive. The default value is the faster OpenCV.')
gflags.DEFINE_string(
'input_folder', '',
'The folder that contains all input images, organized in synsets.')
gflags.DEFINE_integer('output_side_length', 256,
'Expected side length of the output image.')
gflags.DEFINE_string(
'output_folder', '',
'The folder that we write output resized and cropped images to')
FLAGS = gflags.FLAGS
class OpenCVResizeCrop:
def resize_and_crop_image(self,
input_file,
output_file,
output_side_length=256):
'''Takes an image name, resize it and crop the center square
'''
img = cv2.imread(input_file)
height, width, depth = img.shape
message=MISSING_CLIENT_SECRETS_MESSAGE)
# The gflags module makes defining command-line options easy for
# applications. Run this program with the '--help' argument to see
# all the flags that it understands.
gflags.DEFINE_enum('logging_level', 'DEBUG',
['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
'Set the level of logging detail.')
gflags.DEFINE_enum("request_type", 'LIST', ['GET', 'LIST'],
'Type of request to be made')
gflags.DEFINE_enum("output_type", 'STDOUT', ['BOTH', 'HTML', 'STDOUT'],
'Set how to output the results received from the API')
gflags.DEFINE_string('credentials_filename', '../credentials.dat',
'File to store credentials in', short_name='cf')
API_FLAGS = {'relationshipStatus':None, 'publisherId':None, 'role':None,
'roleId':None, 'category':None, 'minSevenDayEpc':None,
'minNinetyDayEpc':None, 'minPayoutRank':None}
gflags.DEFINE_enum(
'relationshipStatus', None,
['APPROVED', 'AVAILABLE', 'PENDING', 'DECLINED', 'DEACTIVATED'],
'Status of the relationship')
gflags.DEFINE_string(
'publisherId', None,
'Publisher ID to lookup (get requests only).')
gflags.DEFINE_string('category', None,
import oauth2client.file as oauthfile
import oauth2client.tools as oauthtools
except ImportError:
print('Could not import oauth2client\n'
'Download available at https://code.google.com/p/'
'google-api-python-client/downloads\nor run '
'`easy_install oauth2client`')
sys.exit(1)
FLAGS = gflags.FLAGS
gflags.DEFINE_enum('logging_level', 'INFO',
['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
'Set the level of logging detail.')
gflags.DEFINE_string(
'client_secrets',
os.path.join(os.path.dirname(__file__), 'client_secrets.json'),
'The OAuth 2.0 client secrets file to use')
gflags.DEFINE_string('project_id', None, 'The Cloud Storage project id')
gflags.DEFINE_string('bucket', None, 'The bucket to upload to')
gflags.DEFINE_string('publish_version', None,
'The version being published (e.g. 1.23)')
gflags.DEFINE_multistring(
'publish', [],
'A file to publish to Cloud Storage; this may be specified multiple times')
gflags.DEFINE_enum('acl', 'private',
['private', 'public-read', 'authenticated-read'],
'The ACLs to assign to the uploaded files')
API_VERSION = '2'
DEFAULT_SECRETS_FILE = os.path.join(os.path.dirname(__file__),
'client_secrets.json')
# import cPickle as pickle
import sys
import numpy
# from scipy import signal
import helpers.paths as paths
import helpers.git_helper as git_helper
import os
import h5py
import gflags
# from sklearn.decomposition import IncrementalPCA
# import shutil
import time
# from sklearn.externals import joblib
import helpers.arg_parsing as arg_parsing
gflags.DEFINE_string("input_dir", None, "Base directory with the experiments.")
gflags.DEFINE_string("out_dir", None, "Ouput directory path.")
def _setup_opts(argv):
"""Setup default arguments for the arg parser.
returns an opt dictionary with default values setup.
"""
FLAGS = gflags.FLAGS
opts = arg_parsing.setup_opts(argv, FLAGS)
return opts
def collect_write_label_stats(file, all_exps, all_labels):
# TODO: Make it easy to display plots from multiple logs
# simultaneously (i.e. results from a simulator run along side a run
# on the real wing; or from two runs of the simulator, etc).
import os
import sys
import gflags
import h5py
from makani.analysis.sys_id import sys_id_util
from makani.lib.python import dict_util
import numpy as np
import pylab
gflags.DEFINE_string('output_dir', '',
'Directory into which output files will be written.')
gflags.DEFINE_boolean('save', True,
'Whether to save the plots into PNG and PDF files.')
gflags.DEFINE_boolean('interactive', False,
'Whether to display the plots interactively.')
gflags.DEFINE_integer(
'nfft', 1024, 'Number of points at which to compute the Fourier '
'transform.')
gflags.DEFINE_string(
'basename', '', 'Base filename for output files. If this option is '
'empty, the base name of the input log file is used.')
gflags.DEFINE_float(
'min_coherence', 0.8,
'Minimum coherence required to include data point on '
'transfer function plot.')
gflags.DEFINE_enum(
# coding=utf-8
import Image, ImageDraw, ImageFont
import sys
import gflags
gflags.DEFINE_integer("draw_alignment_margin", 60,
"the margin for alignment graph")
gflags.DEFINE_integer("draw_space_between_rows", 200, "the space between rows")
gflags.DEFINE_integer("draw_font_size", 40, "font size")
gflags.DEFINE_string("text_font", "simkai.ttf", "text font")
gflags.DEFINE_string(
'src_sentences', 'test/Eng.txt',
'the file containing source sentences. One sentence per line. Words are separated by spaces.'
)
gflags.DEFINE_string(
'trg_sentences', 'test/Chs.txt',
'the file containing translations of the source sentences. One sentence per line. Words are separated by spaces.'
)
gflags.DEFINE_string(
'align_file', 'test/Align.txt',
"the file containing the alignment between source sentences and their translations. Each sentence's alignment is on one separate line. Alignments follow the GIZA++ format."
)
gflags.DEFINE_integer('sentence_id', 0, "index starts from 0")
gflags.DEFINE_string('output_image', "output.png", "the output image")
def GetBoxes(words, (startx, starty), font):
boxes = []
first_word = True
from iceberk import mathutil, mpi
import numpy as np
import os, sys
import gflags
import glob
import h5py
gflags.DEFINE_string("model", "", "The model file")
gflags.DEFINE_string("folder", "", "The input folder that contains the data")
gflags.DEFINE_string("output", "", "The output file")
gflags.DEFINE_float("reg", 1e-8, "The regularization term")
FLAGS = gflags.FLAGS
FLAGS(sys.argv)
if FLAGS.folder == "" or FLAGS.model == "":
sys.exit(1)
model = np.load(FLAGS.model)
w = model['w']
b = model['b']
hw = np.zeros_like(w)
hb = np.zeros_like(b)
files = glob.glob(os.path.join(FLAGS.folder, '*.mat'))
files.sort()
count = 0
for i in range(mpi.RANK, len(files), mpi.SIZE):
file = files[i]
print '%d / %d: %s' % (i, len(files), file)
import sys
from threading import Thread
import string
import random
import time
import boto
import gflags as flags
import gcs_oauth2_boto_plugin # noqa
FLAGS = flags.FLAGS
flags.DEFINE_enum('storage_provider', 'GCS', ['GCS', 'S3', 'AZURE'],
'The target storage provider to test.')
flags.DEFINE_string('host', None, 'The hostname of the storage endpoint.')
flags.DEFINE_string(
'bucket', None, 'The name of the bucket to test with. Caller is '
'responsible to create an empty bucket for a particular '
'invocation of this test and then clean-up the bucket '
'after this test returns.')
flags.DEFINE_enum(
'scenario', 'OneByteRW',
['OneByteRW', 'ListConsistency', 'SingleStreamThroughput'],
'The various scenarios to test. OneByteRW: read and write of single byte. '
'ListConsistency: List-after-write and list-after-update consistency. '
'SingleStreamThroughput: Throughput of single stream large object RW. (not '
'implemented yet)')
