import gflags import sys NYU_NON_PBS = False NAME = "listops20s_07_28" SWEEP_RUNS = 20 LIN = "LIN" EXP = "EXP" BOOL = "BOOL" CHOICE = "CHOICE" SS_BASE = "SS_BASE" FLAGS = gflags.FLAGS gflags.DEFINE_string("training_data_path", "spinn/data/listops/train_d20s.tsv", "") gflags.DEFINE_string("eval_data_path", "spinn/data/listops/test_d20s.tsv", "") gflags.DEFINE_string("log_path", "/scratch/nn1119/spinn/sweep", "") FLAGS(sys.argv) # Instructions: Configure the variables in this block, then run # the following on a machine with qsub access: # python make_sweep.py > my_sweep.sh # bash my_sweep.sh # - # # Non-tunable flags that must be passed in. FIXED_PARAMETERS = {
import serial import os import time from kegbot.util import app from kegbot.util import util from kegbot.pycore import common_defs from kegbot.pycore import kegnet from kegbot.kegboard import kegboard FLAGS = gflags.FLAGS gflags.DEFINE_string( 'kegboard_name', 'kegboard', 'Name of this kegboard that will be used when talking to the core. ' 'If you are using multiple kegboards, you will want to run different ' 'daemons with different names. Otherwise, the default is fine.') gflags.DEFINE_boolean( 'show_messages', True, 'Print all messages going to and from the kegboard. Useful for ' 'debugging.') gflags.DEFINE_integer( 'required_firmware_version', 7, 'Minimum firmware version required. If the device has an older ' 'firmware version, the daemon will refuse to service it. This ' 'value should probably not be changed.') FLAGS.SetDefault('tap_name', common_defs.ALIAS_ALL_TAPS)
# -*- coding:utf-8 -*- import sys import predict_one as po import create_features as cf from collections import defaultdict import gflags FLAGS = gflags.FLAGS gflags.DEFINE_string('model_file', 'model.txt', 'Use model file path.') def predict_all(model_file): weight = defaultdict(lambda : 0.0) # load model_file fin = open(model_file) for line in iter(fin.readline, ""): parts = line.rstrip("\n").split() value = parts.pop() name = " ".join(parts) weight[name] = float(value) fin.close() # predict all for line in iter(sys.stdin.readline, ""): phi = cf.create_features(line) y = po.predict_one(weight, phi) print int(y)
# 获取一个batch的数据 images_batch, labels_batch = ds.next_batch(flags.bsize) loss, train_accuracy = train(images_batch, labels_batch, 0.5) # 每100次迭代输出训练交叉熵损失以batch上的accuracy if i % 100 == 0: print('Loop {}:\tloss={}\ttrain accuracy={}'.format(i, loss, train_accuracy)) # 输出在测试集上的accuracy accuracy = predict(ds.test_images, ds.test_labels, 1.0) print('Accuracy on test set: {}'.format(accuracy)) return 0 if __name__ == '__main__': global_flags = gflags.FLAGS gflags.DEFINE_boolean('help', False, 'Show this help.') gflags.DEFINE_string('gpu', '0', 'Which GPU to use.') gflags.DEFINE_string('directory', './examples', 'Folder to save the origin data.') gflags.DEFINE_integer('height', 28, 'Height of image.') gflags.DEFINE_integer('width', 28, 'Width of image.') gflags.DEFINE_integer('depth', 1, 'Depth of image.') gflags.DEFINE_integer('feature_size', 1024, 'Output dimension of fully-connected layer .') gflags.DEFINE_integer('num_classes', 10, 'Number of classes.') gflags.DEFINE_integer('nloop', 20000, 'Number of loops.') gflags.DEFINE_integer('bsize', 50, 'Batch size.') global_flags(sys.argv) if global_flags.help: print(global_flags.main_module_help()) exit(0) os.environ['CUDA_VISIBLE_DEVICES'] = global_flags.gpu exit(main(global_flags))
# Concatenate, droppping the padding rows. # -------------------- df = pd.concat(dfs_with_feats).dropna(subset=['filename']) # -------------------- # Save images # -------------------- t = time.time() save_results(df, FLAGS) print("Done. Saving to {:d} images took {:.3f} s.".format( len(inputs), time.time() - t)) if __name__ == "__main__": # Parse cmdline options gflags.DEFINE_string("model_def", "", "Model definition file.") gflags.DEFINE_string("pretrained_model", "", "Pretrained model weights file.") gflags.DEFINE_boolean("gpu", False, "Switch for gpu computation.") gflags.DEFINE_string("crop_mode", "center_only", "Crop mode, from {}".format(CROP_MODES)) gflags.DEFINE_string("input_file", "", "Input txt/csv filename.") gflags.DEFINE_string("output_file", "", "Output h5/csv filename.") gflags.DEFINE_string("images_dim", 256, "Canonical dimension of (square) images.") gflags.DEFINE_string( "images_mean_file", os.path.join(os.path.dirname(__file__), '../imagenet/ilsvrc_2012_mean.npy'), "Data set image mean (numpy array).") # AJ:
""" import os import sys import time import gflags import glog from cyber.python.cyber_py3 import cyber from cyber.python.cyber_py3 import record from modules.drivers.gnss.proto.gnss_pb2 import RawData # Requried flags. gflags.DEFINE_string('input_file', None, 'Input record file path.') # Optional flags. gflags.DEFINE_string('output_dir', './', 'Output directory path.') # Stable flags which rarely change. gflags.DEFINE_string('gps_raw_data_channel', '/apollo/sensor/gnss/raw_data', 'gps raw data channel.') def process_record_file(args): """Read record file and extract the message with specified channels""" freader = record.RecordReader(args.input_file) glog.info('#processing record file {}'.format(args.input_file)) time.sleep(1)
The color atom overlap features are loaded as a sanity check for color atom index order. """ import cPickle as pickle import gflags as flags import gzip import logging import numpy as np from oe_utils import chem from oe_utils.utils import h5_utils from openeye.oeshape import * import sys flags.DEFINE_string('model', None, 'Trained model.') flags.DEFINE_string('query', None, 'Query molecule.') flags.DEFINE_string('prefix', None, 'Prefix for output files.') flags.DEFINE_string('color_atom_overlaps', None, 'Color atom overlap features.') flags.DEFINE_integer('ref_index', 0, 'Index for query in color atom overlap features.') FLAGS = flags.FLAGS logging.getLogger().setLevel(logging.INFO) def check_features(mol, color_ff, weights): """Check that the query matches the features.""" features = h5_utils.load(FLAGS.color_atom_overlaps) # Compare molecule titles.
You need to specify "--root=/path/to/cifar-data" to run the code. For other optional flags, run the script with --help or --helpshort. @author: jiayq ''' import cPickle as pickle import cProfile import gflags import logging from iceberk import mpi, visiondata, pipeline, classifier import numpy as np import os import sys gflags.DEFINE_string("root", "", "The root to the cifar dataset (python format)") gflags.RegisterValidator('root', lambda x: x != "", message='--root must be provided.') gflags.DEFINE_string( "output_dir", ".", "The output directory that stores dumped feature and" "classifier files.") gflags.DEFINE_string("model_file", "conv.pickle", "The filename to output the CNN model.") gflags.DEFINE_string("feature_file", "cnn_features", "The filename to output the features.") gflags.DEFINE_string("svm_file", "svm.pickle", "The filename to output the trained svm parameters.") gflags.DEFINE_string( "profile_file", "", "If set, do cProfile analysis, and output the result" "to the filename given by the flag.")
import re import stat import subprocess import sys import tarfile import tempfile import pytz from google.apputils import datelib import gflags as flags import logging FLAGS = flags.FLAGS flags.DEFINE_string('project', '', 'Name of the MOE project') flags.DEFINE_string('project_config_file', '', 'Path to the MOE project configuration file') flags.DEFINE_string('public_username', '', 'Username for public repository client') flags.DEFINE_string('public_password', '', 'Password for public repository client') flags.DEFINE_boolean( 'debug_dry_run', False, "Show what would be done, but don't make any changes to " 'the DB or any VCS. NOTE: this is only for debugging; do ' 'not use if you just want to avoid automatically pushing ' 'to your external repo') # Commit strategy enumeration
import gflags import MySQLdb from sqlalchemy import create_engine FLAGS = gflags.FLAGS gflags.DEFINE_boolean('echosql', False, "is echo sql?") gflags.DEFINE_string('dbuser', "guang", "mysql user") gflags.DEFINE_string('dbpasswd', "guang", "mysql passwd") gflags.DEFINE_string('db', "guang", "mysql database") gflags.DEFINE_string('dbhost', "127.0.0.1", "mysql host") gflags.DEFINE_integer('dbport', 3306, "mysql port") gflags.DEFINE_integer('dbtimeout', 3600 * 4, "db timeout") gflags.DEFINE_list('dbconnstrs', [ "guang:[email protected]:3306/gunag?charset=utf8", ], "database connstrs") def get_db_engines(**kwargs): dbconnstrs = kwargs.get('dbconnstrs') if kwargs.has_key( 'dbconnstrs') else FLAGS.dbconnstrs dbtimeout = kwargs.get('dbtimeout') if kwargs.has_key( 'dbtimeout') else FLAGS.dbtimeout echosql = kwargs.get('echosql') if kwargs.has_key( 'echosql') else FLAGS.echosql dbs = [] for dbconnstr in dbconnstrs: conn = create_engine('mysql://%s' % dbconnstr,
def get_flags(): # Debug settings. gflags.DEFINE_bool("debug", False, "Set to True to disable debug_mode and type_checking.") gflags.DEFINE_bool("show_progress_bar", True, "Turn this off when running experiments on HPC.") gflags.DEFINE_enum( # change to enum "transition_mode", "gt", ["gt", "lb", "rb", "bal"], "Transition sequence (left-branching (lb), right-branching (rb), balanced (bal), ground-truth (gt)" ) gflags.DEFINE_string("git_branch_name", "", "Set automatically.") gflags.DEFINE_string("slurm_job_id", "", "Set automatically.") gflags.DEFINE_integer( "deque_length", 100, "Max trailing examples to use when computing average training statistics." ) gflags.DEFINE_string("git_sha", "", "Set automatically.") gflags.DEFINE_string("experiment_name", "", "") gflags.DEFINE_string("load_experiment_name", None, "") gflags.DEFINE_string("target_vocabulary", None, "MT Target Vocabulary") gflags.DEFINE_string("target_language", "de", "de/ar/zh") # Data types. gflags.DEFINE_enum("data_type", "bl", [ "bl", "sst", "sst-binary", "nli", "arithmetic", "listops", "sign", "eq", "relational", "mt" ], "Which data handler and classifier to use.") # Choose Genre. # 'fiction', 'government', 'slate', 'telephone', 'travel' # 'facetoface', 'letters', 'nineeleven', 'oup', 'verbatim' gflags.DEFINE_string("train_genre", None, "Filter MultiNLI data by genre.") gflags.DEFINE_string("eval_genre", None, "Filter MultiNLI data by genre.") # Where to store checkpoints gflags.DEFINE_string("log_path", "./logs", "A directory in which to write logs.") gflags.DEFINE_string("load_log_path", None, "A directory from which to read logs.") gflags.DEFINE_boolean("write_proto_to_log", False, "Write logs in a protocol buffer format.") gflags.DEFINE_string( "ckpt_path", None, "Where to save/load checkpoints. Can be either " "a filename or a directory. In the latter case, the experiment name serves as the " "base for the filename.") gflags.DEFINE_integer( "ckpt_step", 1000, "Steps to run before considering saving checkpoint.") gflags.DEFINE_boolean("load_best", False, "If True, attempt to load 'best' checkpoint.") gflags.DEFINE_boolean( "with_attention", False, "If True, add all tree internal/leaf nodes to the output of SPINN.") gflags.DEFINE_boolean( "rl_alternate", False, "If True, alternate between RL and MT loss backprop. (only supported for the MT classifier right now)" ) # Data settings. gflags.DEFINE_string("training_data_path", None, "") gflags.DEFINE_string("source_training_path", None, "MT source(SNLI format)") gflags.DEFINE_string("target_training_path", None, "MT target") gflags.DEFINE_string("source_eval_path", None, "MT source(SNLI format)") gflags.DEFINE_string("target_eval_path", None, "MT target") gflags.DEFINE_string("onmt_file_path", None, "onmt path") gflags.DEFINE_string( "eval_data_path", None, "Can contain multiple file paths, separated " "using ':' tokens. The first file should be the dev set, and is used for determining " "when to save the early stopping 'best' checkpoints.") gflags.DEFINE_integer("seq_length", 200, "") gflags.DEFINE_boolean( "allow_cropping", False, "Trim overly long training examples to fit. If not set, skip them.") gflags.DEFINE_integer("eval_seq_length", None, "") gflags.DEFINE_boolean( "allow_eval_cropping", False, "Trim overly long evaluation examples to fit. If not set, crash on overly long examples." ) gflags.DEFINE_boolean("smart_batching", True, "Organize batches using sequence length.") gflags.DEFINE_boolean("use_peano", True, "A mind-blowing sorting key.") gflags.DEFINE_integer( "eval_data_limit", None, "Truncate evaluation set to this many batches. -1 indicates no truncation." ) gflags.DEFINE_boolean("bucket_eval", True, "Bucket evaluation data for speed improvement.") gflags.DEFINE_boolean("shuffle_eval", False, "Shuffle evaluation data.") gflags.DEFINE_integer("shuffle_eval_seed", 123, "Seed shuffling of eval data.") gflags.DEFINE_string("embedding_data_path", None, "If set, load GloVe-formatted embeddings from here.") gflags.DEFINE_boolean( "fine_tune_loaded_embeddings", False, "If set, backpropagate into embeddings even when initializing from pretrained." ) # Model architecture settings. gflags.DEFINE_enum("model_type", "RNN", [ "CBOW", "RNN", "SPINN", "RLSPINN", "ChoiPyramid", "Maillard", "LMS", "CatalanPyramid" ], "") gflags.DEFINE_integer("gpu", -1, "") gflags.DEFINE_integer("model_dim", 8, "") gflags.DEFINE_integer("word_embedding_dim", 8, "") gflags.DEFINE_boolean("lowercase", False, "When True, ignore case.") gflags.DEFINE_boolean("use_internal_parser", False, "Use predicted parse.") gflags.DEFINE_boolean( "validate_transitions", True, "Constrain predicted transitions to ones that give a valid parse tree." ) gflags.DEFINE_float( "embedding_keep_rate", 1.0, "Used for dropout on transformed embeddings and in the encoder RNN.") gflags.DEFINE_boolean("use_difference_feature", True, "") gflags.DEFINE_boolean("use_product_feature", True, "") # SPINN tracking LSTM settings. gflags.DEFINE_integer("tracking_lstm_hidden_dim", None, "Set to none to avoid using tracker.") gflags.DEFINE_boolean( "tracking_ln", False, "When True, layer normalization is used in tracking.") gflags.DEFINE_float("transition_weight", None, "Set to none to avoid predicting transitions.") gflags.DEFINE_boolean( "lateral_tracking", True, "Use previous tracker state as input for new state.") gflags.DEFINE_boolean( "use_tracking_in_composition", True, "Use tracking lstm output as input for the reduce function.") gflags.DEFINE_boolean( "composition_ln", True, "When True, layer normalization is used in TreeLSTM composition.") gflags.DEFINE_boolean("predict_use_cell", True, "Use cell output as feature for transition net.") # SPINN composition function settings. gflags.DEFINE_enum("reduce", "treelstm", ["treelstm", "treegru", "tanh", "lms"], "Specify composition function.") # ChoiPyramid/ST-Gumbel model settings gflags.DEFINE_boolean( "pyramid_trainable_temperature", None, "If set, add a scalar trained temperature parameter.") gflags.DEFINE_float( "pyramid_temperature_decay_per_10k_steps", 0.5, "What it says on the box. Does not impact SparseAdam (for word embedding fine-tuning)." ) gflags.DEFINE_float( "pyramid_temperature_cycle_length", 0.0, "For wake-sleep-style experiments. 0.0 disables this feature.") # Embedding preprocessing settings. gflags.DEFINE_enum("encode", "projection", ["pass", "projection", "gru", "attn", "lstm"], "Encode embeddings with sequential context.") gflags.DEFINE_boolean("encode_reverse", False, "Encode in reverse order.") gflags.DEFINE_boolean("encode_bidirectional", False, "Encode in both directions.") gflags.DEFINE_integer("encode_num_layers", 1, "RNN layers in encoding net.") # RL settings. gflags.DEFINE_float("rl_mu", 0.1, "Use in exponential moving average baseline.") gflags.DEFINE_enum( "rl_baseline", "value", ["ema", "pass", "greedy", "value", "shared", "lbtree"], "Different configurations to approximate reward function.") gflags.DEFINE_integer("rl_value_size", 128, "Size of MLP used in rl baseline \"value\"") gflags.DEFINE_integer("rl_value_lstm", 100, "Size of LSTM used in rl basline \"value\"") gflags.DEFINE_enum("rl_reward", "xent", ["standard", "xent", "sum", "mean"], "Different reward functions to use.") gflags.DEFINE_enum( "rl_value_reward", "mse", ["bce", "mse"], "Different loss function for state value function which estamated reward." ) gflags.DEFINE_float("rl_weight", 1.0, "Hyperparam for REINFORCE loss.") gflags.DEFINE_boolean("rl_whiten", False, "Reduce variance in advantage.") gflags.DEFINE_boolean("rl_valid", True, "Only consider non-validated actions.") gflags.DEFINE_float("rl_epsilon", 1.0, "Percent of sampled actions during train time.") gflags.DEFINE_float("rl_epsilon_decay", 50000, "Step constant in epsilon delay equation.") gflags.DEFINE_float("rl_confidence_interval", 1000, "Penalize probabilities of transitions.") gflags.DEFINE_float("rl_confidence_penalty", None, "Penalize probabilities of transitions.") gflags.DEFINE_boolean( "rl_catalan", False, "Sample over a uniform distribution of binary trees.") gflags.DEFINE_boolean( "rl_catalan_backprop", False, "Sample over a uniform distribution of binary trees.") gflags.DEFINE_boolean( "rl_wake_sleep", False, "Inverse relationship between temperature and rl_weight.") gflags.DEFINE_boolean( "rl_transition_acc_as_reward", False, "Use the transition accuracy as the reward. For debugging only.") gflags.DEFINE_boolean( "rl_lbtree_detach", False, "Don't backpropogate into the treeLSTM when using it to compute the left branching tree for the rl baseline." ) # MLP settings. gflags.DEFINE_integer("mlp_dim", 256, "Dimension of intermediate MLP layers.") gflags.DEFINE_integer("num_mlp_layers", 1, "Number of MLP layers.") gflags.DEFINE_boolean( "mlp_ln", True, "When True, layer normalization is used between MLP layers.") gflags.DEFINE_float("semantic_classifier_keep_rate", 0.9, "Used for dropout in the semantic task classifier.") # Optimization settings. gflags.DEFINE_enum("optimizer_type", "SGD", ["Adam", "SGD"], "") gflags.DEFINE_integer("training_steps", 1000000, "Stop training after this point.") gflags.DEFINE_integer("batch_size", 32, "Minibatch size.") gflags.DEFINE_float( "learning_rate", 0.5, "Used in optimizer." ) # https://twitter.com/karpathy/status/801621764144971776 gflags.DEFINE_float( "learning_rate_decay_when_no_progress", 0.5, "Used in optimizer. Decay the LR by this much every epoch steps if a new best has not been set in the last epoch." ) gflags.DEFINE_float("clipping_max_value", 5.0, "") gflags.DEFINE_float("l2_lambda", 1e-5, "") # Display settings. gflags.DEFINE_integer( "statistics_interval_steps", 100, "Log training set performance statistics at this interval.") gflags.DEFINE_integer("eval_interval_steps", 100, "Evaluate at this interval.") gflags.DEFINE_integer("sample_interval_steps", None, "Sample transitions at this interval.") gflags.DEFINE_integer("ckpt_interval_steps", 5000, "Update the checkpoint on disk at this interval.") gflags.DEFINE_boolean( "ckpt_on_best_dev_error", True, "If error on the first eval set (the dev set) is " "at most 0.99 of error at the previous checkpoint, save a special 'best' checkpoint." ) gflags.DEFINE_integer( "early_stopping_steps_to_wait", 50000, "If development set error doesn't improve significantly in this many steps, cease training." ) gflags.DEFINE_boolean("evalb", False, "Print transition statistics.") gflags.DEFINE_integer("num_samples", 0, "Print sampled transitions.") # Evaluation settings gflags.DEFINE_boolean( "expanded_eval_only_mode", False, "If set, a checkpoint is loaded and a forward pass is done to get the predicted " "transitions. The inferred parses are written to the supplied file(s) along with example-" "by-example accuracy information. Requirements: Must specify checkpoint path." ) # TODO: Rename. gflags.DEFINE_boolean( "expanded_eval_only_mode_use_best_checkpoint", True, "When in expanded_eval_only_mode, load the ckpt_best checkpoint.") gflags.DEFINE_boolean("write_eval_report", False, "") gflags.DEFINE_boolean( "eval_report_use_preds", True, "If False, use the given transitions in the report, " "otherwise use predicted transitions. Note that when predicting transitions but not using them, the " "reported predictions will look very odd / not valid." ) # TODO: Remove. gflags.DEFINE_boolean( "cp_metric", False, "Get evaluation results on consituents parsed in ListOps.") # Chart-Parsing Variables gflags.DEFINE_boolean( "cosine", False, "If true, use cosine similarity instead of dot product in measuring score for compositions." ) gflags.DEFINE_enum( "parent_selection", "st_gumbel", ["gumbel", "st_gumbel", "softmax"], "Which function to use to select or calculate the parent.") gflags.DEFINE_integer( "low_dim", 10, "The size of representation in chart-parser for choosing binary parses." ) gflags.DEFINE_integer( "cp_num", None, "The number of times to run the chart-parser at low dim.") gflags.DEFINE_integer( "topk", 5, "The number of parses to choose from chart-parser to use with SPINN.") gflags.DEFINE_boolean( "enforce_right", False, "Use right branching trees, don't run through Chart-Parser.") gflags.DEFINE_boolean("save_all_ckpts", False, "Save all checkpoints") # Temporary flags gflags.DEFINE_boolean("mp", False, "Use multiprocessing with chart-parser")
"""The sparse model for the parser.""" __author__ = 'kzhao' import sys import tempfile from wvector import WVector import gflags as flags FLAGS = flags.FLAGS flags.DEFINE_string("feats", "feats/budget7.notag.feats", "feature template file", short_name="f") class Model(object): """ templates and weights """ names = {"SHIFT": 0, "REDUCE": 1, "SKIP": 2} start_sym = "<s>" end_sym = "</s>" none_sym = "<NONE>" eval_module = None # will be loaded on the fly def __init__(self): assert FLAGS.feats, "please specify feature templates" WVector.setup(Model.names.values())
from google.cloud.security.common.data_access.sql_queries import snapshot_cycles_sql from google.cloud.security.common.util.email_util import EmailUtil from google.cloud.security.common.util.errors import EmailSendError from google.cloud.security.common.util.log_util import LogUtil from google.cloud.security.inventory.errors import LoadDataPipelineError from google.cloud.security.inventory.pipelines import load_org_iam_policies_pipeline from google.cloud.security.inventory.pipelines import load_projects_iam_policies_pipeline from google.cloud.security.inventory.pipelines import load_projects_pipeline FLAGS = flags.FLAGS flags.DEFINE_integer('max_crm_api_calls_per_100_seconds', 400, 'Cloud Resource Manager queries per 100 seconds.') flags.DEFINE_string('organization_id', None, 'Organization ID.') flags.mark_flag_as_required('organization_id') # YYYYMMDDTHHMMSSZ, e.g. 20170130T192053Z CYCLE_TIMESTAMP_FORMAT = '%Y%m%dT%H%M%SZ' LOGGER = LogUtil.setup_logging(__name__) def _exists_snapshot_cycles_table(dao): """Whether the snapshot_cycles table exists. Args: dao: Data access object.
from mincepie import mapreducer, launcher import gflags import glob import numpy as np import os from skimage import io import uuid import logging from decaf.scripts.jeffnet import JeffNet # constant value FEATURE_DTYPE = np.float32 gflags.DEFINE_string("input_folder", "", "The folder that contains all input tar files") gflags.DEFINE_string("output_folder", "", "The folder that we write output features to") gflags.DEFINE_boolean("single", False, "If True, the input is single image names.") gflags.DEFINE_string("feature_name", "fc6_cudanet_out", "The feature name.") gflags.DEFINE_integer( "randprojection", 0, "If positive, perform random projection down to the " " given dimension.") FLAGS = gflags.FLAGS class DecafnetMapper(mapreducer.BasicMapper): """The ImageNet Compute mapper. The input value would be a synset name. """ def set_up(self): logging.info('Loading jeffnet...')
#!/usr/bin/env python # coding=utf-8 import gflags import time import unittest import sys from ct.crypto import cert, error from ct.crypto.asn1 import oid from ct.crypto.asn1 import x509_extension as x509_ext from ct.crypto.asn1 import x509_name FLAGS = gflags.FLAGS gflags.DEFINE_string("testdata_dir", "ct/crypto/testdata", "Location of test certs") class CertificateTest(unittest.TestCase): _PEM_FILE = "google_cert.pem" # Contains 3 certificates # C=US/ST=California/L=Mountain View/O=Google Inc/CN=www.google.com # C=US/O=Google Inc/CN=Google Internet Authority # C=US/O=Equifax/OU=Equifax Secure Certificate Authority _PEM_CHAIN_FILE = "google_chain.pem" _DER_FILE = "google_cert.der" # An X509v1 certificate _V1_PEM_FILE = "v1_cert.pem" # A old but common (0.5% of all certs as of 2013-10-01) SSL # cert that uses a different or older DER format for Boolean
#!/usr/bin/env python if __name__ != '__main__': raise ImportError(u"%s may only be run as a script" % __file__) import gflags gflags.DEFINE_string('address', u"localhost:1984", u"Address clients will connect to") gflags.DEFINE_string('proxyaddress', u"2000", u"") gflags.DEFINE_string('dbaddress', u"localhost:1984", u"Network address backend database") gflags.DEFINE_string('dbtype', u"dummy", u"Backend type for MinQL server") gflags.DEFINE_string('dbname', u"dbname", u"Backend db name for MinQL server") gflags.DEFINE_string('dbuser', u"dbuser", u"Backend db user for MinQL server") gflags.DEFINE_string('dbpass', u"dbpass", u"Backend db password for MinQL server") gflags.DEFINE_integer('workers', 10, u"Number of proxy Workers", short_name='w') gflags.RegisterValidator( 'workers', lambda workers: 1 <= workers <= 50, message=u"Number of workers must be between 1 and 50.")
# distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import logging import subprocess import sys import gflags as flags FLAGS = flags.FLAGS flags.DEFINE_integer('num_streams', 1, 'Number of netperf processes to run') flags.DEFINE_string('netperf_cmd', None, 'netperf command to run') flags.DEFINE_integer('port_start', None, 'Starting port for netperf command and data ports') def Main(argv=sys.argv): # Parse command-line flags try: argv = FLAGS(argv) except flags.FlagsError as e: logging.error('%s\nUsage: %s ARGS\n%s', e, sys.argv[0], FLAGS) sys.exit(1) netperf_cmd = FLAGS.netperf_cmd num_streams = FLAGS.num_streams
#!/usr/bin/env python # coding=utf8 import os import gflags import sys FLAGS = gflags.FLAGS gflags.DEFINE_string('srcpath', "", "Source path", short_name='s') gflags.DEFINE_string('dstpath', "", "Destination path", short_name='d') gflags.DEFINE_string('srcpre', "hftp://job4:50070", "Source path", short_name='p') gflags.DEFINE_string('dstpre', "hdfs://sdl-job1", "Destination path", short_name='q') class PathItem: is_file = False length = 0 ts = 0 path = "" def __init__(self): self.subdir = []
import logging import os import subprocess import sys import time import gflags from frontend import frontend_util from util import api from util import stdlog FLAGS = gflags.FLAGS gflags.DEFINE_string('cluster_solver', None, 'Path to cluster solver binary.') gflags.MarkFlagAsRequired('cluster_solver') gflags.DEFINE_string('batch_evaluate_solver', None, 'Path to batch evaluate solver binary.') gflags.MarkFlagAsRequired('batch_evaluate_solver') gflags.DEFINE_integer( 'max_cluster_size', None, 'Maximum size of a cluster allowed to proceed before starting to solve ' 'a problem. Specify 0 for no threshold.') gflags.MarkFlagAsRequired('max_cluster_size') gflags.DEFINE_boolean( 'counterexample_filter', None, 'Call batch_evaluate to filter candidates with counterexamples.')
import sys from apiclient.discovery import build from apiclient.http import MediaFileUpload import gdata.apps.groups.client import gflags import httplib2 from oauth2client.client import flow_from_clientsecrets from oauth2client.file import Storage from oauth2client.tools import run FLAGS = gflags.FLAGS gflags.DEFINE_string('domain', None, """Primary domain of Apps account. Will prompt at the command line if not provided.""", short_name='d') gflags.DEFINE_string('group_id', None, 'The group to migrate the sample message to.', short_name='g') gflags.DEFINE_string('message_file', None, 'The file containing the RFC2822 formatted message.', short_name='m') gflags.DEFINE_string( 'oauth2_conf_file', 'client_secrets.json', """The configuration file containing the credentials in JSON format.""") gflags.DEFINE_string('scopes',
from provider_localization import LocalizationProvider from provider_routing import RoutingProvider from obstacle_decider import ObstacleDecider from ad_vehicle import ADVehicle gflags.DEFINE_integer('max_cruise_speed', 20, 'max speed for cruising in meter per second') gflags.DEFINE_boolean('enable_follow', False, 'enable follow function.') gflags.DEFINE_boolean('enable_nudge', True, 'enable nudge function.') gflags.DEFINE_boolean('enable_change_lane', False, 'enable change lane function.') gflags.DEFINE_boolean('enable_routing_aid', True, 'enable planning leveraging routing information.') gflags.DEFINE_string('navigation_planning_node_name', 'navigation_planning', 'node name for navigation planning.') gflags.DEFINE_string('navigation_planning_topic', '/apollo/planning', 'navigation planning publish topic.') planning_pub = None log_file = None path_decider = None speed_decider = None traj_generator = None mobileye_provider = None routing_provider = None obs_decider = None adv = None def routing_callback(routing_str):
""" import os import sys import gflags import makani from makani.analysis.hover_disturbances import batch_sim_params from makani.config import overrides_util from makani.lib.python import flag_types from makani.lib.python.batch_sim import client as client_base from makani.lib.python.batch_sim import parameter_tables import numpy gflags.DEFINE_string( 'output_dir', os.path.join(makani.HOME, 'logs', 'hover_disturbances'), 'Directory in which to output .html files. ' 'This will be created if it does not exist.') gflags.DEFINE_string('azimuth', 0.0, 'Azimuth [deg] of wing.') gflags.DEFINE_string('elevation', 10.0, 'Elevation [deg] of wing.') gflags.DEFINE_string('payout', 20.0, 'Tether payout [m].') gflags.DEFINE_integer('num_amplitudes', 5, 'Number of amplitudes to apply.') flag_types.DEFINE_linspace('wind_speeds', '0.0, 15.0, 7', 'Linspace range of wind speed values.', nonempty=True)
#!/usr/bin/env python from mincepie import mapreducer, launcher import gflags import os import cv2 from PIL import Image # gflags gflags.DEFINE_string( 'image_lib', 'opencv', 'OpenCV or PIL, case insensitive. The default value is the faster OpenCV.') gflags.DEFINE_string( 'input_folder', '', 'The folder that contains all input images, organized in synsets.') gflags.DEFINE_integer('output_side_length', 256, 'Expected side length of the output image.') gflags.DEFINE_string( 'output_folder', '', 'The folder that we write output resized and cropped images to') FLAGS = gflags.FLAGS class OpenCVResizeCrop: def resize_and_crop_image(self, input_file, output_file, output_side_length=256): '''Takes an image name, resize it and crop the center square ''' img = cv2.imread(input_file) height, width, depth = img.shape
message=MISSING_CLIENT_SECRETS_MESSAGE) # The gflags module makes defining command-line options easy for # applications. Run this program with the '--help' argument to see # all the flags that it understands. gflags.DEFINE_enum('logging_level', 'DEBUG', ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], 'Set the level of logging detail.') gflags.DEFINE_enum("request_type", 'LIST', ['GET', 'LIST'], 'Type of request to be made') gflags.DEFINE_enum("output_type", 'STDOUT', ['BOTH', 'HTML', 'STDOUT'], 'Set how to output the results received from the API') gflags.DEFINE_string('credentials_filename', '../credentials.dat', 'File to store credentials in', short_name='cf') API_FLAGS = {'relationshipStatus':None, 'publisherId':None, 'role':None, 'roleId':None, 'category':None, 'minSevenDayEpc':None, 'minNinetyDayEpc':None, 'minPayoutRank':None} gflags.DEFINE_enum( 'relationshipStatus', None, ['APPROVED', 'AVAILABLE', 'PENDING', 'DECLINED', 'DEACTIVATED'], 'Status of the relationship') gflags.DEFINE_string( 'publisherId', None, 'Publisher ID to lookup (get requests only).') gflags.DEFINE_string('category', None,
import oauth2client.file as oauthfile import oauth2client.tools as oauthtools except ImportError: print('Could not import oauth2client\n' 'Download available at https://code.google.com/p/' 'google-api-python-client/downloads\nor run ' '`easy_install oauth2client`') sys.exit(1) FLAGS = gflags.FLAGS gflags.DEFINE_enum('logging_level', 'INFO', ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], 'Set the level of logging detail.') gflags.DEFINE_string( 'client_secrets', os.path.join(os.path.dirname(__file__), 'client_secrets.json'), 'The OAuth 2.0 client secrets file to use') gflags.DEFINE_string('project_id', None, 'The Cloud Storage project id') gflags.DEFINE_string('bucket', None, 'The bucket to upload to') gflags.DEFINE_string('publish_version', None, 'The version being published (e.g. 1.23)') gflags.DEFINE_multistring( 'publish', [], 'A file to publish to Cloud Storage; this may be specified multiple times') gflags.DEFINE_enum('acl', 'private', ['private', 'public-read', 'authenticated-read'], 'The ACLs to assign to the uploaded files') API_VERSION = '2' DEFAULT_SECRETS_FILE = os.path.join(os.path.dirname(__file__), 'client_secrets.json')
# import cPickle as pickle import sys import numpy # from scipy import signal import helpers.paths as paths import helpers.git_helper as git_helper import os import h5py import gflags # from sklearn.decomposition import IncrementalPCA # import shutil import time # from sklearn.externals import joblib import helpers.arg_parsing as arg_parsing gflags.DEFINE_string("input_dir", None, "Base directory with the experiments.") gflags.DEFINE_string("out_dir", None, "Ouput directory path.") def _setup_opts(argv): """Setup default arguments for the arg parser. returns an opt dictionary with default values setup. """ FLAGS = gflags.FLAGS opts = arg_parsing.setup_opts(argv, FLAGS) return opts def collect_write_label_stats(file, all_exps, all_labels):
# TODO: Make it easy to display plots from multiple logs # simultaneously (i.e. results from a simulator run along side a run # on the real wing; or from two runs of the simulator, etc). import os import sys import gflags import h5py from makani.analysis.sys_id import sys_id_util from makani.lib.python import dict_util import numpy as np import pylab gflags.DEFINE_string('output_dir', '', 'Directory into which output files will be written.') gflags.DEFINE_boolean('save', True, 'Whether to save the plots into PNG and PDF files.') gflags.DEFINE_boolean('interactive', False, 'Whether to display the plots interactively.') gflags.DEFINE_integer( 'nfft', 1024, 'Number of points at which to compute the Fourier ' 'transform.') gflags.DEFINE_string( 'basename', '', 'Base filename for output files. If this option is ' 'empty, the base name of the input log file is used.') gflags.DEFINE_float( 'min_coherence', 0.8, 'Minimum coherence required to include data point on ' 'transfer function plot.') gflags.DEFINE_enum(
# coding=utf-8 import Image, ImageDraw, ImageFont import sys import gflags gflags.DEFINE_integer("draw_alignment_margin", 60, "the margin for alignment graph") gflags.DEFINE_integer("draw_space_between_rows", 200, "the space between rows") gflags.DEFINE_integer("draw_font_size", 40, "font size") gflags.DEFINE_string("text_font", "simkai.ttf", "text font") gflags.DEFINE_string( 'src_sentences', 'test/Eng.txt', 'the file containing source sentences. One sentence per line. Words are separated by spaces.' ) gflags.DEFINE_string( 'trg_sentences', 'test/Chs.txt', 'the file containing translations of the source sentences. One sentence per line. Words are separated by spaces.' ) gflags.DEFINE_string( 'align_file', 'test/Align.txt', "the file containing the alignment between source sentences and their translations. Each sentence's alignment is on one separate line. Alignments follow the GIZA++ format." ) gflags.DEFINE_integer('sentence_id', 0, "index starts from 0") gflags.DEFINE_string('output_image', "output.png", "the output image") def GetBoxes(words, (startx, starty), font): boxes = [] first_word = True
from iceberk import mathutil, mpi import numpy as np import os, sys import gflags import glob import h5py gflags.DEFINE_string("model", "", "The model file") gflags.DEFINE_string("folder", "", "The input folder that contains the data") gflags.DEFINE_string("output", "", "The output file") gflags.DEFINE_float("reg", 1e-8, "The regularization term") FLAGS = gflags.FLAGS FLAGS(sys.argv) if FLAGS.folder == "" or FLAGS.model == "": sys.exit(1) model = np.load(FLAGS.model) w = model['w'] b = model['b'] hw = np.zeros_like(w) hb = np.zeros_like(b) files = glob.glob(os.path.join(FLAGS.folder, '*.mat')) files.sort() count = 0 for i in range(mpi.RANK, len(files), mpi.SIZE): file = files[i] print '%d / %d: %s' % (i, len(files), file)
import sys from threading import Thread import string import random import time import boto import gflags as flags import gcs_oauth2_boto_plugin # noqa FLAGS = flags.FLAGS flags.DEFINE_enum('storage_provider', 'GCS', ['GCS', 'S3', 'AZURE'], 'The target storage provider to test.') flags.DEFINE_string('host', None, 'The hostname of the storage endpoint.') flags.DEFINE_string( 'bucket', None, 'The name of the bucket to test with. Caller is ' 'responsible to create an empty bucket for a particular ' 'invocation of this test and then clean-up the bucket ' 'after this test returns.') flags.DEFINE_enum( 'scenario', 'OneByteRW', ['OneByteRW', 'ListConsistency', 'SingleStreamThroughput'], 'The various scenarios to test. OneByteRW: read and write of single byte. ' 'ListConsistency: List-after-write and list-after-update consistency. ' 'SingleStreamThroughput: Throughput of single stream large object RW. (not ' 'implemented yet)')