import logging
import M2Crypto
import time
import hashlib
import os
import utils
import tempfile
import shutil
import contrib
import flags
FLAGS = flags.FLAGS
flags.DEFINE_string('ca_file', 'cacert.pem', 'Filename of root CA')
flags.DEFINE_string('keys_path', utils.abspath('../keys'),
'Where we keep our keys')
flags.DEFINE_string('ca_path', utils.abspath('../CA'),
'Where we keep our root CA')
def ca_path(username):
if username:
return "%s/INTER/%s/cacert.pem" % (FLAGS.ca_path, username)
return "%s/cacert.pem" % (FLAGS.ca_path)
def fetch_ca(username=None, chain=True):
buffer = ""
with open(ca_path(username), "r") as cafile:
buffer += cafile.read()
_VALID_INPUT_FEATURES = frozenset({
data.SEQUENCE_ONE_HOT,
data.SEQUENCE_KMER_COUNT,
})
TUNER_LOSS_LOSS = 'loss'
TUNER_LOSS_AUC = 'auc/true_top_1p'
TUNER_GOAL_MAX = 'MAXIMIZE'
TUNER_GOAL_MIN = 'MINIMIZE'
TUNER_LOSS_TO_GOAL = {
TUNER_LOSS_LOSS: TUNER_GOAL_MIN,
TUNER_LOSS_AUC: TUNER_GOAL_MAX,
}
flags.DEFINE_integer('task', 0, 'Task id when running online')
flags.DEFINE_string('master', '', 'TensorFlow master to use')
flags.DEFINE_string('input_dir', None, 'Path to input data.')
flags.DEFINE_string(
'affinity_target_map', '',
'Name of the affinity map from count values to affinity values. '
'Needed only if using input_dir and running inference or using '
'microarray values.')
flags.DEFINE_enum(
'dataset', None,
sorted(config.INPUT_DATA_DIRS),
'Name of dataset with known input_dir on which to train. Either input_dir '
'or dataset is required.')
flags.DEFINE_integer('val_fold', 0, 'Fold to use for validation.')
flags.DEFINE_string('save_base', None,
'Base path to save any output or weights.')
flags.DEFINE_string('run_name', None, 'Name of folder created in save_base.')
from lasagne.objectives import categorical_accuracy, categorical_crossentropy
from lasagne.updates import adam
from lasagne.utils import floatX
from theano import tensor as T
import flags
from at import adversarial_training
from data import batch_iterator, mnist_load, select_balanced_subset
from deepfool import deepfool
from models import create_network, with_end_points
from utils import (build_result_str, save_images, save_network,
setup_train_experiment)
# experiment parameters
flags.DEFINE_integer("seed", 1, "experiment seed")
flags.DEFINE_string("name", None, "name of the experiment")
flags.DEFINE_string("data_dir", "data", "path to data")
flags.DEFINE_string("train_dir", "runs", "path to working dir")
# gan model parameters
flags.DEFINE_string("model", "mlp", "model name (mlp or mlp_with_bn)")
flags.DEFINE_string("layer_dims", "1000-1000-1000-10",
"dimensions of fully-connected layers")
flags.DEFINE_bool("use_dropout", False, "whenever to use dropout or not")
flags.DEFINE_float("lmbd", 1.0, "regularization coefficient")
flags.DEFINE_float("epsilon", 0.2, "epsilon for generative fgsm perturbation")
# adversary parameters
flags.DEFINE_integer("deepfool_iter", 25,
"maximum number of deepfool iterations")
flags.DEFINE_float("deepfool_clip", 0.5, "perturbation clip during search")
from daemon import pidlockfile
import flags
FLAGS = flags.FLAGS
flags.DEFINE_bool('daemonize', False, 'daemonize this process')
# NOTE(termie): right now I am defaulting to using syslog when we daemonize
# it may be better to do something else -shrug-
# (Devin) I think we should let each process have its own log file
# and put it in /var/logs/nova/(appname).log
# This makes debugging much easier and cuts down on sys log clutter.
flags.DEFINE_bool('use_syslog', True, 'output to syslog when daemonizing')
flags.DEFINE_string('logfile', None, 'log file to output to')
flags.DEFINE_string('pidfile', None, 'pid file to output to')
flags.DEFINE_string('working_directory', './', 'working directory...')
def stop(pidfile):
"""
Stop the daemon
"""
# Get the pid from the pidfile
try:
pf = file(pidfile, 'r')
pid = int(pf.read().strip())
pf.close()
except IOError:
pid = None
from context import firestore
from context import location
from context import logger
from context import network_utils
import datetime
import flags
import json
import os
import time
FLAGS = flags.FLAGS
flags.DEFINE_string('location_name', None, 'Set this location name')
flags.DEFINE_string('base_path', None, 'Set the firebase base path.')
flags.DEFINE_string('service_account_json', None,
'Set the file path to the service account key json file.')
timestamp = datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S')
database_url = (
'https://%s.firebaseio.com' %
json.loads(open(FLAGS.service_account_json).read())['project_id'])
fs = firestore.FireStore(FLAGS.base_path, database_url,
FLAGS.service_account_json)
lan_interfaces = network_utils.get_lan_interfaces()
update_location = location.Location(
name=FLAGS.location_name,
external_ip_address=network_utils.get_external_ip_address(),
local_ip_address=network_utils.get_lan_interfaces(),
metadata={
'project': 'address-locator-python',
import subprocess
import random
import time
from nova.utils import runthis, generate_uid
from nova import rpc, datastore, exception
from nova import contrib
import flags
import socket
from tornado import ioloop
from twisted.internet import defer
FLAGS = flags.FLAGS
flags.DEFINE_string('storage_dev', '/dev/sdb',
'Physical device to use for volumes')
flags.DEFINE_string('volume_group', 'nova-volumes',
'Name for the VG that will contain exported volumes')
flags.DEFINE_string('aoe_eth_dev', 'eth0',
'Which device to export the volumes on')
flags.DEFINE_string('storage_name', socket.gethostname(), 'name of this node')
flags.DEFINE_string('storage_availability_zone', 'nova',
'availability zone of this node')
KEEPER = datastore.keeper(prefix="storage")
class BlockStore(object):
def __init__(self):
super(BlockStore, self).__init__()
self.volume_class = Volume
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Benchmark script for TensorFlow.
See the README for more information.
"""
from __future__ import print_function
#from absl import app
from absl import flags as absl_flags
import tensorflow as tf
import flags
flags.DEFINE_string('network_dir', None, 'network file path.')
flags.DEFINE_string('network', 'network.py', 'network file name')
flags.DEFINE_string('data_dir', None, 'dataset location')
flags.DEFINE_integer('small_chunk', 1, 'accumulate gradients.')
flags.DEFINE_string('memory_saving_method', None,
'setup the memory saving method, 1. recomputing 2. TBD ')
flags.DEFINE_enum('lr_policy', 'multistep', ('multistep', 'exp'),
'learning_rate policy')
flags.DEFINE_boolean('aug_flip', True,
'whether randomly flip left or right dataset')
flags.DEFINE_integer(
'stop_accu_epoch', 0, 'early stop when accuracy does not increase 1% for'
'numbers of epochs')
flags.DEFINE_boolean('save_stop', True,
'whether to save checkpoint when killing process')
flags.DEFINE_list(
# pylint: enable=line-too-long
# Google internal
import apache_beam as beam
import runner
import app
import flags
from ..learning import eval_feedforward
from ..utils import pool
FLAGS = flags.FLAGS
flags.DEFINE_integer('num_batches', 1000, 'Number of batches to run')
flags.DEFINE_integer('batch_size', 10000, 'Number of sequences per batch')
flags.DEFINE_integer('num_to_save', 2000, 'The number of top results to save.')
flags.DEFINE_string('target_name', None,
'The name of the target protein for the inference.')
flags.DEFINE_string('model_dir', None,
'The path to base trained model directory.')
flags.DEFINE_string('checkpoint_path', None,
'String path to the checkpoint of the model.')
flags.DEFINE_string('output_name', None, 'The name of the output file.')
flags.DEFINE_integer('sequence_length', 40, 'The length of sequences to test.')
flags.DEFINE_string('affinity_target_map', None, 'Name of affinity target map')
_METRICS_NAMESPACE = 'SearchInference'
class RunInferenceForBatch(beam.PTransform):
"""Generates random batches and runs inference."""
def __init__(self, sequence_length, target_name, model_dir,
checkpoint_path, affinity_target_map):
import flags
import gfile
import sstable
from ..util import measurement_pb2
from ..preprocess import utils
class Error(Exception):
pass
FLAGS = flags.FLAGS
flags.DEFINE_string("fastq1", None, "Path to the first fastq file.")
flags.DEFINE_string("fastq2", None,
"Path to the second fastq file for paired-end "
"sequencing, or None for single end")
flags.DEFINE_integer("measurement_id", None,
"The measurement data set ID for this fastq pair, from "
"the experiment proto")
flags.DEFINE_integer("sequence_length", 40,
"Expected length of each sequence read")
flags.DEFINE_string("output_name",
"xxx"
"aptitude", "Path and name for the output sstable")
flags.DEFINE_integer("base_qual_threshold", 20, "integer indicating the "
"lowest quality (on scale from 0 to 40) for a single "
"base to be considered acceptable")
flags.DEFINE_integer("bad_base_threshold", 5, "integer indicating the maximum "
from collections import OrderedDict
import numpy as np
import theano
from theano import tensor as T
from lasagne.objectives import categorical_accuracy
import flags
from at import fast_gradient_perturbation
from data import batch_iterator, mnist_load, select_balanced_subset
from deepfool import deepfool
from models import create_network, with_end_points
from utils import (load_network, load_training_params, build_result_str,
save_images)
flags.DEFINE_string("load_dir", None, "path to load checkpoint from")
flags.DEFINE_integer("load_epoch", None, "epoch for which restore model")
flags.DEFINE_string("working_dir", "test", "path to working dir")
flags.DEFINE_bool("sort_labels", True, "sort labels")
flags.DEFINE_integer("batch_size", 100, "batch_index size (default: 100)")
flags.DEFINE_float("fgsm_epsilon", 0.2, "fast gradient epsilon (default: 0.2)")
flags.DEFINE_integer("deepfool_iter", 50,
"maximum number of deepfool iterations (default: 25)")
flags.DEFINE_float("deepfool_clip", 0.5,
"perturbation clip during search (default: 0.1)")
flags.DEFINE_float("deepfool_overshoot", 0.02,
"multiplier for final perturbation")
flags.DEFINE_integer("summary_frequency", 10, "summarize frequency")
FLAGS = flags.FLAGS
logger = logging.getLogger()
import os
import contrib
import anyjson
import flags
import utils
FLAGS = flags.FLAGS
flags.DEFINE_string('datastore_path', utils.abspath('../keeper'),
'where keys are stored on disk')
class keeper(object):
def __init__(self, prefix="nova-"):
self.prefix = prefix
try:
os.mkdir(FLAGS.datastore_path)
except:
pass
def _slugify(self, key):
return key
def __delitem__(self, item):
item = self._slugify(item)
path = "%s/%s%s" % (FLAGS.datastore_path, self.prefix, item)
if os.path.isfile(path):
os.remove(path)
def __getitem__(self, item):
from __future__ import print_function
from Predictor import Predictor
import flags
flags.DEFINE_string('dataset', 'cora', '[cora, citeseer]')
flags.DEFINE_string('subgraph', 'subgraph/',
'Directory of all subgraphs, each file is a subgraph')
flags.DEFINE_string('graph', 'graph.txt', 'Edge list of the complete graph')
flags.DEFINE_string('kernel', 'kernel.json', 'Kernels to be matched')
flags.DEFINE_string('query', 'query',
'Used to create query files used by SubMatch')
flags.DEFINE_string('meta', 'meta/',
'Directory of matched instances of kernels')
flags.DEFINE_string('data', 'data.txt', None)
flags.DEFINE_string('feature', 'feature.txt', None)
flags.DEFINE_string('label', 'label.txt', None)
flags.DEFINE_boolean('use_feature', True, 'Use feature or not')
flags.DEFINE_boolean('use_embedding', True, 'Use embedding or not')
flags.DEFINE_integer('feat_dim', -1, None)
flags.DEFINE_list(
'node_dim', [256],
'Dimension of hidden layers between feature and node embedding')
flags.DEFINE_list(
'instance_h_dim', [256],
'Dimension of hidden layers between node embedding and instance embedding, last element is the dimension of instance embedding'
)
flags.DEFINE_list(
'graph_h_dim', [128],
'Dimension of hidden layers between instance embedding and subgraph embedding, last element is the dimension of subgraph embedding'
