def main(unused_argv):
flags.mark_flag_as_required('model_dir')
flags.mark_flag_as_required('pipeline_config_path')
config = tf.estimator.RunConfig(model_dir=FLAGS.model_dir)
train_and_eval_dict = model_lib.create_estimator_and_inputs(
run_config=config,
hparams=model_hparams.create_hparams(FLAGS.hparams_overrides),
pipeline_config_path=FLAGS.pipeline_config_path,
train_steps=FLAGS.num_train_steps,
eval_steps=FLAGS.num_eval_steps)
estimator = train_and_eval_dict['estimator']
train_input_fn = train_and_eval_dict['train_input_fn']
eval_input_fn = train_and_eval_dict['eval_input_fn']
eval_on_train_input_fn = train_and_eval_dict['eval_on_train_input_fn']
predict_input_fn = train_and_eval_dict['predict_input_fn']
train_steps = train_and_eval_dict['train_steps']
eval_steps = train_and_eval_dict['eval_steps']
train_spec, eval_specs = model_lib.create_train_and_eval_specs(
train_input_fn,
eval_input_fn,
eval_on_train_input_fn,
predict_input_fn,
train_steps,
eval_steps,
eval_on_train_data=False)
# Currently only a single Eval Spec is allowed.
tf.estimator.train_and_evaluate(estimator, train_spec, eval_specs[0])
def main(unused_argv):
flags.mark_flag_as_required('model_dir')
flags.mark_flag_as_required('pipeline_config_path')
tpu_cluster_resolver = (
tf.contrib.cluster_resolver.TPUClusterResolver(
tpu=[FLAGS.tpu_name],
zone=FLAGS.tpu_zone,
project=FLAGS.gcp_project))
tpu_grpc_url = tpu_cluster_resolver.get_master()
config = tf.contrib.tpu.RunConfig(
master=tpu_grpc_url,
evaluation_master=tpu_grpc_url,
model_dir=FLAGS.model_dir,
tpu_config=tf.contrib.tpu.TPUConfig(
iterations_per_loop=FLAGS.iterations_per_loop,
num_shards=FLAGS.num_shards))
kwargs = {}
if FLAGS.train_batch_size:
kwargs['batch_size'] = FLAGS.train_batch_size
train_and_eval_dict = model_lib.create_estimator_and_inputs(
run_config=config,
hparams=model_hparams.create_hparams(FLAGS.hparams_overrides),
pipeline_config_path=FLAGS.pipeline_config_path,
train_steps=FLAGS.num_train_steps,
sample_1_of_n_eval_examples=FLAGS.sample_1_of_n_eval_examples,
sample_1_of_n_eval_on_train_examples=(
FLAGS.sample_1_of_n_eval_on_train_examples),
use_tpu_estimator=True,
use_tpu=FLAGS.use_tpu,
num_shards=FLAGS.num_shards,
save_final_config=FLAGS.mode == 'train',
**kwargs)
estimator = train_and_eval_dict['estimator']
train_input_fn = train_and_eval_dict['train_input_fn']
eval_input_fns = train_and_eval_dict['eval_input_fns']
eval_on_train_input_fn = train_and_eval_dict['eval_on_train_input_fn']
train_steps = train_and_eval_dict['train_steps']
if FLAGS.mode == 'train':
estimator.train(input_fn=train_input_fn, max_steps=train_steps)
# Continuously evaluating.
if FLAGS.mode == 'eval':
if FLAGS.eval_training_data:
name = 'training_data'
input_fn = eval_on_train_input_fn
else:
name = 'validation_data'
# Currently only a single eval input is allowed.
input_fn = eval_input_fns[0]
model_lib.continuous_eval(estimator, FLAGS.model_dir, input_fn, train_steps,
name)
def main(argv):
del argv # Unused.
logging.set_verbosity(FLAGS.log_level)
flags.mark_flag_as_required('logdir')
if FLAGS.num_workers <= 0:
raise ValueError('num_workers flag must be greater than 0.')
if FLAGS.task_id < 0:
raise ValueError('task_id flag must be greater than or equal to 0.')
if FLAGS.task_id >= FLAGS.num_workers:
raise ValueError(
'task_id flag must be strictly less than num_workers flag.')
ns, _ = get_namespace(FLAGS.config)
ns.run_training(is_chief=FLAGS.task_id == 0)
def main(unused_argv):
flags.mark_flag_as_required('model_dir')
flags.mark_flag_as_required('pipeline_config_path')
config = tf.estimator.RunConfig(model_dir=FLAGS.model_dir)
train_and_eval_dict = model_lib.create_estimator_and_inputs(
run_config=config,
hparams=model_hparams.create_hparams(FLAGS.hparams_overrides),
pipeline_config_path=FLAGS.pipeline_config_path,
train_steps=FLAGS.num_train_steps,
sample_1_of_n_eval_examples=FLAGS.sample_1_of_n_eval_examples,
sample_1_of_n_eval_on_train_examples=(
FLAGS.sample_1_of_n_eval_on_train_examples))
estimator = train_and_eval_dict['estimator']
train_input_fn = train_and_eval_dict['train_input_fn']
eval_input_fns = train_and_eval_dict['eval_input_fns']
eval_on_train_input_fn = train_and_eval_dict['eval_on_train_input_fn']
predict_input_fn = train_and_eval_dict['predict_input_fn']
train_steps = train_and_eval_dict['train_steps']
if FLAGS.checkpoint_dir:
if FLAGS.eval_training_data:
name = 'training_data'
input_fn = eval_on_train_input_fn
else:
name = 'validation_data'
# The first eval input will be evaluated.
input_fn = eval_input_fns[0]
if FLAGS.run_once:
estimator.evaluate(input_fn,
steps=None,
checkpoint_path=tf.train.latest_checkpoint(
FLAGS.checkpoint_dir))
else:
model_lib.continuous_eval(estimator, FLAGS.checkpoint_dir, input_fn,
train_steps, name)
else:
train_spec, eval_specs = model_lib.create_train_and_eval_specs(
train_input_fn,
eval_input_fns,
eval_on_train_input_fn,
predict_input_fn,
train_steps,
eval_on_train_data=False)
# Currently only a single Eval Spec is allowed.
tf.estimator.train_and_evaluate(estimator, train_spec, eval_specs[0])
def define_compute_bleu_flags():
"""Add flags for computing BLEU score."""
flags.DEFINE_string(
name="translation", default=None,
help=flags_core.help_wrap("File containing translated text."))
flags.mark_flag_as_required("translation")
flags.DEFINE_string(
name="reference", default=None,
help=flags_core.help_wrap("File containing reference translation."))
flags.mark_flag_as_required("reference")
flags.DEFINE_enum(
name="bleu_variant", short_name="bv", default="both",
enum_values=["both", "uncased", "cased"], case_sensitive=False,
help=flags_core.help_wrap(
"Specify one or more BLEU variants to calculate. Variants: \"cased\""
", \"uncased\", or \"both\"."))
def define_translate_flags():
"""Define flags used for translation script."""
# Model flags
flags.DEFINE_string(
name="model_dir", short_name="md", default="/tmp/transformer_model",
help=flags_core.help_wrap(
"Directory containing Transformer model checkpoints."))
flags.DEFINE_enum(
name="param_set", short_name="mp", default="big",
enum_values=["base", "big"],
help=flags_core.help_wrap(
"Parameter set to use when creating and training the model. The "
"parameters define the input shape (batch size and max length), "
"model configuration (size of embedding, # of hidden layers, etc.), "
"and various other settings. The big parameter set increases the "
"default batch size, embedding/hidden size, and filter size. For a "
"complete list of parameters, please see model/model_params.py."))
flags.DEFINE_string(
name="vocab_file", short_name="vf", default=None,
help=flags_core.help_wrap(
"Path to subtoken vocabulary file. If data_download.py was used to "
"download and encode the training data, look in the data_dir to find "
"the vocab file."))
flags.mark_flag_as_required("vocab_file")
flags.DEFINE_string(
name="text", default=None,
help=flags_core.help_wrap(
"Text to translate. Output will be printed to console."))
flags.DEFINE_string(
name="file", default=None,
help=flags_core.help_wrap(
"File containing text to translate. Translation will be printed to "
"console and, if --file_out is provided, saved to an output file."))
flags.DEFINE_string(
name="file_out", default=None,
help=flags_core.help_wrap(
"If --file flag is specified, save translation to this file."))
from ct.crypto import cert
from ct.crypto import error
from ct.crypto import pem
from ct.crypto import verify
from ct.proto import client_pb2
from ct.serialization import tls_message
FLAGS = gflags.FLAGS
gflags.DEFINE_string("log_list", None, "File containing the list of logs "
"to submit to (see certificate-transparency.org/known-logs"
" for the format description).")
gflags.DEFINE_string("chain", None, "Certificate chain to submit (PEM).")
gflags.DEFINE_string("log_scheme", "http", "Log scheme (http/https)")
gflags.DEFINE_string("output", None, "output file for sct_list")
gflags.mark_flag_as_required("log_list")
gflags.mark_flag_as_required("chain")
gflags.mark_flag_as_required("output")
def _read_ct_log_list(log_list_file):
"""Parses the log list JSON, returns a log url to key map."""
try:
log_list_json = json.loads(log_list_file)
log_url_to_key = {}
for log_info in log_list_json['logs']:
log_url_to_key[FLAGS.log_scheme + '://' + log_info['url']] = (
base64.decodestring(log_info['key']))
return log_url_to_key
except (OSError, IOError) as io_exception:
raise Exception('Could not read log list file %s: %s' %
(log_list_file, io_exception))
logging.info('Setting up project %s', config.project['project_id'])
if not setup_project(config):
# Don't attempt to deploy additional projects if one project failed.
return
if forseti_config:
call = [
FLAGS.rule_generator_binary,
'--project_yaml_path',
FLAGS.project_yaml,
'--generated_fields_path',
FLAGS.generated_fields_path or FLAGS.project_yaml,
'--output_path',
FLAGS.output_rules_path or '',
]
logging.info('Running rule generator: %s', call)
utils.call_go_binary(call)
logging.info('All projects successfully deployed.')
if __name__ == '__main__':
flags.mark_flag_as_required('project_yaml')
flags.mark_flag_as_required('apply_binary')
flags.mark_flag_as_required('apply_forseti_binary')
flags.mark_flag_as_required('rule_generator_binary')
flags.mark_flag_as_required('load_config_binary')
flags.mark_flag_as_required('grant_forseti_access_binary')
app.run(main)
from __future__ import print_function
import os
import sys
import zipfile
# Do not edit this line. Copybara replaces it with PY2 migration helper.
from absl import app
from absl import flags
from tools.android import junction
FLAGS = flags.FLAGS
flags.DEFINE_string("input_aar", None, "Input AAR")
flags.mark_flag_as_required("input_aar")
flags.DEFINE_string("output_proguard_file", None,
"Output parameter file for proguard")
flags.mark_flag_as_required("output_proguard_file")
# Attempt to extract proguard spec from AAR. If the file doesn't exist, an empty
# proguard spec file will be created
def ExtractEmbeddedProguard(aar, output):
proguard_spec = "proguard.txt"
if proguard_spec in aar.namelist():
output.write(aar.read(proguard_spec))
def _Main(input_aar, output_proguard_file):
"ami-005bdb005fb00e791",
"ID of the AMI to use (defaults to Ubuntu 18.04)",
)
flags.DEFINE_integer("cluster_size", 0,
"size of Kubernetes cluster (0 = no cluster)")
flags.DEFINE_string("csv_path", None, "path to AWS credentials CSV")
flags.DEFINE_string("ec2_file", "~/ec2_info.txt",
"file to save EC2 info to")
flags.DEFINE_string("instance_type", "c4.xlarge",
"AWS instance type for worker nodes")
flags.DEFINE_string(
"instance_type_staging",
"c4.xlarge",
"AWS instance type for the staging machine",
)
flags.DEFINE_string("key_dir", "~/aws_keys",
"directory where AWS .pem files are stored")
flags.DEFINE_string("key_name", "ec2-keypair", "name of the .pem keypair")
flags.DEFINE_string("name", "facebook360-dep",
"name of the instance to be loaded/created")
flags.DEFINE_string("region", "us-west-2",
"region where instance will spawn")
flags.DEFINE_string("security_group", "facebook360-dep",
"name of the security group")
flags.DEFINE_string(
"tag", "", "tag of the type of render (either 'depth' or 'export')")
# Required FLAGS.
flags.mark_flag_as_required("csv_path")
app.run(main)
def launch_experiment(create_runner_fn, create_agent_fn):
"""Launches the experiment.
Args:
create_runner_fn: A function that takes as args a base directory and a
function for creating an agent and returns a `Runner` like object.
create_agent_fn: A function that takes as args a Tensorflow session and a
Gym environment, and returns an agent.
"""
run_experiment.load_gin_configs(FLAGS.gin_files, FLAGS.gin_bindings)
runner = create_runner_fn(FLAGS.base_dir,
create_agent_fn,
schedule=FLAGS.schedule)
runner.run_experiment()
def main(unused_argv):
"""This main function acts as a wrapper around a gin-configurable experiment.
Args:
unused_argv: Arguments (unused).
"""
tf.logging.set_verbosity(tf.logging.INFO)
launch_experiment(create_runner,
run_experience_replay_experiment.create_agent)
if __name__ == '__main__':
flags.mark_flag_as_required('base_dir')
app.run(main)
train_checkpointer.save(global_step=global_step_val)
if global_step_val % policy_checkpoint_interval == 0:
policy_checkpointer.save(global_step=global_step_val)
if global_step_val % rb_checkpoint_interval == 0:
rb_checkpointer.save(global_step=global_step_val)
if global_step_val % eval_interval == 0:
metric_utils.compute_summaries(
eval_metrics,
eval_py_env,
eval_py_policy,
num_episodes=num_eval_episodes,
global_step=global_step_val,
callback=eval_metrics_callback,
)
def main(_):
logging.set_verbosity(logging.INFO)
agent_class = dqn_agent.DdqnAgent if FLAGS.use_ddqn else dqn_agent.DqnAgent
train_eval(FLAGS.root_dir,
agent_class=agent_class,
num_iterations=FLAGS.num_iterations)
if __name__ == '__main__':
flags.mark_flag_as_required('root_dir')
app.run(main)
def launch_experiment(create_runner_fn, create_agent_fn):
"""Launches the experiment.
Args:
create_runner_fn: A function that takes as args a base directory and a
function for creating an agent and returns a `Runner`-like object.
create_agent_fn: A function that takes as args a Tensorflow session and an
Atari 2600 Gym environment, and returns an agent.
"""
run_experiment.load_gin_configs(FLAGS.gin_files, FLAGS.gin_bindings)
runner = create_runner_fn(FLAGS.base_dir, create_agent_fn)
runner.run_experiment()
def main(unused_argv):
"""Main method.
Args:
unused_argv: Arguments (unused).
"""
tf.logging.set_verbosity(tf.logging.INFO)
launch_experiment(create_runner, create_agent)
if __name__ == '__main__':
flags.mark_flag_as_required('agent_name')
flags.mark_flag_as_required('base_dir')
app.run(main)
eval_data = make_dataset(is_training=False, do_masking=False)
supervised_model, core_model = _get_supervised_model()
# reload the model weights (necessary because we've obliterated the masking)
checkpoint = tf.train.Checkpoint(model=supervised_model)
checkpoint.restore(saved_path).assert_existing_objects_matched()
supervised_model.compile()
with tf.io.gfile.GFile(FLAGS.prediction_file, "w") as writer:
attribute_names = ['in_test',
'length',
'expected_length',
'index']
header = "\t".join(
attribute_name for attribute_name in attribute_names) + "\n"
writer.write(header)
for f, l in eval_data:
expected_length = supervised_model.predict(f)
attributes = [l['in_test'].numpy(),
expected_length,
l['length'].numpy()]
at_df = pd.DataFrame(attributes).T
writer.write(at_df.to_csv(sep="\t", header=False))
if __name__ == '__main__':
flags.mark_flag_as_required('bert_config_file')
# flags.mark_flag_as_required('input_meta_data_path')
# flags.mark_flag_as_required('model_dir')
app.run(main)
# See the License for the specific language governing permissions and
# limitations under the License.
"""Prints all possible morphological analysis strings for a given Turkish word.
"""
from turkish_morphology import analyze
from absl import app
from absl import flags
FLAGS = flags.FLAGS
flags.DEFINE_string("word", None, "Word to morphologically analyze.")
flags.mark_flag_as_required("word")
def main(unused_argv):
analyses = analyze.surface_form(FLAGS.word)
if analyses:
print(f"Morphological analyses for the word '{FLAGS.word}':")
for analysis in analyses:
print(analysis)
else:
print(f"'{FLAGS.word}' is not accepted as a Turkish word")
if __name__ == "__main__":
app.run(main)
'between the depth and egomotion networks by using a joint '
'encoder architecture. The egomotion network is then '
'operating only on the hidden representation provided by the '
'joint encoder.')
flags.DEFINE_float('egomotion_threshold', 0.01, 'Minimum egomotion magnitude '
'to apply finetuning. If lower, just forwards the ordinary '
'prediction.')
flags.DEFINE_integer('num_steps', 20, 'Number of optimization steps to run.')
flags.DEFINE_boolean('handle_motion', True, 'Whether the checkpoint was '
'trained with motion handling.')
flags.DEFINE_bool('flip', False, 'Whether images should be flipped as well as '
'resulting predictions (for test-time augmentation). This '
'currently applies to the depth network only.')
FLAGS = flags.FLAGS
flags.mark_flag_as_required('output_dir')
flags.mark_flag_as_required('data_dir')
flags.mark_flag_as_required('model_ckpt')
flags.mark_flag_as_required('triplet_list_file')
def main(_):
"""Runs fine-tuning and inference.
There are three categories of images.
1) Images where we have previous and next frame, and that are not filtered
out by the heuristic. For them, we will use the fine-tuned predictions.
2) Images where we have previous and next frame, but that were filtered out
by our heuristic. For them, we will use the ordinary prediction instead.
3) Images where we have at least one missing adjacent frame. For them, we will
use the ordinary prediction as indicated by triplet_list_file_remains (if
sc2_env.Race._member_names_, # pylint: disable=protected-access
"Agent 2's race.")
flags.DEFINE_enum(
"difficulty",
"very_easy",
sc2_env.Difficulty._member_names_, # pylint: disable=protected-access
"If agent2 is a built-in Bot, it's strength.")
flags.DEFINE_bool("profile", False, "Whether to turn on code profiling.")
flags.DEFINE_bool("trace", False, "Whether to trace the code execution.")
flags.DEFINE_integer("parallel", 1, "How many instances to run in parallel.")
flags.DEFINE_bool("save_replay", True, "Whether to save a replay at the end.")
flags.DEFINE_string("map", None, "Name of a map to use.")
flags.mark_flag_as_required("map")
def run_thread(agent_classes, players, map_name, visualize):
"""Run one thread worth of the environment with agents."""
with sc2_env.SC2Env(
map_name=map_name,
players=players,
agent_interface_format=sc2_env.parse_agent_interface_format(
feature_screen=FLAGS.feature_screen_size,
feature_minimap=FLAGS.feature_minimap_size,
rgb_screen=FLAGS.rgb_screen_size,
rgb_minimap=FLAGS.rgb_minimap_size,
action_space=FLAGS.action_space,
use_feature_units=FLAGS.use_feature_units),
step_mul=FLAGS.step_mul,
def make_hash_fn():
session = tf.Session()
placeholder = tf.placeholder(tf.string, [])
hash_bucket = tf.strings.to_hash_bucket_fast(placeholder, 100000)
result = tf.mod(hash_bucket, 10)
def _hash_fn(x):
return session.run(result, feed_dict={placeholder: x})
return _hash_fn
def main(_):
tf.io.gfile.makedirs(FLAGS.output_dir)
resplit_nq()
resplit_wb()
resplit_ct()
if __name__ == "__main__":
flags.mark_flag_as_required("nq_train_path")
flags.mark_flag_as_required("nq_dev_path")
flags.mark_flag_as_required("wb_train_path")
flags.mark_flag_as_required("wb_test_path")
flags.mark_flag_as_required("ct_train_path")
flags.mark_flag_as_required("ct_test_path")
flags.mark_flag_as_required("output_dir")
app.run(main)
if (verify_doc_index(doc_index, fresh_doc_index) and
verify_mention_index(mention_index, fresh_mention_index) and
verify_context(mention_index, FLAGS.output_dir_text)):
print(
"Success! Found all {count} mentions and contexts across {doc_count} "
"documents. Make use of the following:\n"
"\t{m_idx}\n\t{d_idx}\n\t{txt}".format(
count=mention_index.shape[0],
doc_count=doc_index.shape[0],
m_idx=FLAGS.mention_index_path,
d_idx=FLAGS.doc_index_path,
txt=os.path.join(FLAGS.output_dir_text, "*")))
else:
print("Fail! Extracted mentions and contexts do not match all "
"expectations. (Run with --logtostderr for details.)")
else:
raise ValueError("Invalid mode %s", FLAGS.mode)
if __name__ == "__main__":
flags.mark_flag_as_required("language")
flags.mark_flag_as_required("wikinews_archive")
flags.mark_flag_as_required("output_dir_wiki")
flags.mark_flag_as_required("output_dir_text")
flags.mark_flag_as_required("doc_index_path")
flags.mark_flag_as_required("mention_index_path")
app.run(main)
flags.DEFINE_enum(
'inference_crop', INFERENCE_CROP_NONE,
[INFERENCE_CROP_NONE, INFERENCE_CROP_CITYSCAPES],
'Whether to apply a Cityscapes-specific crop on the input '
'images first before running inference.')
flags.DEFINE_bool(
'use_masks', False, 'Whether to mask out potentially '
'moving objects when feeding image input to the egomotion '
'network. This might improve odometry results when using '
'a motion model. For this, pre-computed segmentation '
'masks have to be available for every image, with the '
'background being zero.')
FLAGS = flags.FLAGS
flags.mark_flag_as_required('output_dir')
flags.mark_flag_as_required('model_ckpt')
def check_dir(dir_list):
for d in dir_list:
if not os.path.isdir(d):
os.makedirs(d)
def _run_inference(output_dir=None,
file_extension='png',
depth=True,
egomotion=False,
model_ckpt=None,
input_dir=None,
# Read the keyset into a keyset_handle.
with open(FLAGS.keyset_path, 'rt') as keyset_file:
text = keyset_file.read()
try:
keyset_handle = cleartext_keyset_handle.read(tink.JsonKeysetReader(text))
except tink.TinkError:
logging.exception('Error reading key.')
return 1
# Get the primitive.
try:
primitive = keyset_handle.primitive(hybrid.HybridDecrypt)
except tink.TinkError:
logging.exception(
'Error creating hybrid decrypt primitive from keyset.')
return 1
with open(FLAGS.input_path, 'rb') as input_file:
with open(FLAGS.output_path, 'wb') as output_file:
data = input_file.read()
plaintext = primitive.decrypt(data, context_info)
output_file.write(plaintext)
if __name__ == '__main__':
flags.mark_flag_as_required('keyset_path')
flags.mark_flag_as_required('input_path')
flags.mark_flag_as_required('output_path')
app.run(main)
predict_examples, label_list, FLAGS.max_seq_length, tokenizer,
predict_file)
logging.info("***** Running prediction*****")
logging.info(" Num examples = %d", len(predict_examples))
logging.info(" Batch size = %d", FLAGS.predict_batch_size)
predict_input_fn = file_based_input_fn_builder(
input_file=predict_file,
seq_length=FLAGS.max_seq_length,
is_training=False,
drop_remainder=False)
result = estimator.predict(input_fn=predict_input_fn)
#模型预测输出label_test.txt
output_predict_file = os.path.join(FLAGS.output_dir, "label_test.txt")
#here if the tag is "X" means it belong to its before token, here for convenient evaluate use
# conlleval.pl we discarding it directly
Writer(output_predict_file, result, batch_tokens, batch_labels,
id2label)
get_entity_for_docs(FLAGS.output_dir)
if __name__ == "__main__":
flags.mark_flag_as_required("data_dir")
flags.mark_flag_as_required("task_name")
flags.mark_flag_as_required("vocab_file")
flags.mark_flag_as_required("bert_config_file")
flags.mark_flag_as_required("output_dir")
tf.app.run()
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Script for testing a remote executor on GCP."""
from absl import app
from absl import flags
import grpc
import tensorflow_federated as tff
FLAGS = flags.FLAGS
flags.DEFINE_string('host', None, 'The host to connect to.')
flags.mark_flag_as_required('host')
flags.DEFINE_string('port', '8000', 'The port to connect to.')
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
channel = grpc.insecure_channel('{}:{}'.format(FLAGS.host, FLAGS.port))
ex = tff.framework.RemoteExecutor(channel)
ex = tff.framework.CachingExecutor(ex)
ex = tff.framework.ReferenceResolvingExecutor(ex)
factory = tff.framework.ResourceManagingExecutorFactory(lambda _: ex)
context = tff.framework.ExecutionContext(factory)
tff.framework.set_default_context(context)
for i in range(FLAGS.num_rollouts):
print('iter', i)
obs = env.reset()
done = False
totalr = 0.
steps = 0
while not done:
action = policy.act(obs)
observations.append(obs)
actions.append(action)
obs, r, done, _ = env.step(action)
time.sleep(1./100.)
totalr += r
steps += 1
if steps % 100 == 0:
print('%i/%i' % (steps, config['rollout_length']))
if steps >= config['rollout_length']:
break
returns.append(totalr)
print('returns', returns)
print('mean return', np.mean(returns))
print('std of return', np.std(returns))
if __name__ == '__main__':
flags.mark_flag_as_required('logdir')
flags.mark_flag_as_required('checkpoint')
app.run(main)
--after=/path/to/output_two.textproto
--input_type=textproto
"""
import os
from absl import app
from absl import flags
from google.protobuf import text_format
from src.main.protobuf import analysis_pb2
flags.DEFINE_string("before", None, "Aquery output before the change")
flags.DEFINE_string("after", None, "Aquery output after the change")
flags.DEFINE_enum(
"input_type", "proto", ["proto", "textproto"],
"The format of the aquery proto input. One of 'proto' and 'textproto.")
flags.mark_flag_as_required("before")
flags.mark_flag_as_required("after")
def _map_artifact_id_to_path(artifacts):
return {artifact.id: artifact.exec_path for artifact in artifacts}
def _map_output_files_to_command_line(actions, artifacts):
output_files_to_command_line = {}
for action in actions:
output_files = " ".join(
sorted([artifacts[output_id] for output_id in action.output_ids]))
output_files_to_command_line[output_files] = action.arguments
return output_files_to_command_line
from grr_response_server.bin import config_updater_keys_util
flags.DEFINE_string("dest_server_config_path", None,
"Where to write generated server configuration.")
flags.DEFINE_string("dest_client_config_path", None,
"Where to write generated client configuration.")
# We want the config writer to be a standalone executable and also to be
# importable from run_self_contained.py. As run_self_contained.py already
# defines --mysql_database, --mysql_username and --mysql_password flags,
# we avoid name clash by using the "config_" prefix.
flags.DEFINE_string("config_mysql_database", None,
"MySQL database name to use.")
flags.DEFINE_string("config_mysql_username", None, "MySQL username to use.")
flags.mark_flag_as_required("config_mysql_username")
flags.DEFINE_string("config_mysql_password", None, "MySQL password to use.")
flags.DEFINE_string("config_logging_path", None,
"Base logging path for server components to use.")
def main(argv):
del argv # Unused.
if not flags.FLAGS.dest_server_config_path:
raise ValueError("dest_server_config_path flag has to be provided.")
if not flags.FLAGS.dest_client_config_path:
raise ValueError("dest_client_config_path flag has to be provided.")
# Lint as: python3
"""TODO(rischpater): DO NOT SUBMIT without one-line documentation for python-sheets-uploader.
TODO(rischpater): DO NOT SUBMIT without a detailed description of python-sheets-uploader.
"""
SCOPES = 'https://www.googleapis.com/auth/spreadsheets'
FLAGS = flags.FLAGS
flags.DEFINE_string('spreadsheet_key', None, 'Spreadsheet key')
flags.DEFINE_string('csv_file', None, 'csv file to import')
flags.DEFINE_string('credentials', None, 'sheets credentials')
flags.mark_flag_as_required('spreadsheet_key')
flags.mark_flag_as_required('csv_file')
flags.mark_flag_as_required('credentials')
def main(argv):
SPREADSHEET_KEY = FLAGS.spreadsheet_key
CSV_FILE = FLAGS.csv_file
CREDENTIALS = FLAGS.credentials
# Authorize and get a client
creds = ServiceAccountCredentials.from_json_keyfile_name(CREDENTIALS, SCOPES)
print("authorize")
client = gspread.authorize(creds)
def define_flags():
flags.DEFINE_string('tflite_filename', None,
'File name to save tflite model.')
flags.DEFINE_string('label_filename', None, 'File name to save labels.')
flags.mark_flag_as_required('tflite_filename')
flags.mark_flag_as_required('label_filename')
(LocalSearchAlgorithm, 'local_search'),
(CorrelationClusteringOneHalfAlgorithm, 'onehalf_fair'),
(BaselineAllTogether, 'one_cluster'),
(BaselineRandomFairOneHalf, 'random_onehalf_fair')]
if num_colors > 2:
algorithms.extend([(CorrelationClusteringEqualRepresentation, 'equal_fair'),
(BaselineRandomFairEqual, 'random_equal_fair')])
for t in range(tries):
for algo, algo_label in algorithms:
seed_for_run = seed + t
results.append(RunEval(graph, num_colors, algo, algo_label, seed_for_run))
df = pd.DataFrame(results)
with open(outfile, 'w') as out_file:
df.to_json(out_file)
def main(argv=()):
del argv # Unused.
graph = ReadInput(FLAGS.input_graph, FLAGS.input_color_mapping)
num_colors = len(set([graph.nodes[n]['color'] for n in graph.nodes()]))
RunAnalysis(graph, num_colors, FLAGS.seed, FLAGS.tries, FLAGS.output_results)
if __name__ == '__main__':
flags.mark_flag_as_required('input_graph')
flags.mark_flag_as_required('input_color_mapping')
flags.mark_flag_as_required('output_results')
app.run(main)
root_title=project_full_name,
py_modules=[(project_short_name, tfnlp)],
base_dir=os.path.dirname(tfnlp.__file__),
code_url_prefix=code_url_prefix,
search_hints=search_hints,
site_path=site_path,
gen_report=gen_report,
callbacks=[public_api.explicit_package_contents_filter],
)
doc_generator.build(output_dir)
logging.info('Output docs to: %s', output_dir)
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
gen_api_docs(code_url_prefix=FLAGS.code_url_prefix,
site_path=FLAGS.site_path,
output_dir=FLAGS.output_dir,
gen_report=FLAGS.gen_report,
project_short_name=PROJECT_SHORT_NAME,
project_full_name=PROJECT_FULL_NAME,
search_hints=FLAGS.search_hints)
if __name__ == '__main__':
flags.mark_flag_as_required('output_dir')
app.run(main)
import tensorflow.compat.v2 as tf
flags.DEFINE_integer('num_models', 4, 'Size of ensemble.')
flags.DEFINE_integer('per_core_batch_size', 128,
'Batch size per TPU core/GPU.')
flags.DEFINE_float('random_sign_init', -0.5,
'Use random sign init for fast weights.')
flags.DEFINE_integer('seed', 0, 'Random seed.')
flags.DEFINE_float('base_learning_rate', 0.1,
'Base learning rate when train batch size is 256.')
flags.DEFINE_bool('version2', True, 'Use ensemble version2.')
flags.DEFINE_float('l2', 1e-4, 'L2 coefficient.')
flags.DEFINE_float('fast_weight_lr_multiplier', 0.5,
'fast weights lr multiplier.')
flags.DEFINE_string('data_dir', None, 'Path to training and testing data.')
flags.mark_flag_as_required('data_dir')
flags.DEFINE_string(
'output_dir', '/tmp/imagenet', 'The directory where the model weights and '
'training/evaluation summaries are stored.')
flags.DEFINE_integer('train_epochs', 135, 'Number of training epochs.')
flags.DEFINE_integer(
'corruptions_interval', 135,
'Number of epochs between evaluating on the corrupted '
'test data. Use -1 to never evaluate.')
flags.DEFINE_integer(
'checkpoint_interval', 27,
'Number of epochs between saving checkpoints. Use -1 to '
'never save checkpoints.')
flags.DEFINE_string('alexnet_errors_path', None,
'Path to AlexNet corruption errors file.')
flags.DEFINE_integer('num_bins', 15, 'Number of bins for ECE computation.')
FLAGS = flags.FLAGS
DATASETS = ['kitti_raw_eigen', 'kitti_raw_stereo', 'kitti_odom', 'cityscapes', 'bike', 'endo']
flags.DEFINE_enum('dataset_name', None, DATASETS, 'Dataset name.')
flags.DEFINE_string('dataset_dir', None, 'Location for dataset source files.')
flags.DEFINE_string('data_dir', None, 'Where to save the generated data.')
# Note: Training time grows linearly with sequence length. Use 2 or 3.
flags.DEFINE_integer('seq_length', 3, 'Length of each training sequence.')
flags.DEFINE_integer('img_height', 256, 'Image height.')
flags.DEFINE_integer('img_width', 256, 'Image width.')
flags.DEFINE_integer(
'num_threads', None, 'Number of worker threads. '
'Defaults to number of CPU cores.')
flags.mark_flag_as_required('dataset_name')
flags.mark_flag_as_required('dataset_dir')
flags.mark_flag_as_required('data_dir')
# Process data in chunks for reporting progress.
NUM_CHUNKS = 100
def _generate_data():
if not gfile.Exists(FLAGS.data_dir):
gfile.MakeDirs(FLAGS.data_dir)
global dataloader # pylint: disable=global-variable-undefined
if FLAGS.dataset_name == 'bike':
dataloader = dataset_loader.Bike(FLAGS.dataset_dir,
img_height=FLAGS.img_height,
img_width=FLAGS.img_width,
'converted from float into eight-bit equivalents. See '
'documentation here: '
'https://github.com/tensorflow/tensorflow/tree/master/tensorflow/tools/graph_transforms'
)
flags.DEFINE_boolean(
'channels_first', False,
'Whether channels are the first dimension of a tensor. '
'The default is TensorFlow behaviour where channels are '
'the last dimension.')
flags.DEFINE_boolean(
'output_meta_ckpt', False,
'If set to True, exports the model as .meta, .index, and '
'.data files, with a checkpoint file. These can be later '
'loaded in TensorFlow to continue training.')
flags.mark_flag_as_required('input_model')
flags.mark_flag_as_required('output_model')
def load_input_model(input_model_path,
input_json_path=None,
input_yaml_path=None,
custom_objects=None):
if not Path(input_model_path).exists():
raise FileNotFoundError(
'Model file `{}` does not exist.'.format(input_model_path))
try:
model = load_model(input_model_path, custom_objects=custom_objects)
return model
except FileNotFoundError as err:
logging.error('Input mode file (%s) does not exist.',
print("Starting epoch {}".format(epoch))
session.run(train_iterator_init)
while True:
try:
session.run(dist_train)
except tf.errors.OutOfRangeError:
break
print("Training loss: {:0.4f}, accuracy: {:0.2f}%".format(
session.run(training_loss_result),
session.run(training_accuracy_result) * 100))
training_loss.reset_states()
training_accuracy.reset_states()
# Test
session.run(test_iterator_init)
while True:
try:
session.run(dist_test)
except tf.errors.OutOfRangeError:
break
print("Test loss: {:0.4f}, accuracy: {:0.2f}%".format(
session.run(test_loss_result),
session.run(test_accuracy_result) * 100))
test_loss.reset_states()
test_accuracy.reset_states()
if __name__ == "__main__":
flags.mark_flag_as_required("tpu")
app.run(main)
def main(_):
# Read WAV file.
samples, sample_rate_hz = wav_io.read_wav_file(FLAGS.input,
dtype=np.float32)
samples = samples.mean(axis=1)
# Make the frontend and network.
frontend = carl_frontend.CarlFrontend(input_sample_rate_hz=sample_rate_hz)
phone_net, target_names = get_phone_net(FLAGS.model)
# Run audio-to-phone inference.
frames = phone_util.run_frontend(frontend, samples)
frame_rate = sample_rate_hz / frontend.block_size
scores = phone_net(frames)
fig = plot_output(frames, frame_rate, scores, target_names,
os.path.basename(FLAGS.input) + '\n' + FLAGS.model)
if FLAGS.output: # Save plot as an image file.
plot.save_figure(FLAGS.output, fig)
else: # Show plot interactively.
plt.show()
return 0
if __name__ == '__main__':
flags.mark_flag_as_required('input')
app.run(main)
gin.parse_config_files_and_bindings(gin_files, args.gin_bindings)
# TODO: do this the other way around - put these as gin bindings
if not args.traj_len:
args.traj_len = int(gin.query_parameter('AdvantageActorCriticAgent.traj_len'))
if not args.batch_sz:
args.batch_sz = int(gin.query_parameter('AdvantageActorCriticAgent.batch_sz'))
env_cls = rvr.envs.GymEnv if '-v' in args.env else rvr.envs.SC2Env
env = env_cls(args.env, args.render, max_ep_len=args.max_ep_len)
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
sess_mgr = rvr.utils.tensorflow.SessionManager(sess, expt.path, args.ckpt_freq, training_enabled=not args.test)
agent = agent_cls[args.agent](env.obs_spec(), env.act_spec(), sess_mgr=sess_mgr,
n_envs=args.envs, traj_len=args.traj_len, batch_sz=args.batch_sz)
agent.logger = rvr.utils.StreamLogger(args.envs, args.log_freq, args.eps_avg, sess_mgr, expt.log_path)
if sess_mgr.training_enabled:
expt.save_gin_config()
expt.save_model_summary(agent.model)
agent.run(env, args.updates * args.traj_len * args.batch_sz // args.envs)
if __name__ == '__main__':
flags.mark_flag_as_required('env')
app.run(main)
FLAGS = flags.FLAGS
flags.DEFINE_string('split', 'test',
'The dataset split on which to run inference.')
flags.DEFINE_string('dataset', 'shapenet-autoencoder',
'The input data+GT pair to run on.')
flags.DEFINE_string('category', 'all', 'The category to run on.')
flags.DEFINE_integer('instance_count', '-1',
'The number of instances to run on. -1 means all.')
flags.DEFINE_string('experiment_name', None,
'The name of the experiment to evaluate.')
flags.mark_flag_as_required('experiment_name')
flags.DEFINE_string('xids', '', 'XManager ids to evaluate.')
flags.DEFINE_string('model_name', 'sif-transcoder',
'The model name (usually sif-transcoder).')
flags.DEFINE_string('out_dir', '', 'The root directory for the out path.')
flags.DEFINE_integer('resolution', 256, 'The marching cubes resolution to use.')
flags.DEFINE_integer('ckpt', 0, 'The checkpoint index.')
flags.DEFINE_string(
'xid_to_ckpt', '',
'A (xid1, ckpt1, xid2, ckpt2, ...) comma separated list of '
'ob_dim': ob_dim,
'ac_dim': ac_dim
}
ARS = ars.ARSLearner(
env_callback=config['env'],
policy_params=policy_params,
num_deltas=config['num_directions'],
deltas_used=config['deltas_used'],
step_size=config['step_size'],
delta_std=config['delta_std'],
logdir=logdir,
rollout_length=config['rollout_length'],
shift=config['shift'],
params=config,
seed=config['seed'])
return ARS.train(config['num_iterations'])
def main(argv):
del argv # Unused.
config = getattr(config_ars, FLAGS.config_name)
run_ars(config=config, logdir=FLAGS.logdir)
if __name__ == '__main__':
flags.mark_flag_as_required('logdir')
flags.mark_flag_as_required('config_name')
app.run(main)
from absl import flags
from absl import logging
import numpy as np
import tensorflow.compat.v1 as tf
from stacked_capsule_autoencoders.capsules.configs import data_config
from stacked_capsule_autoencoders.capsules.configs import model_config
from stacked_capsule_autoencoders.capsules.train import create_hooks
from stacked_capsule_autoencoders.capsules.train import tools
flags.DEFINE_string('dataset', 'mnist', 'Choose from: {mnist, constellation.}')
flags.DEFINE_string('model', 'scae', 'Choose from {scae, constellation}.')
flags.DEFINE_string('name', None, '')
flags.mark_flag_as_required('name')
flags.DEFINE_string('logdir',
'stacked_capsule_autoencoders/checkpoints/{name}',
'Log and checkpoint directory for the experiment.')
flags.DEFINE_float('grad_value_clip', 0., '')
flags.DEFINE_float('grad_norm_clip', 0., '')
flags.DEFINE_float(
'ema', .9, 'Exponential moving average weight for smoothing '
'reported results.')
flags.DEFINE_integer('run_updates_every', 10, '')
flags.DEFINE_boolean('global_ema_update', True, '')
HOME_DIR = os.path.expanduser('~')
DEFAULT_OUTPUT_DIR = os.path.join(HOME_DIR, 'vid2depth/inference')
DEFAULT_KITTI_DIR = os.path.join(HOME_DIR, 'kitti-raw-uncompressed')
flags.DEFINE_string('output_dir', DEFAULT_OUTPUT_DIR,
'Directory to store estimated depth maps.')
flags.DEFINE_string('kitti_dir', DEFAULT_KITTI_DIR, 'KITTI dataset directory.')
flags.DEFINE_string('model_ckpt', None, 'Model checkpoint to load.')
flags.DEFINE_string('kitti_video', None, 'KITTI video directory name.')
flags.DEFINE_integer('batch_size', 4, 'The size of a sample batch.')
flags.DEFINE_integer('img_height', 128, 'Image height.')
flags.DEFINE_integer('img_width', 416, 'Image width.')
flags.DEFINE_integer('seq_length', 3, 'Sequence length for each example.')
FLAGS = flags.FLAGS
flags.mark_flag_as_required('kitti_video')
flags.mark_flag_as_required('model_ckpt')
CMAP = 'plasma'
def _run_inference():
"""Runs all images through depth model and saves depth maps."""
ckpt_basename = os.path.basename(FLAGS.model_ckpt)
ckpt_modelname = os.path.basename(os.path.dirname(FLAGS.model_ckpt))
output_dir = os.path.join(FLAGS.output_dir,
FLAGS.kitti_video.replace('/', '_') + '_' +
ckpt_modelname + '_' + ckpt_basename)
if not gfile.Exists(output_dir):
gfile.MakeDirs(output_dir)
inference_model = model.Model(is_training=False,
import smooth_sensitivity as pate_ss
plt.style.use('ggplot')
FLAGS = flags.FLAGS
flags.DEFINE_boolean('cache', False,
'Read results of privacy analysis from cache.')
flags.DEFINE_string('counts_file', None, 'Counts file.')
flags.DEFINE_string('figures_dir', '', 'Path where figures are written to.')
flags.DEFINE_float('threshold', None, 'Threshold for step 1 (selection).')
flags.DEFINE_float('sigma1', None, 'Sigma for step 1 (selection).')
flags.DEFINE_float('sigma2', None, 'Sigma for step 2 (argmax).')
flags.DEFINE_integer('queries', None, 'Number of queries made by the student.')
flags.DEFINE_float('delta', 1e-8, 'Target delta.')
flags.mark_flag_as_required('counts_file')
flags.mark_flag_as_required('threshold')
flags.mark_flag_as_required('sigma1')
flags.mark_flag_as_required('sigma2')
Partition = namedtuple('Partition', ['step1', 'step2', 'ss', 'delta'])
def analyze_gnmax_conf_data_ind(votes, threshold, sigma1, sigma2, delta):
orders = np.logspace(np.log10(1.5), np.log10(500), num=100)
n = votes.shape[0]
rdp_total = np.zeros(len(orders))
answered_total = 0
answered = np.zeros(n)
eps_cum = np.full(n, None, dtype=float)
'order to avoid overly penalizing dis-occlusion effects.')
flags.DEFINE_bool('use_skip', True, 'Whether to use skip connections in the '
'encoder-decoder architecture.')
flags.DEFINE_bool('equal_weighting', False, 'Whether to use equal weighting '
'of the smoothing loss term, regardless of resolution.')
flags.DEFINE_bool('joint_encoder', False, 'Whether to share parameters '
'between the depth and egomotion networks by using a joint '
'encoder architecture. The egomotion network is then '
'operating only on the hidden representation provided by the '
'joint encoder.')
flags.DEFINE_bool('handle_motion', True, 'Whether to try to handle motion by '
'using the provided segmentation masks.')
flags.DEFINE_string('master', 'local', 'Location of the session.')
FLAGS = flags.FLAGS
flags.mark_flag_as_required('data_dir')
flags.mark_flag_as_required('checkpoint_dir')
def main(_):
# Fixed seed for repeatability
seed = 8964
tf.set_random_seed(seed)
np.random.seed(seed)
random.seed(seed)
if FLAGS.handle_motion and FLAGS.joint_encoder:
raise ValueError('Using a joint encoder is currently not supported when '
'modeling object motion.')
if FLAGS.handle_motion and FLAGS.seq_length != 3:
raise ValueError('The current motion model implementation only supports '
from absl import app
from absl import flags
from pysc2.lib import gfile
from s2clientprotocol import common_pb2 as sc_common
from s2clientprotocol import sc2api_pb2 as sc_pb
import numpy as np
import json as json
PROJ_DIR = os.path.dirname(os.path.abspath(__file__))
FLAGS = flags.FLAGS
flags.DEFINE_integer("parallel", 1, "How many instances to run in parallel.")
flags.DEFINE_integer("step_mul", 8, "How many game steps per observation.")
flags.DEFINE_string("replays", "%s/replays/mineral1.SC2Replay" % PROJ_DIR, "Path to a directory of replays.")
flags.mark_flag_as_required("replays")
size = point.Point(32, 32)
interface = sc_pb.InterfaceOptions(
raw=True, score=False,
feature_layer=sc_pb.SpatialCameraSetup(width=24))
size.assign_to(interface.feature_layer.resolution)
size.assign_to(interface.feature_layer.minimap_resolution)
def sorted_dict_str(d):
return "{%s}" % ", ".join("%s: %s" % (k, d[k])
for k in sorted(d, key=d.get, reverse=True))
DATASETS = [
'kitti_raw_eigen', 'kitti_raw_stereo', 'kitti_odom', 'cityscapes', 'bike'
]
flags.DEFINE_enum('dataset_name', None, DATASETS, 'Dataset name.')
flags.DEFINE_string('dataset_dir', None, 'Location for dataset source files.')
flags.DEFINE_string('data_dir', None, 'Where to save the generated data.')
# Note: Training time grows linearly with sequence length. Use 2 or 3.
flags.DEFINE_integer('seq_length', 3, 'Length of each training sequence.')
flags.DEFINE_integer('img_height', 128, 'Image height.')
flags.DEFINE_integer('img_width', 416, 'Image width.')
flags.DEFINE_integer(
'num_threads', None, 'Number of worker threads. '
'Defaults to number of CPU cores.')
flags.mark_flag_as_required('dataset_name')
flags.mark_flag_as_required('dataset_dir')
flags.mark_flag_as_required('data_dir')
# Process data in chunks for reporting progress.
NUM_CHUNKS = 100
def _generate_data():
"""Extract sequences from dataset_dir and store them in data_dir."""
if not gfile.Exists(FLAGS.data_dir):
gfile.MakeDirs(FLAGS.data_dir)
global dataloader # pylint: disable=global-variable-undefined
if FLAGS.dataset_name == 'bike':
dataloader = dataset_loader.Bike(FLAGS.dataset_dir,
import os
import sys
from absl import app
from absl import flags
from google.protobuf import text_format
from src.main.protobuf import analysis_pb2
from tools.aquery_differ.resolvers.dep_set_resolver import DepSetResolver
flags.DEFINE_string("before", None, "Aquery output before the change")
flags.DEFINE_string("after", None, "Aquery output after the change")
flags.DEFINE_enum(
"input_type", "proto", ["proto", "textproto"],
"The format of the aquery proto input. One of 'proto' and 'textproto.")
flags.DEFINE_multi_enum("attrs", ["cmdline"], ["inputs", "cmdline"],
"Attributes of the actions to be compared.")
flags.mark_flag_as_required("before")
flags.mark_flag_as_required("after")
WHITE = "\033[37m%s\033[0m"
CYAN = "\033[36m%s\033[0m"
RED = "\033[31m%s\033[0m"
GREEN = "\033[32m%s\033[0m"
def _colorize(line):
"""Add color to the input string."""
if not sys.stdout.isatty():
return line
if line.startswith("+++") or line.startswith("---"):
return WHITE % line
input_meta_data = json.loads(reader.read().decode('utf-8'))
if not FLAGS.model_dir:
FLAGS.model_dir = '/tmp/bert20/'
strategy = distribution_utils.get_distribution_strategy(
distribution_strategy=FLAGS.distribution_strategy,
num_gpus=FLAGS.num_gpus,
tpu_address=FLAGS.tpu)
max_seq_length = input_meta_data['max_seq_length']
train_input_fn = get_dataset_fn(FLAGS.train_data_path,
max_seq_length,
FLAGS.train_batch_size,
is_training=True)
eval_input_fn = get_dataset_fn(FLAGS.eval_data_path,
max_seq_length,
FLAGS.eval_batch_size,
is_training=False)
bert_config = bert_configs.BertConfig.from_json_file(
FLAGS.bert_config_file)
run_bert(strategy, input_meta_data, bert_config, train_input_fn,
eval_input_fn)
if __name__ == '__main__':
flags.mark_flag_as_required('bert_config_file')
flags.mark_flag_as_required('input_meta_data_path')
flags.mark_flag_as_required('model_dir')
app.run(main)
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from absl import flags
import language.labs.consistent_zero_shot_nmt.data_generators.translate_europarl # pylint: disable=unused-import
import language.labs.consistent_zero_shot_nmt.data_generators.translate_iwslt17 # pylint: disable=unused-import
import language.labs.consistent_zero_shot_nmt.data_generators.translate_uncorpus # pylint: disable=unused-import
import language.labs.consistent_zero_shot_nmt.models.agreement # pylint: disable=unused-import
import language.labs.consistent_zero_shot_nmt.models.basic # pylint: disable=unused-import
import language.labs.consistent_zero_shot_nmt.utils.t2t_tweaks # pylint: disable=unused-import
from tensor2tensor.bin import t2t_trainer
import tensorflow.compat.v1 as tf
# This import triggers the decorations that register the problems.
FLAGS = flags.FLAGS
def main(argv):
if getattr(FLAGS, "brain_jobs", None):
FLAGS.worker_job = "/job:%s" % FLAGS.brain_job_name
return t2t_trainer.main(argv)
if __name__ == "__main__":
flags.mark_flag_as_required("data_dir")
tf.app.run()
flags.DEFINE_integer(
'layer_idx', 30,
'Layer index from the VGG-16 model '
'to be used for extracting proposals')
flags.DEFINE_enum(
'dataset', None, ['awa2', 'cub'],
'Dataset for analysis')
flags.DEFINE_list(
'class_ids', None,
'Comma separated class IDs')
flags.DEFINE_integer(
'num_max_proposals', 5,
'Max. number of proposals '
'to be extracted for each image')
flags.mark_flag_as_required('dataset')
flags.mark_flag_as_required('class_ids')
_make_cuda = lambda x: x.cuda() if torch.cuda.is_available() else x
class Context:
def __init__(self, model, dataset, layer_idx):
self.model = model
self.dataset = dataset
self._layer_idx = layer_idx
self._model_out_dict = dict()
def get_model_output(self, image_idx):
if image_idx not in self._model_out_dict:
Code to visualize images in the unsupervised dataset, and views created of them.
"""
from functools import partial
import tensorflow as tf
from absl import flags
import csf.data
import csf.train
import csf.utils
flags.DEFINE_list(
"visualize_bands", None, "Bands to visualize. Should be grouped into blocks of 3."
)
flags.mark_flag_as_required("visualize_bands")
flags.DEFINE_integer("max_pages", 100, "Maximum number of summary pages to create.")
flags.DEFINE_integer("images_per_page", 1, "Number of summaries on a single page.")
flags.DEFINE_integer("views", 2, "Number of views to show summaries for.")
FLAGS = flags.FLAGS
def visualize_dataset():
FLAGS.batch_size = FLAGS.images_per_page # Prevent loading more bands than we plot
csf.distribution.initialize()
dataset = csf.data.load_dataset()
summary_writer = tf.summary.create_file_writer(FLAGS.out_dir)
page = tf.Variable(0, trainable=False, name="step", dtype=tf.dtypes.int64)
tf.summary.experimental.set_step(page)
def define_flags():
flags.DEFINE_string('export_dir', None,
'The directory to save exported files.')
flags.DEFINE_string('spec', 'efficientnet_lite0',
'The image classifier to run.')
flags.mark_flag_as_required('export_dir')
FLAGS = flags.FLAGS
def main(argv):
"""Tars a frame.
Args:
argv (list[str]): List of arguments (used interally by abseil).
"""
frame_tar_fn = os.path.join(FLAGS.src, f"{FLAGS.frame}.tar")
tar_file = tarfile.open(frame_tar_fn, "w")
print(f"Creating {frame_tar_fn}...")
for root, _, files in os.walk(FLAGS.src):
for file in files:
if FLAGS.frame in file:
arcname = os.path.join(os.path.basename(root), file)
tar_file.add(os.path.join(root, file), arcname=arcname)
tar_file.close()
if __name__ == "__main__":
flags.DEFINE_string("src", None, "Path to the directory to be packed")
flags.DEFINE_string("frame", None,
"Name of the frame (6 digit, zero padded)")
# Required FLAGS.
flags.mark_flag_as_required("src")
flags.mark_flag_as_required("frame")
app.run(main)
import matplotlib
matplotlib.use('TkAgg')
import matplotlib.pyplot as plt # pylint: disable=g-import-not-at-top
import numpy as np
import core as pate
plt.style.use('ggplot')
FLAGS = flags.FLAGS
flags.DEFINE_boolean('cache', False,
'Read results of privacy analysis from cache.')
flags.DEFINE_string('counts_file', None, 'Counts file.')
flags.DEFINE_string('figures_dir', '', 'Path where figures are written to.')
flags.mark_flag_as_required('counts_file')
def run_analysis(votes, mechanism, noise_scale, params):
"""Computes data-dependent privacy.
Args:
votes: A matrix of votes, where each row contains votes in one instance.
mechanism: A name of the mechanism ('lnmax', 'gnmax', or 'gnmax_conf')
noise_scale: A mechanism privacy parameter.
params: Other privacy parameters.
Returns:
Four lists: cumulative privacy cost epsilon, how privacy budget is split,
how many queries were answered, optimal order.
"""
'triplets instead of single frames.')
flags.DEFINE_enum('inference_crop', INFERENCE_CROP_NONE,
[INFERENCE_CROP_NONE,
INFERENCE_CROP_CITYSCAPES],
'Whether to apply a Cityscapes-specific crop on the input '
'images first before running inference.')
flags.DEFINE_bool('use_masks', False, 'Whether to mask out potentially '
'moving objects when feeding image input to the egomotion '
'network. This might improve odometry results when using '
'a motion model. For this, pre-computed segmentation '
'masks have to be available for every image, with the '
'background being zero.')
FLAGS = flags.FLAGS
flags.mark_flag_as_required('output_dir')
flags.mark_flag_as_required('model_ckpt')
def _run_inference(output_dir=None,
file_extension='png',
depth=True,
egomotion=False,
model_ckpt=None,
input_dir=None,
input_list_file=None,
batch_size=1,
img_height=128,
img_width=416,
seq_length=3,
architecture=nets.RESNET,
