def define_flags():
"""Construct flags for the server."""
flags.DEFINE_integer(name="num_workers", default=multiprocessing.cpu_count(),
help="Size of the negative generation worker pool.")
flags.DEFINE_string(name="data_dir", default=None,
help="The data root. (used to construct cache paths.)")
flags.DEFINE_string(name="cache_id", default=None,
help="The cache_id generated in the main process.")
flags.DEFINE_integer(name="num_readers", default=4,
help="Number of reader datasets in training. This sets"
"how the epoch files are sharded.")
flags.DEFINE_integer(name="num_neg", default=None,
help="The Number of negative instances to pair with a "
"positive instance.")
flags.DEFINE_integer(name="num_train_positives", default=None,
help="The number of positive training examples.")
flags.DEFINE_integer(name="num_items", default=None,
help="Number of items from which to select negatives.")
flags.DEFINE_integer(name="num_users", default=None,
help="The number of unique users. Used for evaluation.")
flags.DEFINE_integer(name="epochs_per_cycle", default=1,
help="The number of epochs of training data to produce"
"at a time.")
flags.DEFINE_integer(name="num_cycles", default=None,
help="The number of cycles to produce training data "
"for.")
flags.DEFINE_integer(name="train_batch_size", default=None,
help="The batch size with which training TFRecords will "
"be chunked.")
flags.DEFINE_integer(name="eval_batch_size", default=None,
help="The batch size with which evaluation TFRecords "
"will be chunked.")
flags.DEFINE_boolean(name="redirect_logs", default=False,
help="Catch logs and write them to a file. "
"(Useful if this is run as a subprocess)")
flags.DEFINE_boolean(name="use_tf_logging", default=False,
help="Use tf.logging instead of log file.")
flags.DEFINE_integer(name="seed", default=None,
help="NumPy random seed to set at startup. If not "
"specified, a seed will not be set.")
flags.DEFINE_boolean(name="ml_perf", default=None,
help="Match MLPerf. See ncf_main.py for details.")
flags.DEFINE_bool(name="output_ml_perf_compliance_logging", default=None,
help="Output the MLPerf compliance logging. See "
"ncf_main.py for details.")
flags.mark_flags_as_required(["data_dir", "cache_id"])
def define_flags():
"""Construct flags for the server.
This function does not use offical.utils.flags, as these flags are not meant
to be used by humans. Rather, they should be passed as part of a subprocess
call.
"""
flags.DEFINE_integer(name="num_workers", default=multiprocessing.cpu_count(),
help="Size of the negative generation worker pool.")
flags.DEFINE_string(name="data_dir", default=None,
help="The data root. (used to construct cache paths.)")
flags.DEFINE_string(name="cache_id", default=None,
help="The cache_id generated in the main process.")
flags.DEFINE_integer(name="num_readers", default=4,
help="Number of reader datasets in training. This sets"
"how the epoch files are sharded.")
flags.DEFINE_integer(name="num_neg", default=None,
help="The Number of negative instances to pair with a "
"positive instance.")
flags.DEFINE_integer(name="num_train_positives", default=None,
help="The number of positive training examples.")
flags.DEFINE_integer(name="num_items", default=None,
help="Number of items from which to select negatives.")
flags.DEFINE_integer(name="epochs_per_cycle", default=1,
help="The number of epochs of training data to produce"
"at a time.")
flags.DEFINE_integer(name="train_batch_size", default=None,
help="The batch size with which training TFRecords will "
"be chunked.")
flags.DEFINE_integer(name="eval_batch_size", default=None,
help="The batch size with which evaluation TFRecords "
"will be chunked.")
flags.DEFINE_boolean(
name="spillover", default=True,
help="If a complete batch cannot be provided, return an empty batch and "
"start the next epoch from a non-empty buffer. This guarantees "
"fixed batch sizes.")
flags.DEFINE_boolean(name="redirect_logs", default=False,
help="Catch logs and write them to a file. "
"(Useful if this is run as a subprocess)")
flags.DEFINE_integer(name="seed", default=None,
help="NumPy random seed to set at startup. If not "
"specified, a seed will not be set.")
flags.mark_flags_as_required(
["data_dir", "cache_id", "num_neg", "num_train_positives", "num_items",
"train_batch_size", "eval_batch_size"])
from clu import platform
import train
import jax
from ml_collections import config_flags
import tensorflow as tf
FLAGS = flags.FLAGS
flags.DEFINE_string('workdir', None, 'Directory to store model data.')
config_flags.DEFINE_config_file(
'config',
'configs/default.py',
'File path to the training hyperparameter configuration.',
lock_config=True)
flags.mark_flags_as_required(['config', 'workdir'])
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
# Hide any GPUs form TensorFlow. Otherwise TF might reserve memory and make
# it unavailable to JAX.
tf.config.experimental.set_visible_devices([], 'GPU')
logging.info('JAX process: %d / %d', jax.process_index(),
jax.process_count())
logging.info('JAX local devices: %r', jax.local_devices())
# Add a note so that we can tell which task is which JAX host.
else:
assert FLAGS.schedule == "continuous_eval"
eval_args = _create_eval_args(config_overrides.get("dataset"))
for _ in estimator_runner.continuous_eval(
estimator=estimator, eval_args=eval_args,
train_steps=FLAGS.train_steps):
# continuous_train_and_eval() yields evaluation metrics after each
# checkpoint. It also saves and logs them, so we don't do anything here.
pass
if __name__ == "__main__":
tf.logging.set_verbosity(tf.logging.INFO)
flags.mark_flags_as_required(["dataset", "model_dir", "schedule"])
def _validate_schedule(flag_values):
"""Validates the --schedule flag and the flags it interacts with."""
schedule = flag_values["schedule"]
save_checkpoints_steps = flag_values["save_checkpoints_steps"]
save_checkpoints_secs = flag_values["save_checkpoints_secs"]
if schedule in ["train", "train_and_eval"]:
if not (save_checkpoints_steps or save_checkpoints_secs):
raise flags.ValidationError(
"--schedule='%s' requires --save_checkpoints_steps or "
"--save_checkpoints_secs." % schedule)
return True
keyset_handle = cleartext_keyset_handle.read(tink.JsonKeysetReader(text))
except tink.TinkError as e:
logging.exception('Error reading keyset: %s', e)
return 1
now = datetime.datetime.now(tz=datetime.timezone.utc)
# Get the JwtPublicKeySign primitive
try:
jwt_sign = keyset_handle.primitive(jwt.JwtPublicKeySign)
except tink.TinkError as e:
logging.exception('Error creating JwtPublicKeySign: %s', e)
return 1
# Create token
raw_jwt = jwt.new_raw_jwt(
audiences=[_AUDIENCE.value],
expiration=now + datetime.timedelta(seconds=100))
token = jwt_sign.sign_and_encode(raw_jwt)
with open(_TOKEN_PATH.value, 'wt') as token_file:
token_file.write(token)
logging.info('Token has been written to %s', _TOKEN_PATH.value)
if __name__ == '__main__':
flags.mark_flags_as_required(
['private_keyset_path', 'audience', 'token_path'])
app.run(main)
# [END python-jwt-sign-example]
maxlen=FLAGS.maxlen)
basemodel = BaseModel(input_dim=FLAGS.max_features,
maxlen=FLAGS.maxlen)
model = multi_gpu_model(basemodel.build(name=FLAGS.model_name),
gpus=len(get_available_gpus()))
model.compile(loss='binary_crossentropy',
optimizer=FLAGS.optimizer,
metrics=['accuracy'])
model.fit(
X_train.todense(),
y_train,
validation_data=(X_val.todense(), y_val),
batch_size=FLAGS.batch_size,
epochs=FLAGS.epochs,
callbacks=[
ModelCheckpointToGcs(
remotekey=FLAGS.gcs_path,
filename=
'model_checkpoint.{epoch:02d}_{loss:.6f}_{val_loss:.6f}.h5',
save_best_only=True),
CSVLoggerToGcs(remotekey=FLAGS.gcs_path,
filename=path.join('history_{}_{}.tsv').format(
FLAGS.model_name, FLAGS.optimizer),
separator='\t'),
])
if __name__ == '__main__':
flags.mark_flags_as_required(['model_name', 'gcs_path'])
app.run(main)
logging.info('mode %r num_frames %r feature shape %r', FLAGS.mode,
params.task.model.backbone.assemblenet.num_frames, shape)
# Sets mixed_precision policy. Using 'mixed_float16' or 'mixed_bfloat16'
# can have significant impact on model speeds by utilizing float16 in case of
# GPUs, and bfloat16 in the case of TPUs. loss_scale takes effect only when
# dtype is float16
if params.runtime.mixed_precision_dtype:
performance.set_mixed_precision_policy(params.runtime.mixed_precision_dtype)
distribution_strategy = distribute_utils.get_distribution_strategy(
distribution_strategy=params.runtime.distribution_strategy,
all_reduce_alg=params.runtime.all_reduce_alg,
num_gpus=params.runtime.num_gpus,
tpu_address=params.runtime.tpu)
with distribution_strategy.scope():
task = task_factory.get_task(params.task, logging_dir=model_dir)
train_lib.run_experiment(
distribution_strategy=distribution_strategy,
task=task,
mode=FLAGS.mode,
params=params,
model_dir=model_dir)
train_utils.save_gin_config(FLAGS.mode, model_dir)
if __name__ == '__main__':
tfm_flags.define_flags()
flags.mark_flags_as_required(['experiment', 'mode', 'model_dir'])
app.run(main)
from absl import flags
import jax
from ml_collections import config_flags
import tensorflow as tf
from vdvae_flax import experiment
FLAGS = flags.FLAGS
config_flags.DEFINE_config_file("config",
None,
"Training configuration.",
lock_config=True)
flags.DEFINE_string("workdir", None, "Work unit directory.")
flags.mark_flags_as_required(["config", "workdir"])
def main(argv):
del argv
# Hide any GPUs form TensorFlow. Otherwise TF might reserve memory and make
# it unavailable to JAX.
tf.config.experimental.set_visible_devices([], "GPU")
exp = experiment.Experiment("train", FLAGS.config)
exp.train_and_evaluate(FLAGS.workdir)
if __name__ == "__main__":
jax.config.config_with_absl()
logging.exception('Error reading key: %s', e)
return 1
# Get the primitive
try:
cipher = keyset_handle.primitive(aead.Aead)
except tink.TinkError as e:
logging.error('Error creating primitive: %s', e)
return 1
with open(FLAGS.input_path, 'rb') as input_file:
input_data = input_file.read()
if FLAGS.mode == 'decrypt':
output_data = cipher.decrypt(input_data, b'envelope_example')
elif FLAGS.mode == 'encrypt':
output_data = cipher.encrypt(input_data, b'envelope_example')
else:
logging.error(
'Error mode not supported. Please choose "encrypt" or "decrypt".'
)
return 1
with open(FLAGS.output_path, 'wb') as output_file:
output_file.write(output_data)
if __name__ == '__main__':
flags.mark_flags_as_required(['mode', 'keyset_path'])
app.run(main)
# [END cleartext-keyset-example]
model = tf.keras.models.load_model(FLAGS.saved_model_dir,
compile=False)
logging.info('Model initialized. Beginning prediction loop.')
logits_fn = tf.function(
functools.partial(prediction.distributed_logits_fn, model))
decode_logits_fn = tf.function(
functools.partial(prediction.decode_logits, FLAGS.decode_top_k,
FLAGS.decode_max_size))
split_and_pad_fn = tf.function(
functools.partial(prediction.split_and_pad, strategy,
FLAGS.batch_size))
# Prediction strategy.
predict_fn = functools.partial(predict,
sp_processor=sp_processor,
features_map_fn=features_map_fn,
logits_fn=logits_fn,
decode_logits_fn=decode_logits_fn,
split_and_pad_fn=split_and_pad_fn,
distribute_strategy=strategy,
dataset=dataset)
with worker_context():
predictions = predict_fn()
with tf.io.gfile.GFile(FLAGS.predictions_path, 'w') as f:
json.dump(predictions, f)
if __name__ == '__main__':
flags.mark_flags_as_required(
['split', 'predictions_path', 'saved_model_dir'])
app.run(main)
with tf.compat.v1.Session() as sess:
sess.run(tf.compat.v1.global_variables_initializer())
load_ema(sess)
graph_def = sess.graph.as_graph_def()
graph_def = tf.compat.v1.graph_util.convert_variables_to_constants(
sess, graph_def, [out_node])
graph_def = optimize_for_inference_lib.optimize_for_inference(
graph_def, [in_node], [out_node], tf.float32.as_datatype_enum)
with tf.io.gfile.GFile(openvino_model_pb, 'wb') as f:
f.write(graph_def.SerializeToString())
def main(_):
tensor_shape = [FLAGS.height, FLAGS.width, FLAGS.channels]
logging.info('Processing ckpt=%s, tensor_shape=%s.', FLAGS.checkpoint,
tensor_shape)
freeze_graph(modeling.get_model('inception_v3'), FLAGS.checkpoint,
tensor_shape, FLAGS.output)
logging.info('Output written to %s.', FLAGS.output)
if __name__ == '__main__':
flags.mark_flags_as_required([
'checkpoint',
'output',
])
tf.compat.v1.app.run()
ds = json.load(open(filename))
except:
print('Cannot read {filename}'.format(filename=filename))
quit()
epoch = max(map(int, ds['epoch'].keys()))
epoch = min(FLAGS.epoch or epoch, epoch)
df = {}
for metric in FLAGS.metrics:
data = ds['epoch'][str(
epoch)][0]['test'][metric]['classification_report']
df[metric] = pd.read_table(io.StringIO(data),
delimiter=r'\s{2,}',
engine='python')
return types.SimpleNamespace(df=df, epoch=epoch)
def main(unused_argv):
for dataset in FLAGS.datasets:
res = df_for_dataset(dataset)
print('dataset={dataset} epoch={epoch}'.format(dataset=dataset,
epoch=res.epoch))
for d in res.df.values():
print(d.to_csv() if FLAGS.csv else d)
if FLAGS.shell:
IPython.start_ipython(argv=[], user_ns=dict(globals(), **locals()))
if __name__ == '__main__':
flags.mark_flags_as_required(['datasets'])
app.run(main)
try:
start_timestamp = time.time(
) # This time will include compilation time
eval_results = estimator.evaluate(input_fn=eval_input_fn,
steps=max_eval_steps,
checkpoint_path=ckpt)
elapsed_time = int(time.time() - start_timestamp)
logging.info("Eval results: %s. Elapsed seconds: %d",
eval_results, elapsed_time)
# Terminate eval job when final checkpoint is reached.
current_step = int(os.path.basename(ckpt).split("-")[1])
if current_step >= FLAGS.num_train_steps:
logging.info("Evaluation finished after training step %d",
current_step)
break
except tf.errors.NotFoundError:
# Since the coordinator is on a different job than the TPU worker,
# sometimes the TPU worker does not finish initializing until long after
# the CPU job tells it to start evaluating. In this case, the checkpoint
# file could have been deleted already.
logging.info(
"Checkpoint %s no longer exists, skipping checkpoint",
ckpt)
if __name__ == "__main__":
flags.mark_flags_as_required(["input_file", "output_dir"])
app.run(main)
'positional arguments but some are present on the command line: '
'"{}".'.format(str(argv[1:])), errors.CommandLineError)
with vcf.VcfReader(FLAGS.input_vcf) as reader:
sample_names = reader.header.sample_names
if len(sample_names) != 1:
raise ValueError(
'There must be exactly one sample in VCF: {}'.format(
FLAGS.input_vcf))
sample_name = sample_names[0]
# Missing GT causes error later while reading, so throw a clearer error here
vcf_columns = [col.id for col in reader.header.formats]
if 'GT' not in vcf_columns:
errors.log_and_raise('ERROR: No GT sub-column in VCF.')
if FLAGS.num_records == -1:
variants = reader.iterate()
else:
variants = itertools.islice(reader.iterate(), FLAGS.num_records)
vcf_stats.create_vcf_report(variants,
output_basename=FLAGS.outfile_base,
sample_name=sample_name,
vcf_reader=reader)
if __name__ == '__main__':
flags.mark_flags_as_required(['input_vcf', 'outfile_base'])
tf.compat.v1.app.run()
import datetime
import hashlib
import os
import pprint
import shutil
import sys
from absl import app, flags
from bigtable_input import METADATA, MODEL_PREFIX
from tensorflow.compat.v1 import gfile
from google.cloud import bigtable
sys.path.insert(0, ".")
flags.mark_flags_as_required(["cbt_project", "cbt_instance", "cbt_table"])
FLAGS = flags.FLAGS
PLAYER_FOLDER = "gs://minigo-pub/eval_server/models/"
GTP_BIN = "bazel-bin/cc/gtp"
LIB_TF_FW_SO = "cc/tensorflow/lib/libtensorflow_framework.so"
LIB_TF_CC_SO = "cc/tensorflow/lib/libtensorflow_cc.so"
def verify_params(model_path, model_flags, binary_path):
# Verify model_path = .../<model>.pb
assert model_path.endswith(".pb"), model_path
assert model_flags, "Flags must be non-empty"
import json
import logging
from multiprocessing import Process
from multiprocessing import Queue
import time
from absl import app
from absl import flags
import unistream_profile_driver
flags.DEFINE_list('profile_list', None, 'List of profiles. Each will be run on '
'its own process to simulate '
'concurrency.')
flags.mark_flags_as_required(['profile_list'])
FLAGS = flags.FLAGS
def process_profile(profile, response_q):
"""Method to execute a profile (list of sql scripts) on a cluster.
Args:
profile: The profile to run.
response_q: Communication channel between processes.
"""
profile_output = unistream_profile_driver.execute_profile(profile)
response_q.put([profile, profile_output])
for worker in workers:
worker.join()
else:
logging.info('Computing metric in a single process.')
annotation_pairs = _matched_annotations(gt_json, pred_json)
_compute_metric(metric_aggregator, gt_folder, pred_folder,
annotation_pairs)
is_thing = _is_thing_array(gt_json['categories'], ignored_label)
metric_aggregator.print_detailed_results(is_thing=is_thing,
print_digits=print_digits)
return metric_aggregator.detailed_results(is_thing=is_thing)
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
eval_coco_format(FLAGS.gt_json_file, FLAGS.pred_json_file, FLAGS.gt_folder,
FLAGS.pred_folder, FLAGS.metric, FLAGS.num_categories,
FLAGS.ignored_label, FLAGS.max_instances_per_category,
FLAGS.intersection_offset, FLAGS.normalize_by_image_size,
FLAGS.num_workers, FLAGS.print_digits)
if __name__ == '__main__':
flags.mark_flags_as_required(
['gt_json_file', 'gt_folder', 'pred_json_file', 'pred_folder'])
app.run(main)
os.makedirs(FLAGS.output_dir)
for input_file in sorted(os.listdir(FLAGS.input_dir)):
if not input_file.endswith('.wav'):
continue
wav_filename = input_file
midi_filename = input_file.replace('.wav', '.mid')
logging.info('Aligning %s to %s', midi_filename, wav_filename)
samples = audio_io.load_audio(
os.path.join(FLAGS.input_dir, wav_filename), align_fine_lib.SAMPLE_RATE)
ns = midi_io.midi_file_to_sequence_proto(
os.path.join(FLAGS.input_dir, midi_filename))
aligned_ns, unused_stats = align_fine_lib.align_cpp(
samples,
align_fine_lib.SAMPLE_RATE,
ns,
align_fine_lib.CQT_HOP_LENGTH_FINE,
sf2_path=FLAGS.sf2_path,
penalty_mul=FLAGS.penalty_mul)
midi_io.sequence_proto_to_midi_file(
aligned_ns, os.path.join(FLAGS.output_dir, midi_filename))
logging.info('Done')
if __name__ == '__main__':
flags.mark_flags_as_required(['input_dir', 'output_dir'])
app.run(main)
def main(_):
reference_it = _text_reader(FLAGS.references, multiple=True)
generation_it = _text_reader(FLAGS.generations)
table_it = _table_reader(FLAGS.tables)
if FLAGS.entailment_fn == "cooccurrence":
assert FLAGS.cooccurrence_counts is not None
logging.info("Reading %s...", FLAGS.cooccurrence_counts)
with tf.gfile.Open(FLAGS.cooccurrence_counts) as f:
cooccur_counts = json.load(f)
entail_method = cooccur_probability_fn(cooccur_counts)
else:
entail_method = overlap_probability
precision, recall, f_score, all_f = parent(
generation_it,
reference_it,
table_it,
lambda_weight=FLAGS.lambda_weight,
smoothing=FLAGS.smoothing,
entailment_fn=entail_method)
logging.info("Evaluated %d examples.", len(all_f))
logging.info("Precision = %.4f Recall = %.4f F-score = %.4f", precision,
recall, f_score)
if __name__ == "__main__":
flags.mark_flags_as_required(["references", "generations", "tables"])
app.run(main)
img_prob = cv2.imread(img_path_vis)
for label_img in labels_image:
x_min = int(min(label_img[0]))
x_max = int(max(label_img[0]))
y_min = int(min(label_img[1]))
y_max = int(max(label_img[1]))
cv2.rectangle(img_prob, (x_min, y_min), (x_max, y_max), (0, 0, 255), 2)
cv2.line(img_prob, (int(label_img[0][0]), int(label_img[1][0])),
(int(label_img[0][1]), int(label_img[1][1])), (0, 255, 0), 2)
cv2.line(img_prob, (int(label_img[0][1]), int(label_img[1][1])),
(int(label_img[0][2]), int(label_img[1][2])), (0, 255, 0), 2)
cv2.line(img_prob, (int(label_img[0][2]), int(label_img[1][2])),
(int(label_img[0][3]), int(label_img[1][3])), (0, 255, 0), 2)
cv2.line(img_prob, (int(label_img[0][3]), int(label_img[1][3])),
(int(label_img[0][0]), int(label_img[1][0])), (0, 255, 0), 2)
cv2.imwrite(img_output_vis, img_prob)
def main(_):
vis_results = False
create_tf_record(os.path.join(FLAGS.output, 'training.record'), 'train',
vis_results) # erledigt
# create_tf_record(os.path.join(FLAGS.output, 'validation.record'), 'val', vis_results) # zurzeit nicht vorhanden
create_tf_record_train_as_val(
os.path.join(FLAGS.output, 'validation.record'), 'train', vis_results)
if __name__ == '__main__':
flags.mark_flags_as_required(['data', 'param'])
app.run(main)
kek_uri=FLAGS.kek_uri,
dek_template=aead.aead_key_templates.AES256_GCM)
handle = tink.KeysetHandle.generate_new(template)
env_aead = handle.primitive(aead.Aead)
except tink.TinkError as e:
logging.error('Error creating primitive: %s', e)
return 1
with open(FLAGS.input_path, 'rb') as input_file:
input_data = input_file.read()
if FLAGS.mode == 'decrypt':
output_data = env_aead.decrypt(input_data, associated_data)
elif FLAGS.mode == 'encrypt':
output_data = env_aead.encrypt(input_data, associated_data)
else:
logging.error(
'Error mode not supported. Please choose "encrypt" or "decrypt".'
)
return 1
with open(FLAGS.output_path, 'wb') as output_file:
output_file.write(output_data)
if __name__ == '__main__':
flags.mark_flags_as_required([
'mode', 'kek_uri', 'gcp_credential_path', 'input_path', 'output_path'
])
app.run(main)
# [END envelope-example]
# IMDB LSTM model.
model = tf.keras.Sequential()
model.add(tf.keras.layers.Embedding(20000, 128))
model.add(tf.keras.layers.LSTM(128, dropout=0.2, recurrent_dropout=0.2))
model.add(tf.keras.layers.Dense(1, activation='sigmoid'))
model.compile('sgd', 'mse')
# Warm up the model with one epoch.
model.fit(x_train, y_train, batch_size=512, epochs=3)
def main(_):
# Add the model for memory profile.
models = {
'lstm': _imdb_lstm_model,
}
if FLAGS.model in models:
logging.info('Run memory profile on %s.', FLAGS.model)
run_model = models[FLAGS.model]
run_model()
else:
logging.info('The model does not exist. Please verify the model name.')
if __name__ == '__main__':
flags.mark_flags_as_required(['model'])
if memory_profiler:
app.run(main)
"""Generates the quantized function library contained header file."""
import ast
import re
import string
from typing import Sequence
from absl import app
from absl import flags
_OUTPUT_FILE = flags.DEFINE_string('output_file', None, 'output file location')
_SRC = flags.DEFINE_string('src', None, 'source file location')
_NAMESPACE = flags.DEFINE_string('namespace', 'mlir::quant',
'namespace in the generated file')
flags.mark_flags_as_required(['output_file', 'src'])
def _substitute_function_template(module: str) -> str:
"""Substitutes all the function templates in the given module."""
while True:
pattern = r'^\s*parameters(\[.*?\])\n?(^\s*(?:func\.)+func.*?\{.*?(?:func\.)+return.*?\}\n)'
func_match = re.search(pattern, module, re.MULTILINE | re.DOTALL)
if func_match is None:
break
try:
value_list = ast.literal_eval(func_match.group(1))
func_template = string.Template(func_match.group(2))
except Exception as e: # pylint: disable=broad-except
raise ValueError('The function template is in wrong format') from e
loaner_path=FLAGS.loaner_path,
web_app_dir=FLAGS.web_app_dir,
yaml_files=FLAGS.yaml_files,
version=FLAGS.version)
app_engine_server_config.DeployWebApp()
elif application == 'chrome':
# If application is chrome, begin build and deploy for the chrome app.
chrome_app_config = ChromeAppConfig(
chrome_app_dir=FLAGS.chrome_app_dir,
deployment_type=deployment_type,
loaner_path=FLAGS.loaner_path)
try:
chrome_app_config.DeployChromeApp()
except ManifestError as err:
sys.exit(err.errno)
else:
app.usage(shorthelp=True, exitcode=1)
if __name__ == '__main__':
flags.register_validator('app_servers', _AppServerValidator)
flags.mark_flags_as_required([
'app_servers', 'build_target', 'chrome_app_dir', 'loaner_path',
'web_app_dir', 'yaml_files'
])
app.run(main)
FLAGS.conversion_types)
if not metadata:
raise ValueError(
f'No data found: {FLAGS.base_experiment_dir}, {FLAGS.xids}')
logging.info('%i models in %i xids.', len(metadata), len(FLAGS.xids))
# Check that models don't already exist, and create directories if necessary.
for m in metadata:
utils.sanity_check_output_filename(m.output_filename)
logging.info('Starting to create flume pipeline...')
def _convert_and_write_model(m):
utils.convert_and_write_model(m,
include_frontend=FLAGS.include_frontend,
sanity_check=FLAGS.sanity_check)
return m
# Make and run beam pipeline.
with beam.Pipeline(beam_options) as root:
_ = (root
| 'MakeMetadataCollection' >> beam.Create(metadata)
| 'ConvertAndWriteModelsToDisk' >>
beam.Map(_convert_and_write_model))
if __name__ == '__main__':
tf.compat.v2.enable_v2_behavior()
flags.mark_flags_as_required(['xids', 'base_experiment_dir', 'output_dir'])
app.run(main)
predictions.append(line)
json_examples = []
with open(input_target_path, "r", encoding="utf-8") as input_file:
for line in input_file:
line = six.ensure_text(line, "utf-8")
json_example = json.loads(line)
json_examples.append(json_example)
assert len(predictions) == len(json_examples)
for index, prediction in enumerate(predictions):
json_example = json_examples[index]
if json_example["overlap_subset"]:
overlap_predictions.append(prediction)
else:
nonoverlap_predictions.append(prediction)
print("Writing predictions.")
all_output_path = os.path.join(output_dir, "predictions")
overlap_output_path = os.path.join(output_dir, "overlap_predictions")
nonoverlap_output_path = os.path.join(output_dir, "nonoverlap_predictions")
write_predictions(predictions, all_output_path)
write_predictions(overlap_predictions, overlap_output_path)
write_predictions(nonoverlap_predictions, nonoverlap_output_path)
if __name__ == "__main__":
flags.mark_flags_as_required(
["input_prediction_path", "input_target_path", "output_dir"])
app.run(main)
for line in f:
array.append('-isystem')
array.append(line.strip())
except IOError:
pass
return array
def main(c_flags):
# remove path to current binary
c_flags.pop(0)
logging.debug(FLAGS.sapi_functions)
extract_includes(FLAGS.sapi_isystem, c_flags)
tus = code.Analyzer.process_files(FLAGS.sapi_in, c_flags,
FLAGS.sapi_limit_scan_depth)
generator = code.Generator(tus)
result = generator.generate(FLAGS.sapi_name, FLAGS.sapi_functions,
FLAGS.sapi_ns, FLAGS.sapi_out,
FLAGS.sapi_embed_dir, FLAGS.sapi_embed_name)
if FLAGS.sapi_out:
with open(FLAGS.sapi_out, 'w') as out_file:
out_file.write(result)
else:
sys.stdout.write(result)
if __name__ == '__main__':
flags.mark_flags_as_required(['sapi_name', 'sapi_in'])
app.run(main)
from glob import glob
from absl import app, flags, logging
from tqdm import tqdm
import subprocess
FLAGS = flags.FLAGS
flags.DEFINE_string('dir', default=None, help='Path of dataset to transform')
def main(argv):
file_list = glob(FLAGS.dir + '/*.hwp')
logging.info(f'Transforming {len(file_list)} hwp files.')
for file in tqdm(file_list, desc='transforming'):
with open(file[:-3] + 'txt', 'w') as f:
subprocess.call(['hwp5txt', file], stdout=f)
if __name__ == '__main__':
flags.mark_flags_as_required(['dir'])
app.run(main)
import json
import logging
from multiprocessing import Process
from multiprocessing import Queue
import time
from absl import app
from absl import flags
import unistream_profile_driver
flags.DEFINE_list('profile_list', None, 'List of profiles. Each will be run on '
'its own process to simulate '
'concurrency.')
flags.mark_flags_as_required(['profile_list'])
FLAGS = flags.FLAGS
def process_profile(profile, response_q):
"""Method to execute a profile (list of sql scripts) on a cluster.
Args:
profile: The profile to run.
response_q: Communication channel between processes.
"""
profile_output = unistream_profile_driver.execute_profile(profile)
response_q.put([profile, profile_output])
'Path to directory where pretrained model checkpoints are stored.')
flags.DEFINE_integer('num_gpus',
1,
'Number of gpus to use for training.',
lower_bound=0)
flags.DEFINE_integer('save_checkpoints_step',
None,
'Number of training steps to save checkpoints.',
lower_bound=0)
flags.DEFINE_bool('mixed_precision', False,
'Whether to use mixed precision during training.')
flags.DEFINE_bool('use_tfrecord', False,
'Whether data should be loaded from tfrecord files.')
flags.DEFINE_bool('debug', False, 'Whether to run in debug mode.')
flags.mark_flags_as_required(['config', 'train_file_pattern', 'model_dir'])
FLAGS = flags.FLAGS
def main(_):
assert '.yaml' in FLAGS.config, 'Please provide path to yaml file.'
cfg = get_default_config()
cfg.merge_from_file(FLAGS.config)
cfg.freeze()
model_dir = FLAGS.model_dir
if not tf.io.gfile.exists(model_dir):
tf.io.gfile.makedirs(model_dir)
# init wandb
metric_aggregator.merge(result_queue.get(block=True))
for worker in workers:
worker.join()
else:
logging.info('Computing metric in a single process.')
annotation_pairs = _matched_annotations(gt_json, pred_json)
_compute_metric(metric_aggregator, gt_folder, pred_folder, annotation_pairs)
is_thing = _is_thing_array(gt_json['categories'], ignored_label)
metric_aggregator.print_detailed_results(
is_thing=is_thing, print_digits=print_digits)
return metric_aggregator.detailed_results(is_thing=is_thing)
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
eval_coco_format(FLAGS.gt_json_file, FLAGS.pred_json_file, FLAGS.gt_folder,
FLAGS.pred_folder, FLAGS.metric, FLAGS.num_categories,
FLAGS.ignored_label, FLAGS.max_instances_per_category,
FLAGS.intersection_offset, FLAGS.normalize_by_image_size,
FLAGS.num_workers, FLAGS.print_digits)
if __name__ == '__main__':
flags.mark_flags_as_required(
['gt_json_file', 'gt_folder', 'pred_json_file', 'pred_folder'])
app.run(main)
'bazel-bin/tensorflow/java/ops/src/main/java/org/tensorflow/op')
def main(unused_argv):
merged_source = pathlib.Path(tempfile.mkdtemp())
shutil.copytree(SOURCE_PATH, merged_source / 'java')
if FLAGS.gen_ops:
# `$ yes | configure`
yes = subprocess.Popen(['yes', ''], stdout=subprocess.PIPE)
configure = subprocess.Popen([TENSORFLOW_ROOT / 'configure'],
stdin=yes.stdout,
cwd=TENSORFLOW_ROOT)
configure.communicate()
subprocess.check_call(
['bazel', 'build', '//tensorflow/java:java_op_gen_sources'],
cwd=TENSORFLOW_ROOT)
shutil.copytree(OP_SOURCE_PATH,
merged_source / 'java/org/tensorflow/ops')
gen_java.gen_java_docs(package='org.tensorflow',
source_path=merged_source / 'java',
output_dir=pathlib.Path(FLAGS.output_dir),
site_path=pathlib.Path(FLAGS.site_path))
if __name__ == '__main__':
flags.mark_flags_as_required(['output_dir'])
app.run(main)
from interestingness_xdrl.util.io import create_clear_dir, load_object, get_file_name_without_extension
from interestingness_xdrl.util.logging import change_log_handler
__author__ = 'Pedro Sequeira'
__email__ = '*****@*****.**'
__description__ = 'Saves a dataset on a CSV file containing labels for the different interestingness dimensions ' \
'at each timestep of analysis.'
flags.DEFINE_string('data', None,
'Pickle file containing the interaction data collected using `collect_data_*` scripts.')
flags.DEFINE_string('analysis', None,
'Pickle file containing the full introspection analysis collected using the `analyze` script.')
flags.DEFINE_string('output', 'output/report-highlights', 'Path to the directory in which to save the highlights.')
flags.DEFINE_bool('clear', False, 'Whether to clear output directories before generating results.')
flags.mark_flags_as_required(['data', 'analysis'])
def main(unused_argv):
args = flags.FLAGS
# checks output dir and log file
out_dir = os.path.join(args.output)
create_clear_dir(out_dir, args.clear)
change_log_handler(os.path.join(out_dir, 'dataset.log'), args.verbosity)
logging.info('===================================================================')
# save args
with open(os.path.join(out_dir, 'args.json'), 'w') as fp:
json.dump({k: args[k].value for k in args}, fp, indent=4)
specific arguments (flag values).
"""
__author__ = '*****@*****.**'
from absl import flags
flags.DEFINE_string('bq_project_id', None, 'Project Id which contains the query'
' dataset and table.')
flags.DEFINE_string('bq_dataset_id', None, 'Dataset Id which contains the query'
' table.')
flags.DEFINE_boolean('collect_output', False, 'Flag indicating if query results'
' should be collected.')
flags.DEFINE_string('output_table', 'result_table', 'Table for query output.')
flags.mark_flags_as_required(['bq_project_id', 'bq_dataset_id'])
FLAGS = flags.FLAGS
def generate_provider_specific_cmd_list(script, driver, output, error):
"""Method to compile the BigQuery specific script execution command.
Arguments:
script: SQL script which contains the query.
driver: Driver that contains the BigQuery specific script executor.
output: Output log file.
error: Error log file.
Returns:
Command list to execute the supplied script.
os.make_dirs(probing_dir)
with open(settings_path, 'w') as settings_file:
settings = {
'task_name': task_name,
'trial_id': trial_id,
'train_size': len(train_indexes),
'dev_size': len(dev_indexes),
'test_size': len(test_indexes),
'train_size_unique': len(set(train_indexes)),
'dev_size_unique': len(set(dev_indexes)),
'test_size_unique': len(set(test_indexes)),
}
logging.info('Settings:\n%r', settings)
json.dump(settings, settings_file, indent=2)
with open(data_out_path, 'w') as data_file:
# Don't add quoting to retain the original format unaltered.
df[['set', 'target', 'text']].to_csv(data_file,
sep='\t',
header=False,
index=False,
quoting=csv.QUOTE_NONE,
doublequote=False)
if __name__ == '__main__':
flags.mark_flags_as_required(['senteval_path', 'base_out_dir'])
app.run(main)
from google.cloud import bigtable
from google.cloud.bigtable import row_filters
from tqdm import tqdm
from tensorflow import gfile
import sgf_wrapper
from bigtable_input import METADATA, TABLE_STATE
flags.DEFINE_string(
'sgf_glob', None,
'Glob for SGFs to backfill into eval_games bigtable.')
flags.mark_flags_as_required([
'sgf_glob', 'cbt_project', 'cbt_instance', 'cbt_table'
])
FLAGS = flags.FLAGS
# Constants
EVAL_PREFIX = 'e_{:0>10}'
EVAL_GAME_COUNTER = b'eval_game_counter'
SGF_FILENAME = b'sgf'
#### Common Filters
EVAL_COUNT_FILTER = row_filters.ColumnRangeFilter(
METADATA, EVAL_GAME_COUNTER, EVAL_GAME_COUNTER)
A tuple of the evaluation metrics, and the exported objects from Estimator.
"""
hparams = get_hparams(hparams_set_name)
label_vocab = parse_label_vocab(label_vocab_path)
(estimator, train_spec,
eval_spec) = _make_estimator_and_inputs(hparams=hparams,
label_vocab=label_vocab,
data_base_path=data_base_path,
output_dir=output_dir,
train_fold=train_fold,
eval_fold=eval_fold)
return tf.estimator.train_and_evaluate(estimator=estimator,
train_spec=train_spec,
eval_spec=eval_spec)
def main(_):
train(data_base_path=FLAGS.data_base_path,
output_dir=FLAGS.output_dir,
label_vocab_path=FLAGS.label_vocab_path,
hparams_set_name=FLAGS.hparams_set,
train_fold=FLAGS.train_fold,
eval_fold=FLAGS.eval_fold)
if __name__ == '__main__':
FLAGS.alsologtostderr = True # Shows training output.
flags.mark_flags_as_required(['data_base_path', 'label_vocab_path'])
app.run(main)
if not FLAGS.signer_key:
raise app.UsageError("ERROR: --signer_key flag not set.")
if not is_signature_valid(json_data, FLAGS.signature, FLAGS.signer_key):
raise app.UsageError(
"ERROR: Signature over list of logs is not valid.")
parsed_json = json.loads(json_data)
if not FLAGS.log_list_schema:
raise app.UsageError("ERROR: --log_list_schema flag not set.")
if not is_log_list_valid(parsed_json, FLAGS.log_list_schema):
raise app.UsageError(
"ERROR: Log list is signed but does not conform to the schema.", 2)
if FLAGS.header_output:
generate_cpp_header(parsed_json, FLAGS.header_output)
if FLAGS.java_output:
generate_java_source(parsed_json, FLAGS.java_output, FLAGS.java_class)
if FLAGS.openssl_output:
generate_openssl_conf(parsed_json, FLAGS.openssl_output)
if not FLAGS.header_output and \
not FLAGS.java_output and \
not FLAGS.openssl_output:
print_formatted_log_list(parsed_json)
if __name__ == "__main__":
gflags.mark_flags_as_required(["log_list", "log_list_schema"])
gflags.mark_flags_as_mutual_exclusive(["signature", "skip_signature_check"],
required=True)
app.run(main)
Contains the method to compile the Azure SQL data warehouse specific script
execution command based on generic arguments (sql script, output destination)
and Azure SQL specific arguments (flag values).
"""
__author__ = '*****@*****.**'
from absl import flags
flags.DEFINE_string('server', None, 'SQL server.')
flags.DEFINE_string('database', None, 'SQL Database.')
flags.DEFINE_string('user', None, 'SQL User.')
flags.DEFINE_string('password', None, 'SQL Password.')
flags.mark_flags_as_required(['server', 'database', 'user', 'password'])
FLAGS = flags.FLAGS
def generate_provider_specific_cmd_list(script, driver, output, error):
"""Method to compile the Azure SQL specific script execution command.
Arguments:
script: SQL script which contains the query.
driver: Driver that contains the Azure SQL data warehouse specific script
executor.
output: Output log file.
error: Error log file.
Returns: