def define_flags(): """Construct flags for the server.""" flags.DEFINE_integer(name="num_workers", default=multiprocessing.cpu_count(), help="Size of the negative generation worker pool.") flags.DEFINE_string(name="data_dir", default=None, help="The data root. (used to construct cache paths.)") flags.DEFINE_string(name="cache_id", default=None, help="The cache_id generated in the main process.") flags.DEFINE_integer(name="num_readers", default=4, help="Number of reader datasets in training. This sets" "how the epoch files are sharded.") flags.DEFINE_integer(name="num_neg", default=None, help="The Number of negative instances to pair with a " "positive instance.") flags.DEFINE_integer(name="num_train_positives", default=None, help="The number of positive training examples.") flags.DEFINE_integer(name="num_items", default=None, help="Number of items from which to select negatives.") flags.DEFINE_integer(name="num_users", default=None, help="The number of unique users. Used for evaluation.") flags.DEFINE_integer(name="epochs_per_cycle", default=1, help="The number of epochs of training data to produce" "at a time.") flags.DEFINE_integer(name="num_cycles", default=None, help="The number of cycles to produce training data " "for.") flags.DEFINE_integer(name="train_batch_size", default=None, help="The batch size with which training TFRecords will " "be chunked.") flags.DEFINE_integer(name="eval_batch_size", default=None, help="The batch size with which evaluation TFRecords " "will be chunked.") flags.DEFINE_boolean(name="redirect_logs", default=False, help="Catch logs and write them to a file. " "(Useful if this is run as a subprocess)") flags.DEFINE_boolean(name="use_tf_logging", default=False, help="Use tf.logging instead of log file.") flags.DEFINE_integer(name="seed", default=None, help="NumPy random seed to set at startup. If not " "specified, a seed will not be set.") flags.DEFINE_boolean(name="ml_perf", default=None, help="Match MLPerf. See ncf_main.py for details.") flags.DEFINE_bool(name="output_ml_perf_compliance_logging", default=None, help="Output the MLPerf compliance logging. See " "ncf_main.py for details.") flags.mark_flags_as_required(["data_dir", "cache_id"])
def define_flags(): """Construct flags for the server. This function does not use offical.utils.flags, as these flags are not meant to be used by humans. Rather, they should be passed as part of a subprocess call. """ flags.DEFINE_integer(name="num_workers", default=multiprocessing.cpu_count(), help="Size of the negative generation worker pool.") flags.DEFINE_string(name="data_dir", default=None, help="The data root. (used to construct cache paths.)") flags.DEFINE_string(name="cache_id", default=None, help="The cache_id generated in the main process.") flags.DEFINE_integer(name="num_readers", default=4, help="Number of reader datasets in training. This sets" "how the epoch files are sharded.") flags.DEFINE_integer(name="num_neg", default=None, help="The Number of negative instances to pair with a " "positive instance.") flags.DEFINE_integer(name="num_train_positives", default=None, help="The number of positive training examples.") flags.DEFINE_integer(name="num_items", default=None, help="Number of items from which to select negatives.") flags.DEFINE_integer(name="epochs_per_cycle", default=1, help="The number of epochs of training data to produce" "at a time.") flags.DEFINE_integer(name="train_batch_size", default=None, help="The batch size with which training TFRecords will " "be chunked.") flags.DEFINE_integer(name="eval_batch_size", default=None, help="The batch size with which evaluation TFRecords " "will be chunked.") flags.DEFINE_boolean( name="spillover", default=True, help="If a complete batch cannot be provided, return an empty batch and " "start the next epoch from a non-empty buffer. This guarantees " "fixed batch sizes.") flags.DEFINE_boolean(name="redirect_logs", default=False, help="Catch logs and write them to a file. " "(Useful if this is run as a subprocess)") flags.DEFINE_integer(name="seed", default=None, help="NumPy random seed to set at startup. If not " "specified, a seed will not be set.") flags.mark_flags_as_required( ["data_dir", "cache_id", "num_neg", "num_train_positives", "num_items", "train_batch_size", "eval_batch_size"])
from clu import platform import train import jax from ml_collections import config_flags import tensorflow as tf FLAGS = flags.FLAGS flags.DEFINE_string('workdir', None, 'Directory to store model data.') config_flags.DEFINE_config_file( 'config', 'configs/default.py', 'File path to the training hyperparameter configuration.', lock_config=True) flags.mark_flags_as_required(['config', 'workdir']) def main(argv): if len(argv) > 1: raise app.UsageError('Too many command-line arguments.') # Hide any GPUs form TensorFlow. Otherwise TF might reserve memory and make # it unavailable to JAX. tf.config.experimental.set_visible_devices([], 'GPU') logging.info('JAX process: %d / %d', jax.process_index(), jax.process_count()) logging.info('JAX local devices: %r', jax.local_devices()) # Add a note so that we can tell which task is which JAX host.
else: assert FLAGS.schedule == "continuous_eval" eval_args = _create_eval_args(config_overrides.get("dataset")) for _ in estimator_runner.continuous_eval( estimator=estimator, eval_args=eval_args, train_steps=FLAGS.train_steps): # continuous_train_and_eval() yields evaluation metrics after each # checkpoint. It also saves and logs them, so we don't do anything here. pass if __name__ == "__main__": tf.logging.set_verbosity(tf.logging.INFO) flags.mark_flags_as_required(["dataset", "model_dir", "schedule"]) def _validate_schedule(flag_values): """Validates the --schedule flag and the flags it interacts with.""" schedule = flag_values["schedule"] save_checkpoints_steps = flag_values["save_checkpoints_steps"] save_checkpoints_secs = flag_values["save_checkpoints_secs"] if schedule in ["train", "train_and_eval"]: if not (save_checkpoints_steps or save_checkpoints_secs): raise flags.ValidationError( "--schedule='%s' requires --save_checkpoints_steps or " "--save_checkpoints_secs." % schedule) return True
keyset_handle = cleartext_keyset_handle.read(tink.JsonKeysetReader(text)) except tink.TinkError as e: logging.exception('Error reading keyset: %s', e) return 1 now = datetime.datetime.now(tz=datetime.timezone.utc) # Get the JwtPublicKeySign primitive try: jwt_sign = keyset_handle.primitive(jwt.JwtPublicKeySign) except tink.TinkError as e: logging.exception('Error creating JwtPublicKeySign: %s', e) return 1 # Create token raw_jwt = jwt.new_raw_jwt( audiences=[_AUDIENCE.value], expiration=now + datetime.timedelta(seconds=100)) token = jwt_sign.sign_and_encode(raw_jwt) with open(_TOKEN_PATH.value, 'wt') as token_file: token_file.write(token) logging.info('Token has been written to %s', _TOKEN_PATH.value) if __name__ == '__main__': flags.mark_flags_as_required( ['private_keyset_path', 'audience', 'token_path']) app.run(main) # [END python-jwt-sign-example]
maxlen=FLAGS.maxlen) basemodel = BaseModel(input_dim=FLAGS.max_features, maxlen=FLAGS.maxlen) model = multi_gpu_model(basemodel.build(name=FLAGS.model_name), gpus=len(get_available_gpus())) model.compile(loss='binary_crossentropy', optimizer=FLAGS.optimizer, metrics=['accuracy']) model.fit( X_train.todense(), y_train, validation_data=(X_val.todense(), y_val), batch_size=FLAGS.batch_size, epochs=FLAGS.epochs, callbacks=[ ModelCheckpointToGcs( remotekey=FLAGS.gcs_path, filename= 'model_checkpoint.{epoch:02d}_{loss:.6f}_{val_loss:.6f}.h5', save_best_only=True), CSVLoggerToGcs(remotekey=FLAGS.gcs_path, filename=path.join('history_{}_{}.tsv').format( FLAGS.model_name, FLAGS.optimizer), separator='\t'), ]) if __name__ == '__main__': flags.mark_flags_as_required(['model_name', 'gcs_path']) app.run(main)
logging.info('mode %r num_frames %r feature shape %r', FLAGS.mode, params.task.model.backbone.assemblenet.num_frames, shape) # Sets mixed_precision policy. Using 'mixed_float16' or 'mixed_bfloat16' # can have significant impact on model speeds by utilizing float16 in case of # GPUs, and bfloat16 in the case of TPUs. loss_scale takes effect only when # dtype is float16 if params.runtime.mixed_precision_dtype: performance.set_mixed_precision_policy(params.runtime.mixed_precision_dtype) distribution_strategy = distribute_utils.get_distribution_strategy( distribution_strategy=params.runtime.distribution_strategy, all_reduce_alg=params.runtime.all_reduce_alg, num_gpus=params.runtime.num_gpus, tpu_address=params.runtime.tpu) with distribution_strategy.scope(): task = task_factory.get_task(params.task, logging_dir=model_dir) train_lib.run_experiment( distribution_strategy=distribution_strategy, task=task, mode=FLAGS.mode, params=params, model_dir=model_dir) train_utils.save_gin_config(FLAGS.mode, model_dir) if __name__ == '__main__': tfm_flags.define_flags() flags.mark_flags_as_required(['experiment', 'mode', 'model_dir']) app.run(main)
from absl import flags import jax from ml_collections import config_flags import tensorflow as tf from vdvae_flax import experiment FLAGS = flags.FLAGS config_flags.DEFINE_config_file("config", None, "Training configuration.", lock_config=True) flags.DEFINE_string("workdir", None, "Work unit directory.") flags.mark_flags_as_required(["config", "workdir"]) def main(argv): del argv # Hide any GPUs form TensorFlow. Otherwise TF might reserve memory and make # it unavailable to JAX. tf.config.experimental.set_visible_devices([], "GPU") exp = experiment.Experiment("train", FLAGS.config) exp.train_and_evaluate(FLAGS.workdir) if __name__ == "__main__": jax.config.config_with_absl()
logging.exception('Error reading key: %s', e) return 1 # Get the primitive try: cipher = keyset_handle.primitive(aead.Aead) except tink.TinkError as e: logging.error('Error creating primitive: %s', e) return 1 with open(FLAGS.input_path, 'rb') as input_file: input_data = input_file.read() if FLAGS.mode == 'decrypt': output_data = cipher.decrypt(input_data, b'envelope_example') elif FLAGS.mode == 'encrypt': output_data = cipher.encrypt(input_data, b'envelope_example') else: logging.error( 'Error mode not supported. Please choose "encrypt" or "decrypt".' ) return 1 with open(FLAGS.output_path, 'wb') as output_file: output_file.write(output_data) if __name__ == '__main__': flags.mark_flags_as_required(['mode', 'keyset_path']) app.run(main) # [END cleartext-keyset-example]
model = tf.keras.models.load_model(FLAGS.saved_model_dir, compile=False) logging.info('Model initialized. Beginning prediction loop.') logits_fn = tf.function( functools.partial(prediction.distributed_logits_fn, model)) decode_logits_fn = tf.function( functools.partial(prediction.decode_logits, FLAGS.decode_top_k, FLAGS.decode_max_size)) split_and_pad_fn = tf.function( functools.partial(prediction.split_and_pad, strategy, FLAGS.batch_size)) # Prediction strategy. predict_fn = functools.partial(predict, sp_processor=sp_processor, features_map_fn=features_map_fn, logits_fn=logits_fn, decode_logits_fn=decode_logits_fn, split_and_pad_fn=split_and_pad_fn, distribute_strategy=strategy, dataset=dataset) with worker_context(): predictions = predict_fn() with tf.io.gfile.GFile(FLAGS.predictions_path, 'w') as f: json.dump(predictions, f) if __name__ == '__main__': flags.mark_flags_as_required( ['split', 'predictions_path', 'saved_model_dir']) app.run(main)
with tf.compat.v1.Session() as sess: sess.run(tf.compat.v1.global_variables_initializer()) load_ema(sess) graph_def = sess.graph.as_graph_def() graph_def = tf.compat.v1.graph_util.convert_variables_to_constants( sess, graph_def, [out_node]) graph_def = optimize_for_inference_lib.optimize_for_inference( graph_def, [in_node], [out_node], tf.float32.as_datatype_enum) with tf.io.gfile.GFile(openvino_model_pb, 'wb') as f: f.write(graph_def.SerializeToString()) def main(_): tensor_shape = [FLAGS.height, FLAGS.width, FLAGS.channels] logging.info('Processing ckpt=%s, tensor_shape=%s.', FLAGS.checkpoint, tensor_shape) freeze_graph(modeling.get_model('inception_v3'), FLAGS.checkpoint, tensor_shape, FLAGS.output) logging.info('Output written to %s.', FLAGS.output) if __name__ == '__main__': flags.mark_flags_as_required([ 'checkpoint', 'output', ]) tf.compat.v1.app.run()
ds = json.load(open(filename)) except: print('Cannot read {filename}'.format(filename=filename)) quit() epoch = max(map(int, ds['epoch'].keys())) epoch = min(FLAGS.epoch or epoch, epoch) df = {} for metric in FLAGS.metrics: data = ds['epoch'][str( epoch)][0]['test'][metric]['classification_report'] df[metric] = pd.read_table(io.StringIO(data), delimiter=r'\s{2,}', engine='python') return types.SimpleNamespace(df=df, epoch=epoch) def main(unused_argv): for dataset in FLAGS.datasets: res = df_for_dataset(dataset) print('dataset={dataset} epoch={epoch}'.format(dataset=dataset, epoch=res.epoch)) for d in res.df.values(): print(d.to_csv() if FLAGS.csv else d) if FLAGS.shell: IPython.start_ipython(argv=[], user_ns=dict(globals(), **locals())) if __name__ == '__main__': flags.mark_flags_as_required(['datasets']) app.run(main)
try: start_timestamp = time.time( ) # This time will include compilation time eval_results = estimator.evaluate(input_fn=eval_input_fn, steps=max_eval_steps, checkpoint_path=ckpt) elapsed_time = int(time.time() - start_timestamp) logging.info("Eval results: %s. Elapsed seconds: %d", eval_results, elapsed_time) # Terminate eval job when final checkpoint is reached. current_step = int(os.path.basename(ckpt).split("-")[1]) if current_step >= FLAGS.num_train_steps: logging.info("Evaluation finished after training step %d", current_step) break except tf.errors.NotFoundError: # Since the coordinator is on a different job than the TPU worker, # sometimes the TPU worker does not finish initializing until long after # the CPU job tells it to start evaluating. In this case, the checkpoint # file could have been deleted already. logging.info( "Checkpoint %s no longer exists, skipping checkpoint", ckpt) if __name__ == "__main__": flags.mark_flags_as_required(["input_file", "output_dir"]) app.run(main)
'positional arguments but some are present on the command line: ' '"{}".'.format(str(argv[1:])), errors.CommandLineError) with vcf.VcfReader(FLAGS.input_vcf) as reader: sample_names = reader.header.sample_names if len(sample_names) != 1: raise ValueError( 'There must be exactly one sample in VCF: {}'.format( FLAGS.input_vcf)) sample_name = sample_names[0] # Missing GT causes error later while reading, so throw a clearer error here vcf_columns = [col.id for col in reader.header.formats] if 'GT' not in vcf_columns: errors.log_and_raise('ERROR: No GT sub-column in VCF.') if FLAGS.num_records == -1: variants = reader.iterate() else: variants = itertools.islice(reader.iterate(), FLAGS.num_records) vcf_stats.create_vcf_report(variants, output_basename=FLAGS.outfile_base, sample_name=sample_name, vcf_reader=reader) if __name__ == '__main__': flags.mark_flags_as_required(['input_vcf', 'outfile_base']) tf.compat.v1.app.run()
import datetime import hashlib import os import pprint import shutil import sys from absl import app, flags from bigtable_input import METADATA, MODEL_PREFIX from tensorflow.compat.v1 import gfile from google.cloud import bigtable sys.path.insert(0, ".") flags.mark_flags_as_required(["cbt_project", "cbt_instance", "cbt_table"]) FLAGS = flags.FLAGS PLAYER_FOLDER = "gs://minigo-pub/eval_server/models/" GTP_BIN = "bazel-bin/cc/gtp" LIB_TF_FW_SO = "cc/tensorflow/lib/libtensorflow_framework.so" LIB_TF_CC_SO = "cc/tensorflow/lib/libtensorflow_cc.so" def verify_params(model_path, model_flags, binary_path): # Verify model_path = .../<model>.pb assert model_path.endswith(".pb"), model_path assert model_flags, "Flags must be non-empty"
import json import logging from multiprocessing import Process from multiprocessing import Queue import time from absl import app from absl import flags import unistream_profile_driver flags.DEFINE_list('profile_list', None, 'List of profiles. Each will be run on ' 'its own process to simulate ' 'concurrency.') flags.mark_flags_as_required(['profile_list']) FLAGS = flags.FLAGS def process_profile(profile, response_q): """Method to execute a profile (list of sql scripts) on a cluster. Args: profile: The profile to run. response_q: Communication channel between processes. """ profile_output = unistream_profile_driver.execute_profile(profile) response_q.put([profile, profile_output])
for worker in workers: worker.join() else: logging.info('Computing metric in a single process.') annotation_pairs = _matched_annotations(gt_json, pred_json) _compute_metric(metric_aggregator, gt_folder, pred_folder, annotation_pairs) is_thing = _is_thing_array(gt_json['categories'], ignored_label) metric_aggregator.print_detailed_results(is_thing=is_thing, print_digits=print_digits) return metric_aggregator.detailed_results(is_thing=is_thing) def main(argv): if len(argv) > 1: raise app.UsageError('Too many command-line arguments.') eval_coco_format(FLAGS.gt_json_file, FLAGS.pred_json_file, FLAGS.gt_folder, FLAGS.pred_folder, FLAGS.metric, FLAGS.num_categories, FLAGS.ignored_label, FLAGS.max_instances_per_category, FLAGS.intersection_offset, FLAGS.normalize_by_image_size, FLAGS.num_workers, FLAGS.print_digits) if __name__ == '__main__': flags.mark_flags_as_required( ['gt_json_file', 'gt_folder', 'pred_json_file', 'pred_folder']) app.run(main)
os.makedirs(FLAGS.output_dir) for input_file in sorted(os.listdir(FLAGS.input_dir)): if not input_file.endswith('.wav'): continue wav_filename = input_file midi_filename = input_file.replace('.wav', '.mid') logging.info('Aligning %s to %s', midi_filename, wav_filename) samples = audio_io.load_audio( os.path.join(FLAGS.input_dir, wav_filename), align_fine_lib.SAMPLE_RATE) ns = midi_io.midi_file_to_sequence_proto( os.path.join(FLAGS.input_dir, midi_filename)) aligned_ns, unused_stats = align_fine_lib.align_cpp( samples, align_fine_lib.SAMPLE_RATE, ns, align_fine_lib.CQT_HOP_LENGTH_FINE, sf2_path=FLAGS.sf2_path, penalty_mul=FLAGS.penalty_mul) midi_io.sequence_proto_to_midi_file( aligned_ns, os.path.join(FLAGS.output_dir, midi_filename)) logging.info('Done') if __name__ == '__main__': flags.mark_flags_as_required(['input_dir', 'output_dir']) app.run(main)
def main(_): reference_it = _text_reader(FLAGS.references, multiple=True) generation_it = _text_reader(FLAGS.generations) table_it = _table_reader(FLAGS.tables) if FLAGS.entailment_fn == "cooccurrence": assert FLAGS.cooccurrence_counts is not None logging.info("Reading %s...", FLAGS.cooccurrence_counts) with tf.gfile.Open(FLAGS.cooccurrence_counts) as f: cooccur_counts = json.load(f) entail_method = cooccur_probability_fn(cooccur_counts) else: entail_method = overlap_probability precision, recall, f_score, all_f = parent( generation_it, reference_it, table_it, lambda_weight=FLAGS.lambda_weight, smoothing=FLAGS.smoothing, entailment_fn=entail_method) logging.info("Evaluated %d examples.", len(all_f)) logging.info("Precision = %.4f Recall = %.4f F-score = %.4f", precision, recall, f_score) if __name__ == "__main__": flags.mark_flags_as_required(["references", "generations", "tables"]) app.run(main)
img_prob = cv2.imread(img_path_vis) for label_img in labels_image: x_min = int(min(label_img[0])) x_max = int(max(label_img[0])) y_min = int(min(label_img[1])) y_max = int(max(label_img[1])) cv2.rectangle(img_prob, (x_min, y_min), (x_max, y_max), (0, 0, 255), 2) cv2.line(img_prob, (int(label_img[0][0]), int(label_img[1][0])), (int(label_img[0][1]), int(label_img[1][1])), (0, 255, 0), 2) cv2.line(img_prob, (int(label_img[0][1]), int(label_img[1][1])), (int(label_img[0][2]), int(label_img[1][2])), (0, 255, 0), 2) cv2.line(img_prob, (int(label_img[0][2]), int(label_img[1][2])), (int(label_img[0][3]), int(label_img[1][3])), (0, 255, 0), 2) cv2.line(img_prob, (int(label_img[0][3]), int(label_img[1][3])), (int(label_img[0][0]), int(label_img[1][0])), (0, 255, 0), 2) cv2.imwrite(img_output_vis, img_prob) def main(_): vis_results = False create_tf_record(os.path.join(FLAGS.output, 'training.record'), 'train', vis_results) # erledigt # create_tf_record(os.path.join(FLAGS.output, 'validation.record'), 'val', vis_results) # zurzeit nicht vorhanden create_tf_record_train_as_val( os.path.join(FLAGS.output, 'validation.record'), 'train', vis_results) if __name__ == '__main__': flags.mark_flags_as_required(['data', 'param']) app.run(main)
kek_uri=FLAGS.kek_uri, dek_template=aead.aead_key_templates.AES256_GCM) handle = tink.KeysetHandle.generate_new(template) env_aead = handle.primitive(aead.Aead) except tink.TinkError as e: logging.error('Error creating primitive: %s', e) return 1 with open(FLAGS.input_path, 'rb') as input_file: input_data = input_file.read() if FLAGS.mode == 'decrypt': output_data = env_aead.decrypt(input_data, associated_data) elif FLAGS.mode == 'encrypt': output_data = env_aead.encrypt(input_data, associated_data) else: logging.error( 'Error mode not supported. Please choose "encrypt" or "decrypt".' ) return 1 with open(FLAGS.output_path, 'wb') as output_file: output_file.write(output_data) if __name__ == '__main__': flags.mark_flags_as_required([ 'mode', 'kek_uri', 'gcp_credential_path', 'input_path', 'output_path' ]) app.run(main) # [END envelope-example]
# IMDB LSTM model. model = tf.keras.Sequential() model.add(tf.keras.layers.Embedding(20000, 128)) model.add(tf.keras.layers.LSTM(128, dropout=0.2, recurrent_dropout=0.2)) model.add(tf.keras.layers.Dense(1, activation='sigmoid')) model.compile('sgd', 'mse') # Warm up the model with one epoch. model.fit(x_train, y_train, batch_size=512, epochs=3) def main(_): # Add the model for memory profile. models = { 'lstm': _imdb_lstm_model, } if FLAGS.model in models: logging.info('Run memory profile on %s.', FLAGS.model) run_model = models[FLAGS.model] run_model() else: logging.info('The model does not exist. Please verify the model name.') if __name__ == '__main__': flags.mark_flags_as_required(['model']) if memory_profiler: app.run(main)
"""Generates the quantized function library contained header file.""" import ast import re import string from typing import Sequence from absl import app from absl import flags _OUTPUT_FILE = flags.DEFINE_string('output_file', None, 'output file location') _SRC = flags.DEFINE_string('src', None, 'source file location') _NAMESPACE = flags.DEFINE_string('namespace', 'mlir::quant', 'namespace in the generated file') flags.mark_flags_as_required(['output_file', 'src']) def _substitute_function_template(module: str) -> str: """Substitutes all the function templates in the given module.""" while True: pattern = r'^\s*parameters(\[.*?\])\n?(^\s*(?:func\.)+func.*?\{.*?(?:func\.)+return.*?\}\n)' func_match = re.search(pattern, module, re.MULTILINE | re.DOTALL) if func_match is None: break try: value_list = ast.literal_eval(func_match.group(1)) func_template = string.Template(func_match.group(2)) except Exception as e: # pylint: disable=broad-except raise ValueError('The function template is in wrong format') from e
loaner_path=FLAGS.loaner_path, web_app_dir=FLAGS.web_app_dir, yaml_files=FLAGS.yaml_files, version=FLAGS.version) app_engine_server_config.DeployWebApp() elif application == 'chrome': # If application is chrome, begin build and deploy for the chrome app. chrome_app_config = ChromeAppConfig( chrome_app_dir=FLAGS.chrome_app_dir, deployment_type=deployment_type, loaner_path=FLAGS.loaner_path) try: chrome_app_config.DeployChromeApp() except ManifestError as err: sys.exit(err.errno) else: app.usage(shorthelp=True, exitcode=1) if __name__ == '__main__': flags.register_validator('app_servers', _AppServerValidator) flags.mark_flags_as_required([ 'app_servers', 'build_target', 'chrome_app_dir', 'loaner_path', 'web_app_dir', 'yaml_files' ]) app.run(main)
FLAGS.conversion_types) if not metadata: raise ValueError( f'No data found: {FLAGS.base_experiment_dir}, {FLAGS.xids}') logging.info('%i models in %i xids.', len(metadata), len(FLAGS.xids)) # Check that models don't already exist, and create directories if necessary. for m in metadata: utils.sanity_check_output_filename(m.output_filename) logging.info('Starting to create flume pipeline...') def _convert_and_write_model(m): utils.convert_and_write_model(m, include_frontend=FLAGS.include_frontend, sanity_check=FLAGS.sanity_check) return m # Make and run beam pipeline. with beam.Pipeline(beam_options) as root: _ = (root | 'MakeMetadataCollection' >> beam.Create(metadata) | 'ConvertAndWriteModelsToDisk' >> beam.Map(_convert_and_write_model)) if __name__ == '__main__': tf.compat.v2.enable_v2_behavior() flags.mark_flags_as_required(['xids', 'base_experiment_dir', 'output_dir']) app.run(main)
predictions.append(line) json_examples = [] with open(input_target_path, "r", encoding="utf-8") as input_file: for line in input_file: line = six.ensure_text(line, "utf-8") json_example = json.loads(line) json_examples.append(json_example) assert len(predictions) == len(json_examples) for index, prediction in enumerate(predictions): json_example = json_examples[index] if json_example["overlap_subset"]: overlap_predictions.append(prediction) else: nonoverlap_predictions.append(prediction) print("Writing predictions.") all_output_path = os.path.join(output_dir, "predictions") overlap_output_path = os.path.join(output_dir, "overlap_predictions") nonoverlap_output_path = os.path.join(output_dir, "nonoverlap_predictions") write_predictions(predictions, all_output_path) write_predictions(overlap_predictions, overlap_output_path) write_predictions(nonoverlap_predictions, nonoverlap_output_path) if __name__ == "__main__": flags.mark_flags_as_required( ["input_prediction_path", "input_target_path", "output_dir"]) app.run(main)
for line in f: array.append('-isystem') array.append(line.strip()) except IOError: pass return array def main(c_flags): # remove path to current binary c_flags.pop(0) logging.debug(FLAGS.sapi_functions) extract_includes(FLAGS.sapi_isystem, c_flags) tus = code.Analyzer.process_files(FLAGS.sapi_in, c_flags, FLAGS.sapi_limit_scan_depth) generator = code.Generator(tus) result = generator.generate(FLAGS.sapi_name, FLAGS.sapi_functions, FLAGS.sapi_ns, FLAGS.sapi_out, FLAGS.sapi_embed_dir, FLAGS.sapi_embed_name) if FLAGS.sapi_out: with open(FLAGS.sapi_out, 'w') as out_file: out_file.write(result) else: sys.stdout.write(result) if __name__ == '__main__': flags.mark_flags_as_required(['sapi_name', 'sapi_in']) app.run(main)
from glob import glob from absl import app, flags, logging from tqdm import tqdm import subprocess FLAGS = flags.FLAGS flags.DEFINE_string('dir', default=None, help='Path of dataset to transform') def main(argv): file_list = glob(FLAGS.dir + '/*.hwp') logging.info(f'Transforming {len(file_list)} hwp files.') for file in tqdm(file_list, desc='transforming'): with open(file[:-3] + 'txt', 'w') as f: subprocess.call(['hwp5txt', file], stdout=f) if __name__ == '__main__': flags.mark_flags_as_required(['dir']) app.run(main)
import json import logging from multiprocessing import Process from multiprocessing import Queue import time from absl import app from absl import flags import unistream_profile_driver flags.DEFINE_list('profile_list', None, 'List of profiles. Each will be run on ' 'its own process to simulate ' 'concurrency.') flags.mark_flags_as_required(['profile_list']) FLAGS = flags.FLAGS def process_profile(profile, response_q): """Method to execute a profile (list of sql scripts) on a cluster. Args: profile: The profile to run. response_q: Communication channel between processes. """ profile_output = unistream_profile_driver.execute_profile(profile) response_q.put([profile, profile_output])
'Path to directory where pretrained model checkpoints are stored.') flags.DEFINE_integer('num_gpus', 1, 'Number of gpus to use for training.', lower_bound=0) flags.DEFINE_integer('save_checkpoints_step', None, 'Number of training steps to save checkpoints.', lower_bound=0) flags.DEFINE_bool('mixed_precision', False, 'Whether to use mixed precision during training.') flags.DEFINE_bool('use_tfrecord', False, 'Whether data should be loaded from tfrecord files.') flags.DEFINE_bool('debug', False, 'Whether to run in debug mode.') flags.mark_flags_as_required(['config', 'train_file_pattern', 'model_dir']) FLAGS = flags.FLAGS def main(_): assert '.yaml' in FLAGS.config, 'Please provide path to yaml file.' cfg = get_default_config() cfg.merge_from_file(FLAGS.config) cfg.freeze() model_dir = FLAGS.model_dir if not tf.io.gfile.exists(model_dir): tf.io.gfile.makedirs(model_dir) # init wandb
metric_aggregator.merge(result_queue.get(block=True)) for worker in workers: worker.join() else: logging.info('Computing metric in a single process.') annotation_pairs = _matched_annotations(gt_json, pred_json) _compute_metric(metric_aggregator, gt_folder, pred_folder, annotation_pairs) is_thing = _is_thing_array(gt_json['categories'], ignored_label) metric_aggregator.print_detailed_results( is_thing=is_thing, print_digits=print_digits) return metric_aggregator.detailed_results(is_thing=is_thing) def main(argv): if len(argv) > 1: raise app.UsageError('Too many command-line arguments.') eval_coco_format(FLAGS.gt_json_file, FLAGS.pred_json_file, FLAGS.gt_folder, FLAGS.pred_folder, FLAGS.metric, FLAGS.num_categories, FLAGS.ignored_label, FLAGS.max_instances_per_category, FLAGS.intersection_offset, FLAGS.normalize_by_image_size, FLAGS.num_workers, FLAGS.print_digits) if __name__ == '__main__': flags.mark_flags_as_required( ['gt_json_file', 'gt_folder', 'pred_json_file', 'pred_folder']) app.run(main)
'bazel-bin/tensorflow/java/ops/src/main/java/org/tensorflow/op') def main(unused_argv): merged_source = pathlib.Path(tempfile.mkdtemp()) shutil.copytree(SOURCE_PATH, merged_source / 'java') if FLAGS.gen_ops: # `$ yes | configure` yes = subprocess.Popen(['yes', ''], stdout=subprocess.PIPE) configure = subprocess.Popen([TENSORFLOW_ROOT / 'configure'], stdin=yes.stdout, cwd=TENSORFLOW_ROOT) configure.communicate() subprocess.check_call( ['bazel', 'build', '//tensorflow/java:java_op_gen_sources'], cwd=TENSORFLOW_ROOT) shutil.copytree(OP_SOURCE_PATH, merged_source / 'java/org/tensorflow/ops') gen_java.gen_java_docs(package='org.tensorflow', source_path=merged_source / 'java', output_dir=pathlib.Path(FLAGS.output_dir), site_path=pathlib.Path(FLAGS.site_path)) if __name__ == '__main__': flags.mark_flags_as_required(['output_dir']) app.run(main)
from interestingness_xdrl.util.io import create_clear_dir, load_object, get_file_name_without_extension from interestingness_xdrl.util.logging import change_log_handler __author__ = 'Pedro Sequeira' __email__ = '*****@*****.**' __description__ = 'Saves a dataset on a CSV file containing labels for the different interestingness dimensions ' \ 'at each timestep of analysis.' flags.DEFINE_string('data', None, 'Pickle file containing the interaction data collected using `collect_data_*` scripts.') flags.DEFINE_string('analysis', None, 'Pickle file containing the full introspection analysis collected using the `analyze` script.') flags.DEFINE_string('output', 'output/report-highlights', 'Path to the directory in which to save the highlights.') flags.DEFINE_bool('clear', False, 'Whether to clear output directories before generating results.') flags.mark_flags_as_required(['data', 'analysis']) def main(unused_argv): args = flags.FLAGS # checks output dir and log file out_dir = os.path.join(args.output) create_clear_dir(out_dir, args.clear) change_log_handler(os.path.join(out_dir, 'dataset.log'), args.verbosity) logging.info('===================================================================') # save args with open(os.path.join(out_dir, 'args.json'), 'w') as fp: json.dump({k: args[k].value for k in args}, fp, indent=4)
specific arguments (flag values). """ __author__ = '*****@*****.**' from absl import flags flags.DEFINE_string('bq_project_id', None, 'Project Id which contains the query' ' dataset and table.') flags.DEFINE_string('bq_dataset_id', None, 'Dataset Id which contains the query' ' table.') flags.DEFINE_boolean('collect_output', False, 'Flag indicating if query results' ' should be collected.') flags.DEFINE_string('output_table', 'result_table', 'Table for query output.') flags.mark_flags_as_required(['bq_project_id', 'bq_dataset_id']) FLAGS = flags.FLAGS def generate_provider_specific_cmd_list(script, driver, output, error): """Method to compile the BigQuery specific script execution command. Arguments: script: SQL script which contains the query. driver: Driver that contains the BigQuery specific script executor. output: Output log file. error: Error log file. Returns: Command list to execute the supplied script.
os.make_dirs(probing_dir) with open(settings_path, 'w') as settings_file: settings = { 'task_name': task_name, 'trial_id': trial_id, 'train_size': len(train_indexes), 'dev_size': len(dev_indexes), 'test_size': len(test_indexes), 'train_size_unique': len(set(train_indexes)), 'dev_size_unique': len(set(dev_indexes)), 'test_size_unique': len(set(test_indexes)), } logging.info('Settings:\n%r', settings) json.dump(settings, settings_file, indent=2) with open(data_out_path, 'w') as data_file: # Don't add quoting to retain the original format unaltered. df[['set', 'target', 'text']].to_csv(data_file, sep='\t', header=False, index=False, quoting=csv.QUOTE_NONE, doublequote=False) if __name__ == '__main__': flags.mark_flags_as_required(['senteval_path', 'base_out_dir']) app.run(main)
from google.cloud import bigtable from google.cloud.bigtable import row_filters from tqdm import tqdm from tensorflow import gfile import sgf_wrapper from bigtable_input import METADATA, TABLE_STATE flags.DEFINE_string( 'sgf_glob', None, 'Glob for SGFs to backfill into eval_games bigtable.') flags.mark_flags_as_required([ 'sgf_glob', 'cbt_project', 'cbt_instance', 'cbt_table' ]) FLAGS = flags.FLAGS # Constants EVAL_PREFIX = 'e_{:0>10}' EVAL_GAME_COUNTER = b'eval_game_counter' SGF_FILENAME = b'sgf' #### Common Filters EVAL_COUNT_FILTER = row_filters.ColumnRangeFilter( METADATA, EVAL_GAME_COUNTER, EVAL_GAME_COUNTER)
A tuple of the evaluation metrics, and the exported objects from Estimator. """ hparams = get_hparams(hparams_set_name) label_vocab = parse_label_vocab(label_vocab_path) (estimator, train_spec, eval_spec) = _make_estimator_and_inputs(hparams=hparams, label_vocab=label_vocab, data_base_path=data_base_path, output_dir=output_dir, train_fold=train_fold, eval_fold=eval_fold) return tf.estimator.train_and_evaluate(estimator=estimator, train_spec=train_spec, eval_spec=eval_spec) def main(_): train(data_base_path=FLAGS.data_base_path, output_dir=FLAGS.output_dir, label_vocab_path=FLAGS.label_vocab_path, hparams_set_name=FLAGS.hparams_set, train_fold=FLAGS.train_fold, eval_fold=FLAGS.eval_fold) if __name__ == '__main__': FLAGS.alsologtostderr = True # Shows training output. flags.mark_flags_as_required(['data_base_path', 'label_vocab_path']) app.run(main)
if not FLAGS.signer_key: raise app.UsageError("ERROR: --signer_key flag not set.") if not is_signature_valid(json_data, FLAGS.signature, FLAGS.signer_key): raise app.UsageError( "ERROR: Signature over list of logs is not valid.") parsed_json = json.loads(json_data) if not FLAGS.log_list_schema: raise app.UsageError("ERROR: --log_list_schema flag not set.") if not is_log_list_valid(parsed_json, FLAGS.log_list_schema): raise app.UsageError( "ERROR: Log list is signed but does not conform to the schema.", 2) if FLAGS.header_output: generate_cpp_header(parsed_json, FLAGS.header_output) if FLAGS.java_output: generate_java_source(parsed_json, FLAGS.java_output, FLAGS.java_class) if FLAGS.openssl_output: generate_openssl_conf(parsed_json, FLAGS.openssl_output) if not FLAGS.header_output and \ not FLAGS.java_output and \ not FLAGS.openssl_output: print_formatted_log_list(parsed_json) if __name__ == "__main__": gflags.mark_flags_as_required(["log_list", "log_list_schema"]) gflags.mark_flags_as_mutual_exclusive(["signature", "skip_signature_check"], required=True) app.run(main)
Contains the method to compile the Azure SQL data warehouse specific script execution command based on generic arguments (sql script, output destination) and Azure SQL specific arguments (flag values). """ __author__ = '*****@*****.**' from absl import flags flags.DEFINE_string('server', None, 'SQL server.') flags.DEFINE_string('database', None, 'SQL Database.') flags.DEFINE_string('user', None, 'SQL User.') flags.DEFINE_string('password', None, 'SQL Password.') flags.mark_flags_as_required(['server', 'database', 'user', 'password']) FLAGS = flags.FLAGS def generate_provider_specific_cmd_list(script, driver, output, error): """Method to compile the Azure SQL specific script execution command. Arguments: script: SQL script which contains the query. driver: Driver that contains the Azure SQL data warehouse specific script executor. output: Output log file. error: Error log file. Returns: