def __init__(self, config: TrainerConfig):
"""
Args:
config (TrainerConfig): configuration used to construct this trainer
"""
Trainer._trainer_progress = _TrainerProgress()
root_dir = os.path.expanduser(config.root_dir)
os.makedirs(root_dir, exist_ok=True)
logging.get_absl_handler().use_absl_log_file(log_dir=root_dir)
self._root_dir = root_dir
self._train_dir = os.path.join(root_dir, 'train')
self._eval_dir = os.path.join(root_dir, 'eval')
self._algorithm_ctor = config.algorithm_ctor
self._algorithm = None
self._num_checkpoints = config.num_checkpoints
self._checkpointer = None
self._evaluate = config.evaluate
self._eval_interval = config.eval_interval
self._summary_interval = config.summary_interval
self._summaries_flush_secs = config.summaries_flush_secs
self._summary_max_queue = config.summary_max_queue
self._debug_summaries = config.debug_summaries
self._summarize_grads_and_vars = config.summarize_grads_and_vars
self._config = config
self._random_seed = common.set_random_seed(config.random_seed)
def main(_):
def get_run_name() -> str:
from rl_project import _get_run_name
return _get_run_name(
algorithm=FLAGS.algorithm,
scenario_number=FLAGS.scenario_number,
number_of_steps=FLAGS.number_of_steps,
number_of_envs=FLAGS.number_of_envs,
)
run_name = get_run_name()
logger.get_absl_handler().use_absl_log_file(run_name, './')
logger.info("STARTED")
time_start = time.time()
from rl_project.trainer import create_rl_algorithm_model
trained_model = create_rl_algorithm_model(
algorithm=FLAGS.algorithm,
algorithm_policy=FLAGS.algorithm_policy,
scenario_number=FLAGS.scenario_number,
number_of_envs=FLAGS.number_of_envs,
representation=FLAGS.representation,
stacked=FLAGS.stacked,
).learn(FLAGS.number_of_steps)
time_elapsed = time.time() - time_start
logger.info("Time elapsed " +
time.strftime("%H:%M:%S", time.gmtime(time_elapsed)))
trained_model.save(save_path=run_name)
def main(_):
if FLAGS.log_dir:
os.makedirs(FLAGS.log_dir, exist_ok=True)
logging.get_absl_handler().use_absl_log_file(FLAGS.task, FLAGS.log_dir)
logging.debug('Arguments:')
for k, v in FLAGS.flag_values_dict().items():
logging.debug(f'- {k}: {v}')
# Load original data
idir = os.path.join(FLAGS.data_npy_dir, FLAGS.input_name)
loader = DataLoader(idir)
# Find feature indices
idxs = []
for f in FLAGS.features:
i = np.argwhere(loader.feature == f)
assert len(i) >= 1, f'Unknown feature "{f}"!'
assert len(i) <= 1, f'Duplicated feature "{f}"!'
idxs.append(i[0, 0])
idxs = np.asarray(idxs)
# Select feature and columns of X
loader.feature = loader.feature[idxs]
loader.train_set.x = loader.train_set.x[:, idxs]
loader.test_set.x = loader.test_set.x[:, idxs]
# Dump new data
odir = os.path.join(FLAGS.data_npy_dir, FLAGS.output_name)
loader.dump(loader.feature,
loader.train_set,
loader.test_set,
output_dir=odir)
def __init__(self, flags):
super().__init__(flags)
self.seed_everything()
if not os.path.exists(flags.summaries_dir):
os.makedirs(flags.summaries_dir)
logging.get_absl_handler().use_absl_log_file(
program_name='DeePray',
log_dir=flags.summaries_dir
)
logging.info(' {} Initialize training'.format(
time.strftime("%Y%m%d %H:%M:%S")))
if 'use_autotuner' in flags.__dict__:
self.flags = flags
logging.info(self.flags)
else:
self.flags = FLAGS
logging.info('\ttf.app.flags.FLAGS:')
for key, value in sorted(self.flags.flag_values_dict().items()):
logging.info('\t{:25}= {}'.format(key, value))
self.next_step_to_trace = 2
self.batch_size = self.flags.prebatch * self.flags.batch_size
self.best_loss = float('Inf')
self.best_checkpoint = None
self.patient_pass = 0
self.max_patient_passes = self.flags.patient_valid_passes
self.prediction_signature = None
self.last_model_path = None
self.LABEL, self.CATEGORY_FEATURES, self.NUMERICAL_FEATURES, \
self.VOC_SIZE, self.VARIABLE_FEATURES = self.get_summary()
self.metrics_object = self.build_metrics()
self.loss_object = self.build_loss()
def main(_):
def get_run_name() -> str:
return "EVALUATOR_{algorithm}_scenario-{scenario}_accuracy-{accuracy}".format(
algorithm=FLAGS.algorithm,
scenario=FLAGS.scenario_number,
accuracy=str(FLAGS.accuracy),
)
if FLAGS.path is None:
raise ValueError(
"path to trained model must be given. Please run script with --help option"
)
logger.get_absl_handler().use_absl_log_file(get_run_name(), './')
from rl_project.algorithm import load_model
model = load_model(FLAGS.path, FLAGS.algorithm)
from rl_project.evaluator import evaluate_model
total_reward = evaluate_model(
model=model,
scenario_number=FLAGS.scenario_number,
accuracy=FLAGS.accuracy,
reward=FLAGS.reward,
representation=FLAGS.representation,
stacked=FLAGS.stacked,
render=FLAGS.render,
logging=logger.info,
)
logger.info(
"EVALUATOR: ended with {runs} runs and receive in total reward: {reward}. Average reward: {average}"
.format(
runs=FLAGS.accuracy,
reward=total_reward,
average=total_reward / FLAGS.accuracy,
))
def setup_logging():
"""Logging related setup.
Creates the log directory if it does not exist and sets up the output of
logging to use absl log file.
"""
logging.set_verbosity(logging.INFO)
log_dir = ''
if FLAGS.log_dir:
log_dir = FLAGS.log_dir + LOG_FOLDER_NAME
else:
log_dir = logging.find_log_dir() + LOG_FOLDER_NAME
if FLAGS.enable_mount_directory:
log_dir = MOUNT_DIR + log_dir
try:
if not os.path.exists(log_dir):
os.makedirs(log_dir)
logging.get_absl_handler().use_absl_log_file('', log_dir)
except OSError as err:
logging.error('Failed to create log directory %s, err %s', log_dir,
str(err))
except Exception as ex: # pylint: disable=broad-except
logging.error('Unknown exception occurs in setup_logging, err: %s',
str(ex))
def train(self,
train_dataset,
valid_dataset,
test_dataset,
max_epochs,
valid_gap_epochs=1,
auto_save=True):
""" Run the training """
folder_path, model_path = self.make_dir()
logging.get_absl_handler().use_absl_log_file('logs', folder_path)
logging.info('Start training with max epochs {}'.format(max_epochs))
logging.info('Model and logs saved in {}'.format(folder_path))
logging.info('Number of trainable parameters - {}'.format(
self.num_params()))
if self.write_summary:
self.model_wrapper.init_summary_writer(folder_path)
best_eval_loss = float('inf')
for i in range(1, max_epochs + 1):
loss, prediction = self.run_one_epoch(train_dataset,
RunnerPhase.TRAIN, self.lr)
# Record the train loss
loss_metrics = {'loss': np.mean(loss)}
logging.info('Epoch {0} -- Training loss {1}'.format(
i, loss_metrics['loss']))
self.model_wrapper.write_summary(i, loss_metrics,
RunnerPhase.TRAIN)
# Evaluate the model
if i % valid_gap_epochs == 0:
loss, prediction = self.run_one_epoch(valid_dataset,
RunnerPhase.VALIDATE,
self.lr)
# Record the validation loss
loss_metrics = {'loss': np.mean(loss)}
logging.info('Epoch {0} -- Validation loss {1}'.format(
i, loss_metrics['loss']))
self.model_wrapper.write_summary(i, loss_metrics,
RunnerPhase.VALIDATE)
# If it is the best model on valid set
if best_eval_loss > loss_metrics['loss']:
metrics = self.evaluate(test_dataset)
self.model_wrapper.write_summary(i, metrics,
RunnerPhase.PREDICT)
best_eval_loss = loss_metrics['loss']
if auto_save:
self.model_wrapper.save(model_path)
if not auto_save:
self.model_wrapper.save(model_path)
else:
# use the saved model based on valid performance
self.restore(model_path)
logging.info('Training finished')
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
logging.get_absl_handler().use_absl_log_file()
logging.info('Opening %s', FLAGS.output_sqlite)
db = smu_sqlite.SMUSQLite(FLAGS.output_sqlite, 'c')
if FLAGS.bond_topology_csv:
logging.info('Starting smiles to btid inserts')
smiles_id_dict = smu_utils_lib.smiles_id_dict_from_csv(
open(FLAGS.bond_topology_csv))
db.bulk_insert_smiles(smiles_id_dict.items())
logging.info('Finished smiles to btid inserts')
else:
logging.info('Skipping smiles inserts')
logging.info('Starting main inserts')
dataset = tf.data.TFRecordDataset(gfile.glob(FLAGS.input_tfrecord))
db.bulk_insert((raw.numpy() for raw in dataset), batch_size=10000)
logging.info('Starting vacuuming')
db.vacuum()
logging.info('Vacuuming finished')
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
FLAGS.log_dir = FLAGS.workdir
FLAGS.stderrthreshold = 'info'
logging.get_absl_handler().start_logging_to_file()
# Hide any GPUs form TensorFlow. Otherwise TF might reserve memory and make
# it unavailable to JAX.
tf.config.experimental.set_visible_devices([], 'GPU')
logging.info('JAX host: %d / %d', jax.host_id(), jax.host_count())
logging.info('JAX local devices: %r', jax.local_devices())
# Add a note so that we can tell which task is which JAX host.
# (Depending on the platform task 0 is not guaranteed to be host 0)
platform.work_unit().set_task_status(
f'host_id: {jax.host_id()}, host_count: {jax.host_count()}')
platform.work_unit().create_artifact(platform.ArtifactType.DIRECTORY,
FLAGS.workdir, 'workdir')
if FLAGS.sample:
sample.save_images(sample.generate_sample(FLAGS.config, FLAGS.workdir),
'sample.png')
else:
train.train_and_evaluate(FLAGS.config, FLAGS.workdir)
def setup_logging_and_exp_folder():
# Random string if debug
if FLAGS.debug:
FLAGS.experiment = "{}_{}".format(FLAGS.mode, random_string())
# Use time stamp or user specified if not debug
else:
ts = time.time()
FLAGS.experiment = FLAGS.experiment if FLAGS.experiment is not None else \
"{}_{}".format(FLAGS.mode,
datetime.datetime.fromtimestamp(ts).strftime('%Y%m%d%H%M'))
training_folder = os.path.join(EXP_DIR, FLAGS.experiment)
# Create train folder
if os.path.exists(training_folder):
print('{} exists!'.format(training_folder))
exit(-1)
else:
os.makedirs(training_folder, exist_ok=False)
# set up logging
if FLAGS.debug:
logging.get_absl_handler().python_handler.stream = sys.stdout
else:
logging.get_absl_handler().use_absl_log_file('absl_logging',
training_folder)
return training_folder
def main(_):
if FLAGS.log_dir:
os.makedirs(FLAGS.log_dir, exist_ok=True)
logging.get_absl_handler().use_absl_log_file(FLAGS.task, FLAGS.log_dir)
logging.debug('Arguments:')
for k, v in FLAGS.flag_values_dict().items():
logging.debug(f'- {k}: {v}')
# Set paths
FLAGS.full_name = f'{FLAGS.model_name}_{FLAGS.input_name}'
FLAGS.input_dir = os.path.join(FLAGS.data_npy_dir, FLAGS.input_name)
FLAGS.model_file = os.path.join(FLAGS.result_model_dir, FLAGS.full_name)
FLAGS.figure_file = os.path.join(FLAGS.result_figure_dir, FLAGS.full_name)
# Run trainer
trainer = get_trainer()
if not FLAGS.no_train:
logging.info("==== Training ====")
trainer.run_train()
if not FLAGS.no_test:
logging.info("==== Testing ====")
trainer.run_test()
def main(_):
gin.parse_config_file(FLAGS.gin_file)
# warnings.filterwarnings('ignore') #TODO: revisit if this is necessary
logging.get_absl_handler().use_absl_log_file('mlserver_cxr')
ae = ApplicationEntity()
ae.start_server(evt_handlers=Helper().handlers)
def main(argv):
"""Create directories and configure python settings"""
# Setup Directory
experiment_dir = os.path.join(FLAGS.dir, FLAGS.id)
if not os.path.exists(experiment_dir):
os.makedirs(os.path.join(experiment_dir, "logs"), exist_ok=True)
# Setup Logging
FLAGS.alsologtostderr = True
logging.get_absl_handler().use_absl_log_file(
FLAGS.logfile, os.path.join(experiment_dir, "logs"))
# Setup seeds
if FLAGS.random_seed:
np.random.seed(FLAGS.random_seed)
tf.random.set_seed(FLAGS.random_seed)
# Log Flags
log_flags(FLAGS)
try:
experiment()
except:
exception = traceback.format_exc()
logging.info(exception)
def main(argv):
del argv
with open(FLAGS.config_path) as json_data_file:
config = json.load(json_data_file)
config = munch.munchify(config)
bsde = getattr(eqn, config.eqn_config.eqn_name)(config.eqn_config)
tf.keras.backend.set_floatx(config.net_config.dtype)
if not os.path.exists(FLAGS.log_dir):
os.mkdir(FLAGS.log_dir)
path_prefix = os.path.join(FLAGS.log_dir, FLAGS.exp_name)
with open('{}_config.json'.format(path_prefix), 'w') as outfile:
json.dump(dict((name, getattr(config, name)) for name in dir(config)
if not name.startswith('__')),
outfile,
indent=2)
absl_logging.get_absl_handler().setFormatter(
logging.Formatter('%(levelname)-6s %(message)s'))
absl_logging.set_verbosity('info')
logging.info('Begin to solve %s ' % config.eqn_config.eqn_name)
bsde_solver = BSDESolver(config, bsde)
training_history = bsde_solver.train()
if bsde.y_init:
logging.info('Y0_true: %.4e' % bsde.y_init)
logging.info(
'relative error of Y0: %s', '{:.2%}'.format(
abs(bsde.y_init - training_history[-1, 2]) / bsde.y_init))
np.savetxt('{}_training_history.csv'.format(path_prefix),
training_history,
fmt=['%d', '%.5e', '%.5e', '%d'],
delimiter=",",
header='step,loss_function,target_value,elapsed_time',
comments='')
def logging_config(to_file=True, log_dir=None, program_name='more_keras'):
if to_file and log_dir is not None:
log_dir = os.path.expanduser(os.path.expandvars(log_dir))
if not os.path.exists(log_dir):
os.makedirs(log_dir)
logging.info('Logging to {}'.format(log_dir))
logging.get_absl_handler().use_absl_log_file(log_dir=log_dir,
program_name=program_name)
def main(_):
opt = FLAGS
# logging
logging.set_verbosity(logging.INFO)
logging.set_stderrthreshold(logging.INFO)
if FLAGS.log_dir:
if not os.path.exists(FLAGS.log_dir):
os.makedirs(FLAGS.log_dir)
logging.get_absl_handler().use_absl_log_file(FLAGS.dataset,
log_dir=FLAGS.log_dir)
# dataset
if opt.dataset == 'mnist':
data_train, data_test = tf.keras.datasets.mnist.load_data()
elif opt.dataset == 'cifar10':
data_train, data_test = tf.keras.datasets.cifar10.load_data()
else:
raise NotImplementError
x_train, y_train = data_train
x_test, y_test = data_test
x_train = x_train.astype(np.float32)
x_test = x_test.astype(np.float32)
y_train = y_train.reshape([
-1,
])
y_test = y_test.reshape([
-1,
])
# resize to (32, 32)
if opt.dataset == 'mnist':
x_train = batch_resize(x_train, (32, 32))[..., None]
x_test = batch_resize(x_test, (32, 32))[..., None]
# normalization
mean = x_train.mean()
stddev = x_train.std()
x_train = (x_train - mean) / stddev
x_test = (x_test - mean) / stddev
logging.info('{}, {}'.format(x_train.shape, x_test.shape))
# define abnoraml data and normal
# training data only contains normal
x_train = x_train[y_train != opt.anomaly, ...]
y_train = y_train[y_train != opt.anomaly, ...]
y_test = (y_test == opt.anomaly).astype(np.float32)
# tf.data.Dataset
train_dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train))
test_dataset = tf.data.Dataset.from_tensor_slices((x_test, y_test))
train_dataset = train_dataset.shuffle(opt.shuffle_buffer_size).batch(
opt.batch_size, drop_remainder=True)
test_dataset = test_dataset.batch(opt.batch_size, drop_remainder=False)
# training
ganomaly = GANomaly(opt,
train_dataset,
valid_dataset=None,
test_dataset=test_dataset)
ganomaly.fit(opt.niter)
# evaluating
ganomaly.evaluate_best(test_dataset)
def setup_logging_handler(log_dir):
"""Sets up logging.
Args:
log_dir (str): Path to directory where logs should be saved.
"""
if log_dir:
os.makedirs(log_dir, exist_ok=True)
program_name = os.path.splitext(os.path.basename(sys.argv[0]))[0]
logging.get_absl_handler().use_absl_log_file(program_name, log_dir)
def set_test_logging(self):
self.log_path = os.path.join(self.workspace, 'test_log')
if not os.path.exists(self.log_path):
os.mkdir(self.log_path)
logging.flush()
logging.get_absl_handler().use_absl_log_file(self.exp_name + '.log',
self.log_path)
def main(argv):
logging.get_absl_handler().start_logging_to_file(FLAGS.log_file)
print("logging path: ", logging.get_log_file_name())
logging.info('Running under Python {0[0]}.{0[1]}.{0[2]}'.format(
sys.version_info))
logging.info("logging level: %d" % logging.get_verbosity())
for i in range(50):
logging.log_every_n(logging.INFO, "log_every_10", 10)
def main(_):
model_helpers.apply_clean(flags.FLAGS)
logdir = '/tmp/logs'
if not os.path.exists(logdir):
os.makedirs(logdir)
logname = 'imagenet_strategy_{}_model_{}_node_{}_gpu_{}_patch_{}_proxy_{}'.format(
flags.FLAGS.autodist_strategy, flags.FLAGS.cnn_model, node_num,
gpu_num, flags.FLAGS.autodist_patch_tf, flags.FLAGS.proxy)
logging.get_absl_handler().use_absl_log_file(logname, logdir)
with logger.benchmark_context(flags.FLAGS):
run(flags.FLAGS)
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
logging.get_absl_handler().use_absl_log_file()
logging.info('Opening %s', FLAGS.output_sqlite)
db = smu_sqlite.SMUSQLite(FLAGS.output_sqlite, 'c')
dataset = tf.data.TFRecordDataset(gfile.glob(FLAGS.input_tfrecord))
db.bulk_insert((raw.numpy() for raw in dataset), batch_size=10000)
def main(argv):
# Don't know the purpose of the following line, seems to be related to different os
del argv
# Load config file
with open(FLAGS.config_path) as json_data_file:
config = json.load(json_data_file)
# A munch is a python dictionary type, subclass of dict
config = munch.munchify(config)
# Get eqn_name problem object from equation.py and generate object using eqn_config
### --> See equation.py for details
bsde = getattr(eqn, config.eqn_config.eqn_name)(config.eqn_config)
# Set dtype globally
tf.keras.backend.set_floatx(config.net_config.dtype)
# Path for logging
if not os.path.exists(FLAGS.log_dir):
os.mkdir(FLAGS.log_dir)
path_prefix = os.path.join(FLAGS.log_dir, FLAGS.exp_name)
# Copy used configuration to log directory
with open('{}_config.json'.format(path_prefix), 'w') as outfile:
json.dump(dict((name, getattr(config, name)) for name in dir(config)
if not name.startswith('__')),
outfile,
indent=2)
# ABSL - Abseil consists of source code repositories for C++ and Python
absl_logging.get_absl_handler().setFormatter(
logging.Formatter('%(levelname)-6s %(message)s'))
absl_logging.set_verbosity('info')
# Start logging
logging.info('Begin to solve %s ' % config.eqn_config.eqn_name)
# Call BSDE solver with config and problem equation
bsde_solver = BSDESolver(config, bsde)
training_history = bsde_solver.train()
# If explicit solution is available, print some final statistics
if bsde.y_init:
logging.info('Y0_true: %.4e' % bsde.y_init)
logging.info(
'relative error of Y0: %s', '{:.2%}'.format(
abs(bsde.y_init - training_history[-1, 2]) / bsde.y_init))
np.savetxt('{}_training_history.csv'.format(path_prefix),
training_history,
fmt=['%d', '%.5e', '%.5e', '%d'],
delimiter=",",
header='step,loss_function,target_value,elapsed_time',
comments='')
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
logging.get_absl_handler().use_absl_log_file()
logging.info('Opening %s', FLAGS.input_sqlite)
db = smu_sqlite.SMUSQLite(FLAGS.input_sqlite, 'r')
if FLAGS.output_format == OutputFormat.pbtxt:
outputter = PBTextOutputter(FLAGS.output_path)
elif FLAGS.output_format == OutputFormat.sdf_init:
outputter = SDFOutputter(
FLAGS.output_path, init_geometry=True, opt_geometry=False)
elif FLAGS.output_format == OutputFormat.sdf_opt:
outputter = SDFOutputter(
FLAGS.output_path, init_geometry=False, opt_geometry=True)
elif FLAGS.output_format == OutputFormat.sdf_init_opt:
outputter = SDFOutputter(
FLAGS.output_path, init_geometry=True, opt_geometry=True)
elif FLAGS.output_format == OutputFormat.atomic_input:
outputter = AtomicInputOutputter(FLAGS.output_path)
elif FLAGS.output_format == OutputFormat.tfdata:
outputter = TfDataOutputter(FLAGS.output_path)
else:
raise ValueError(f'Bad output format {FLAGS.output_format}')
if FLAGS.redetect_geometry:
outputter = ReDetectTopologiesOutputter(outputter)
with contextlib.closing(outputter):
for cid in (int(x) for x in FLAGS.cids):
conformer = db.find_by_conformer_id(cid)
outputter.output(conformer)
for btid in (int(x) for x in FLAGS.btids):
conformers = db.find_by_bond_topology_id(btid)
if not conformers:
raise KeyError(f'Bond topology {btid} not found')
for c in conformers:
outputter.output(c)
for smiles in FLAGS.smiles:
conformers = db.find_by_smiles(smiles)
if not conformers:
raise KeyError(f'SMILES {smiles} not found')
for c in conformers:
outputter.output(c)
for smiles in FLAGS.topology_query_smiles:
for c in topology_query(db, smiles):
outputter.output(c)
if FLAGS.random_fraction:
for conformer in db:
if conformer.fate == dataset_pb2.Conformer.FATE_SUCCESS and random.random(
) < FLAGS.random_fraction:
outputter.output(conformer)
def get_logger():
"""Retrieves tensorflow logger and changes log formatting."""
formatting = "%(asctime)s: %(levelname)s %(filename)s:%(lineno)s] %(message)s"
formatter = logging.Formatter(formatting)
absl_logging.get_absl_handler().setFormatter(formatter)
for h in tf.get_logger().handlers:
h.setFormatter(formatter)
logger = tf.get_logger()
logger.setLevel(logging.INFO)
return logger
def main(argv):
"""driver for flask application"""
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
# absl logging set to file logging if specified
if FLAGS.log_dir:
if not os.path.exists(FLAGS.log_dir):
os.makedirs(FLAGS.log_dir)
logging.get_absl_handler().use_absl_log_file()
logging.info('Techex started')
flask_app = _create_flask_app()
flask_app.run(host=FLAGS.flask_host, port=FLAGS.flask_port)
def main(_):
logging.get_absl_handler().setFormatter(CustomPythonFormatter())
logging.set_verbosity(logging.DEBUG)
model = IMDBSentimentClassifier(learning_rate=FLAGS.lr)
trainer = pl.Trainer(
default_root_dir='logs',
gpus=(1 if th.cuda.is_available() else 0),
max_epochs=FLAGS.epochs,
fast_dev_run=FLAGS.debug,
logger=pl.loggers.TensorBoardLogger('logs/', name='imdb', version=0),
)
trainer.fit(model)
def main(argv):
"""Create directories and configure python settings"""
# Setup Directory
experiment_dir = os.path.join(FLAGS.dir, FLAGS.id)
if not os.path.exists(experiment_dir):
os.makedirs(os.path.join(experiment_dir, "logs"), exist_ok=True)
# Setup Logging
FLAGS.alsologtostderr = True
logging.get_absl_handler().use_absl_log_file(
FLAGS.logfile, os.path.join(experiment_dir, "logs"))
# Setup Distributed
if FLAGS.distributed:
try:
hvd.init()
gpus = tf.config.list_physical_devices('GPU')
logging.info("Distributed training enabled.")
logging.info("GPUS: %s", str(gpus))
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
if gpus:
tf.config.experimental.set_visible_devices(
gpus[hvd.local_rank()], 'GPU')
FLAGS.model_cfgs = (FLAGS.model_cfgs +
",distributed=True").strip(',')
except:
logging.info(
"Distributed training training setup failed. Disabling distributed training."
)
if FLAGS.random_seed:
logging.info("Setting seed to %s", FLAGS.random_seed + hvd.rank())
np.random.seed(FLAGS.random_seed + hvd.rank())
tf.random.set_seed(FLAGS.random_seed + hvd.rank())
else:
# Setup seeds
if FLAGS.random_seed:
logging.info("Setting seed to %s", FLAGS.random_seed)
np.random.seed(FLAGS.random_seed)
tf.random.set_seed(FLAGS.random_seed)
# Log Flags
if (not FLAGS.distributed) or (hvd.rank() == 0):
log_flags(FLAGS)
try:
experiment()
except:
exception = traceback.format_exc()
logging.info(exception)
def main(argv):
del argv # Unused.
test_name = os.environ.get('TEST_NAME', None)
test_fn = globals().get('_test_%s' % test_name)
if test_fn is None:
raise AssertionError('TEST_NAME must be set to a valid value')
# Flush so previous messages are written to file before we switch to a new
# file with use_absl_log_file.
logging.flush()
if os.environ.get('USE_ABSL_LOG_FILE') == '1':
logging.get_absl_handler().use_absl_log_file('absl_log_file', FLAGS.log_dir)
test_fn()
def main(unused_argv) -> None:
logging.get_absl_handler().use_absl_log_file()
es = None
try:
while True:
logging.info('Starting Run')
es = ExitSpeed()
es.Run()
except KeyboardInterrupt:
logging.info('Keyboard interrupt')
finally:
if hasattr(es, 'point'):
logging.info('Logging last point\n %s', es.point)
logging.info('Done.\nExiting.')
logging.exception('Ensure we log any exceptions')
def __init__(self) -> None:
rest = gerrit.get_gerrit_rest_api(COOKIE_JAR_PATH, GERRIT_URL)
self.gerrit = gerrit.Gerrit(rest)
self.gerrit_git = git.GerritGit(
git_dir='gerrit_git_dir',
cookie_jar_path=COOKIE_JAR_PATH,
url=GOB_URL,
project='linux/kernel/git/torvalds/linux',
branch='master')
self.message_dao = message_dao.MessageDao()
self.archive_index = ArchiveMessageIndex(self.message_dao)
self.last_hash = self.message_dao.get_last_hash()
archive_updater.setup_archive(GIT_PATH)
os.makedirs(FILE_DIR, exist_ok=True)
os.makedirs(LOG_PATH, exist_ok=True)
logging.get_absl_handler().use_absl_log_file('server_logs', LOG_PATH)
