def resolve_proj_root(args):
'''Update proj_root based on how DVSim is invoked.
If --remote env var is set, a location in the scratch area is chosen as the
new proj_root. The entire repo is copied over to this location. Else, the
proj_root is discovered using get_proj_root() method, unless the user
overrides it on the command line.
This function returns the updated proj_root src and destination path. If
--remote env var is not set, the destination path is identical to the src path.
'''
proj_root_src = args.proj_root or get_proj_root()
# Check if jobs are dispatched to external compute machines. If yes,
# then the repo needs to be copied over to the scratch area
# accessible to those machines.
# If --purge arg is set, then purge the repo_top that was copied before.
if args.remote:
proj_root_dest = os.path.join(args.scratch_root, args.branch,
"repo_top")
if args.purge:
rm_path(proj_root_dest)
copy_repo(proj_root_src, proj_root_dest, args.dry_run)
else:
proj_root_dest = proj_root_src
return proj_root_src, proj_root_dest
def odir_limiter(self, odir):
"""Clean previous output directories.
When running jobs, we may want to maintain a limited history of
previous invocations. This method finds and deletes the output
directories at the base of input arg 'odir' with the oldest timestamps,
if that limit is reached. It returns a list of directories that
remain after deletion.
"""
if not os.path.exists(odir):
return []
# If output directory exists, back it up.
ts = datetime.fromtimestamp(os.stat(odir).st_ctime)
ts = ts.strftime(self.sim_cfg.ts_format)
shutil.move(odir, odir + "_" + ts)
# Get list of past output directories sorted by creation time.
pdir = Path(odir).resolve().parent
dirs = sorted([old for old in pdir.iterdir() if old.is_dir()],
key=os.path.getctime,
reverse=True)
for old in dirs[self.max_odirs - 1:]:
rm_path(old)
return dirs[0:self.max_odirs - 2]
def post_finish(self, status):
"""Extract the coverage results summary for the dashboard.
If that fails for some reason, report the job as a failure.
"""
if self.dry_run or status != 'P':
return
results, self.cov_total, ex_msg = get_cov_summary_table(
self.cov_report_txt, self.sim_cfg.tool)
if ex_msg:
self.launcher.fail_msg += ex_msg
log.error(ex_msg)
return
# Succeeded in obtaining the coverage data.
colalign = (("center", ) * len(results[0]))
self.cov_results = tabulate(results,
headers="firstrow",
tablefmt="pipe",
colalign=colalign)
# Delete the cov report - not needed.
rm_path(self.get_log_path())
def _test_passed(self):
# Add an extra check to Deploy._test_passed where we extract the
# coverage results summary for the dashboard (and fail the job if
# something goes wrong).
if not super()._test_passed():
return False
results, self.cov_total, ex_msg = get_cov_summary_table(
self.cov_report_txt, self.sim_cfg.tool)
if ex_msg:
self.fail_msg += ex_msg
log.error(ex_msg)
return False
# Succeeded in obtaining the coverage data.
colalign = (("center", ) * len(results[0]))
self.cov_results = tabulate(results,
headers="firstrow",
tablefmt="pipe",
colalign=colalign)
# Delete the cov report - not needed.
rm_path(self.log)
return True
def publish_results_summary(self):
'''Public facing API for publishing md format results to the opentitan
web server.
'''
results_html_file = "summary_" + self.timestamp + ".html"
results_page_url = self.results_summary_server_page.replace(
self.results_server_prefix, self.results_server_url_prefix)
# Publish the results page.
# First, write the results html file temporarily to the scratch area.
f = open(results_html_file, 'w')
f.write(
md_results_to_html(self.results_title, self.css_file,
self.results_summary_md))
f.close()
log.info("Publishing results summary to %s", results_page_url)
cmd = (self.results_server_cmd + " cp " + results_html_file + " " +
self.results_summary_server_page)
log.log(VERBOSE, cmd)
try:
cmd_output = subprocess.run(args=cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
log.log(VERBOSE, cmd_output.stdout.decode("utf-8"))
except Exception as e:
log.error("%s: Failed to publish results:\n\"%s\"", e, str(cmd))
rm_path(results_html_file)
def run():
# paths to data
content_paths = []
for c in FLAGS.contents:
p = Path(c)
if not p.exists():
raise ValueError('The content image or directory is not exist: {}'.format(p))
if p.is_dir():
for f in p.glob('**/*.*'):
content_paths.append(f)
else:
content_paths.append(p)
style_path = Path(FLAGS.style)
if not style_path.exists():
raise ValueError('The style image is not exist: {}'.format(style_path))
# output directory
output_dir = Path(FLAGS.output) / style_path.stem
if output_dir.exists():
logging.warning('The folder will be deleted: {}'.format(output_dir))
rm_path(output_dir)
output_dir.mkdir(exist_ok=True, parents=True)
# create model
if not Path(FLAGS.decoder).exists():
raise ValueError('The decoder model is not found: {}'.format(FLAGS.decoder))
encoder = Encoder(input_shape=(None, None, 3), pretrained=True)
content_feature_input = Input(shape=encoder.output_shape[-1][1:])
style_feature_input = Input(shape=encoder.output_shape[-1][1:])
adain = AdaIN(alpha=FLAGS.alpha)
adain = Model(inputs=[content_feature_input, style_feature_input], outputs=[adain([content_feature_input, style_feature_input])])
decoder = Decoder(input_shape=encoder.output_shape[-1][1:])
decoder.load_weights(FLAGS.decoder)
# load and encode style image
style = np.expand_dims(load_image(style_path, image_shape=(FLAGS.style_size, FLAGS.style_size)), axis=0)
style_feature = encoder.predict(style)[-1]
for content_path in tqdm(content_paths):
# load and encode content image
content = load_image(content_path)
content = np.expand_dims(content, axis=0)
content_feature = encoder.predict(content)[-1]
# normalize the feature
normalized_feature = adain.predict([content_feature, style_feature])
# generate image
generated = decoder.predict(normalized_feature)
# save image
img_path = output_dir / '{}.{}'.format(content_path.stem, FLAGS.ext)
img = array_to_img(generated[0])
img.save(img_path)
def _link_odir(self, status):
"""Soft-links the job's directory based on job's status.
The dispatched, passed and failed directories in the scratch area
provide a quick way to get to the job that was executed.
"""
dest = Path(self.deploy.sim_cfg.links[status], self.deploy.qual_name)
mk_symlink(self.deploy.odir, dest)
# Delete the symlink from dispatched directory if it exists.
if status != "D":
old = Path(self.deploy.sim_cfg.links['D'], self.deploy.qual_name)
rm_path(old)
def _link_odir(self, status):
'''Soft-links the job's directory based on job's status, into
dispatched, running, passed, failed or killed directories in the
scratch area.'''
dest = Path(self.sim_cfg.links[status], self.odir_ln)
# If dest exists, then atomically remove it and link the odir again.
while True:
try:
os.symlink(self.odir, dest)
break
except FileExistsError:
rm_path(dest)
# Delete the symlink from dispatched directory if it exists.
if status != "D":
old = Path(self.sim_cfg.links['D'], self.odir_ln)
rm_path(old)
def _link_odir(self, status):
"""Soft-links the job's directory based on job's status.
The dispatched, passed and failed directories in the scratch area
provide a quick way to get to the job that was executed.
"""
dest = Path(self.deploy.sim_cfg.links[status], self.deploy.qual_name)
# If dest exists, then atomically remove it and link the odir again.
while True:
try:
os.symlink(self.deploy.odir, dest)
break
except FileExistsError:
rm_path(dest)
# Delete the symlink from dispatched directory if it exists.
if status != "D":
old = Path(self.deploy.sim_cfg.links['D'], self.deploy.qual_name)
rm_path(old)
def _load_child_cfg(self, entry, mk_config):
'''Load a child configuration for a primary cfg'''
if type(entry) is str:
# Treat this as a file entry. Substitute wildcards in cfg_file
# files since we need to process them right away.
cfg_file = subst_wildcards(entry, self.__dict__, ignore_error=True)
self.cfgs.append(self.create_instance(mk_config, cfg_file))
elif type(entry) is dict:
# Treat this as a cfg expanded in-line
temp_cfg_file = self._conv_inline_cfg_to_hjson(entry)
if not temp_cfg_file:
return
self.cfgs.append(self.create_instance(mk_config, temp_cfg_file))
# Delete the temp_cfg_file once the instance is created
log.log(VERBOSE, "Deleting temp cfg file:\n%s", temp_cfg_file)
rm_path(temp_cfg_file, ignore_error=True)
else:
log.error(
"Type of entry \"%s\" in the \"use_cfgs\" key is invalid: %s",
entry, str(type(entry)))
sys.exit(1)
def _publish_results(self):
'''Publish results to the opentitan web server.
Results are uploaded to {results_server_path}/latest/results.
If the 'latest' directory exists, then it is renamed to its 'timestamp'
directory. If the list of directories in this area is > 14, then the
oldest entry is removed. Links to the last 7 regression results are
appended at the end if the results page.
'''
if which('gsutil') is None or which('gcloud') is None:
log.error("Google cloud SDK not installed! Cannot access the "
"results server")
return
# Construct the paths
results_page_url = self.results_server_page.replace(
self.results_server_prefix, self.results_server_url_prefix)
# Timeformat for moving the dir
tf = "%Y.%m.%d_%H.%M.%S"
# Extract the timestamp of the existing self.results_server_page
cmd = self.results_server_cmd + " ls -L " + self.results_server_page + \
" | grep \'Creation time:\'"
log.log(VERBOSE, cmd)
cmd_output = subprocess.run(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL)
log.log(VERBOSE, cmd_output.stdout.decode("utf-8"))
old_results_ts = cmd_output.stdout.decode("utf-8")
old_results_ts = old_results_ts.replace("Creation time:", "")
old_results_ts = old_results_ts.strip()
# Move the 'latest' to its timestamp directory if lookup succeeded
if cmd_output.returncode == 0:
try:
if old_results_ts != "":
ts = datetime.datetime.strptime(
old_results_ts, "%a, %d %b %Y %H:%M:%S %Z")
old_results_ts = ts.strftime(tf)
except ValueError as e:
log.error(
"%s: \'%s\' Timestamp conversion value error raised!", e)
old_results_ts = ""
# If the timestamp conversion failed - then create a dummy one with
# yesterday's date.
if old_results_ts == "":
log.log(VERBOSE,
"Creating dummy timestamp with yesterday's date")
ts = datetime.datetime.now(
datetime.timezone.utc) - datetime.timedelta(days=1)
old_results_ts = ts.strftime(tf)
old_results_dir = self.results_server_path + "/" + old_results_ts
cmd = (self.results_server_cmd + " mv " + self.results_server_dir +
" " + old_results_dir)
log.log(VERBOSE, cmd)
cmd_output = subprocess.run(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL)
log.log(VERBOSE, cmd_output.stdout.decode("utf-8"))
if cmd_output.returncode != 0:
log.error("Failed to mv old results page \"%s\" to \"%s\"!",
self.results_server_dir, old_results_dir)
# Do an ls in the results root dir to check what directories exist.
results_dirs = []
cmd = self.results_server_cmd + " ls " + self.results_server_path
log.log(VERBOSE, cmd)
cmd_output = subprocess.run(args=cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL)
log.log(VERBOSE, cmd_output.stdout.decode("utf-8"))
if cmd_output.returncode == 0:
# Some directories exist. Check if 'latest' is one of them
results_dirs = cmd_output.stdout.decode("utf-8").strip()
results_dirs = results_dirs.split("\n")
else:
log.log(VERBOSE, "Failed to run \"%s\"!", cmd)
# Start pruning
log.log(VERBOSE, "Pruning %s area to limit last 7 results",
self.results_server_path)
rdirs = []
for rdir in results_dirs:
dirname = rdir.replace(self.results_server_path, '')
dirname = dirname.replace('/', '')
if dirname == "latest":
continue
rdirs.append(dirname)
rdirs.sort(reverse=True)
rm_cmd = ""
history_txt = "\n## Past Results\n"
history_txt += "- [Latest](" + results_page_url + ")\n"
if len(rdirs) > 0:
for i in range(len(rdirs)):
if i < 7:
rdir_url = self.results_server_path + '/' + rdirs[
i] + "/" + self.results_server_html
rdir_url = rdir_url.replace(self.results_server_prefix,
self.results_server_url_prefix)
history_txt += "- [{}]({})\n".format(rdirs[i], rdir_url)
elif i > 14:
rm_cmd += self.results_server_path + '/' + rdirs[i] + " "
if rm_cmd != "":
rm_cmd = self.results_server_cmd + " -m rm -r " + rm_cmd + "; "
# Append the history to the results.
publish_results_md = self.publish_results_md or self.results_md
publish_results_md = publish_results_md + history_txt
# Publish the results page.
# First, write the results html file temporarily to the scratch area.
results_html_file = self.scratch_path + "/results_" + self.timestamp + \
".html"
f = open(results_html_file, 'w')
f.write(
md_results_to_html(self.results_title, self.css_file,
publish_results_md))
f.close()
log.info("Publishing results to %s", results_page_url)
cmd = (self.results_server_cmd + " cp " + results_html_file + " " +
self.results_server_page)
log.log(VERBOSE, cmd)
try:
cmd_output = subprocess.run(args=cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
log.log(VERBOSE, cmd_output.stdout.decode("utf-8"))
except Exception as e:
log.error("%s: Failed to publish results:\n\"%s\"", e, str(cmd))
rm_path(results_html_file)
def _create_dirs(self):
'''Create initial set of directories
'''
for link in self.links.keys():
rm_path(self.links[link])
os.makedirs(self.links[link])
def _purge(self):
assert self.scratch_path
log.info("Purging scratch path %s", self.scratch_path)
rm_path(self.scratch_path)
def pre_launch(self):
# Delete old coverage database directories before building again. We
# need to do this becuase build directory is not 'renewed'.
rm_path(self.cov_db_dir)
def _on_finish(self, status):
super()._on_finish(status)
if status != 'P':
# Delete the coverage data if available.
rm_path(self.cov_db_test_dir)
def dispatch_cmd(self):
# Delete old coverage database directories before building again. We
# need to do this becuase build directory is not 'renewed'.
rm_path(self.cov_db_dir)
super().dispatch_cmd()
def post_finish(self, status):
if status != 'P':
# Delete the coverage data if available.
rm_path(self.cov_db_test_dir)
def run():
# create directories
save_dir = Path(FLAGS.save_dir)
if save_dir.exists():
logging.warning('The directory can be overwritten: {}'.format(
FLAGS.save_dir))
save_dir.mkdir(exist_ok=True, parents=True)
log_dir = Path(FLAGS.tensorboard)
if log_dir.exists():
logging.warning('The directory will be removed: {}'.format(
FLAGS.tensorboard))
rm_path(log_dir)
log_dir.mkdir(exist_ok=True, parents=True)
# to handle errors while loading images
Image.MAX_IMAGE_PIXELS = None
ImageFile.LOAD_TRUNCATED_IMAGES = True
# image generator
dataset = ContentStyleLoader(content_root=FLAGS.content_dir,
content_image_shape=(FLAGS.image_size,
FLAGS.image_size),
content_crop='random',
content_crop_size=FLAGS.crop_size,
style_root=FLAGS.style_dir,
style_image_shape=(FLAGS.image_size,
FLAGS.image_size),
style_crop='random',
style_crop_size=FLAGS.crop_size,
n_per_epoch=FLAGS.dataset_size,
batch_size=FLAGS.batch_size)
# create model
encoder = Encoder(input_shape=(FLAGS.crop_size, FLAGS.crop_size, 3),
pretrained=True,
name='encoder')
# freeze the model
for l in encoder.layers:
l.trainable = False
adain = AdaIN(alpha=1.0, name='adain')
decoder = Decoder(input_shape=encoder.output_shape[-1][1:], name='decoder')
# place holders for inputs
content_input = Input(shape=(FLAGS.crop_size, FLAGS.crop_size, 3),
name='content_input')
style_input = Input(shape=(FLAGS.crop_size, FLAGS.crop_size, 3),
name='style_input')
# forwarding
content_features = encoder(content_input)
style_features = encoder(style_input)
normalized_feature = adain([content_features[-1], style_features[-1]])
generated = decoder(normalized_feature)
# loss calculation
generated_features = encoder(generated)
content_loss = Lambda(calculate_content_loss, name='content_loss')(
[normalized_feature, generated_features[-1]])
style_loss = Lambda(calculate_style_loss, name='style_loss')(
[style_features, generated_features])
loss = Lambda(
lambda x: FLAGS.content_weight * x[0] + FLAGS.style_weight * x[1],
name='loss')([content_loss, style_loss])
# trainer
trainer = Model(inputs=[content_input, style_input], outputs=[loss])
optim = optimizers.Adam(learning_rate=FLAGS.learning_rate)
trainer.compile(optimizer=optim, loss=lambda _, y_pred: y_pred)
trainer.summary()
# callbacks
callbacks = [
# learning rate scheduler
LearningRateScheduler(lambda epoch, _: FLAGS.learning_rate / (
1.0 + FLAGS.learning_rate_decay * FLAGS.dataset_size * epoch)),
# Tensor Board
TensorBoard(str(log_dir), write_graph=False, update_freq='batch'),
# save model
SubmodelCheckpoint(
str(save_dir / 'decoder.epoch-{epoch:d}.h5'),
submodel_name='decoder',
save_weights_only=True,
save_best_only=FLAGS.save_best_only,
save_freq=FLAGS.save_every if FLAGS.save_every else 'epoch')
]
# train
trainer.fit_generator(dataset,
epochs=FLAGS.epochs,
workers=FLAGS.workers,
callbacks=callbacks)
