def setup_substrate():
"""
Sets up the substrate
"""
#Constants
SUBSTRATE_SPEC = './substrate.yaml'
IMAGE="TBonTB_C4"
template = read_yaml(template_file=SUBSTRATE_SPEC)
server_manager = ServerManager(os.environ["OS_USERNAME"],
os.environ["OS_PASSWORD"],
os.environ["OS_REGION_NAME"],
os.environ["OS_TENANT_NAME"])
#First boot the controller
contr_flavor = template['Cloud'].get('controller_flavor', 'm1.medium')
contr_id = server_manager.create_server('span-contr-1', IMAGE, contr_flavor,
key_name='key_spandan', secgroups=['default','spandantb'])
#Next boot the agents
agent_flavor = template['Cloud'].get('agent_flavor', 'm1.medium')
agent_count = template['Cloud'].get('agent_count', 1)
agent_ids = []
for i in range(agent_count):
agent_ids.append(server_manager.create_server('span-agent-1', IMAGE, agent_flavor,
key_name='key_spandan', secgroups=['default','spandantb']))
contr_ip = server_manager.wait_until_sshable(contr_id)
agent_ips = [server_manager.wait_until_sshable(agent_id) for agent_id in agent_ids ]
return {'contr_ip':contr_ip, 'agents_ips':agent_ip}
def __init__(self, path, settings):
super(TravisRepoDispatcher, self).__init__(path, settings)
self._travisyml_path = os.path.join(self.repo_path, '.travis.yml')
if not os.path.exists(self._travisyml_path):
raise IOError('The file .travis.yml does not exist in the directory %s' % self.repo_path)
self._travisyml = utils.read_yaml(self._travisyml_path)
def animate(args):
# get context
ctx = get_extension_context(args.context)
nn.set_default_context(ctx)
logger.setLevel(logging.ERROR) # to supress minor messages
if not args.config:
assert not args.params, "pretrained weights file is given, but corresponding config file is not. Please give both."
download_provided_file(
"https://nnabla.org/pretrained-models/nnabla-examples/GANs/first-order-model/voxceleb_trained_info.yaml"
)
args.config = 'voxceleb_trained_info.yaml'
download_provided_file(
"https://nnabla.org/pretrained-models/nnabla-examples/GANs/first-order-model/pretrained_fomm_params.h5"
)
config = read_yaml(args.config)
dataset_params = config.dataset_params
model_params = config.model_params
if args.detailed:
vis_params = config.visualizer_params
visualizer = Visualizer(**vis_params)
if not args.params:
assert "log_dir" in config, "no log_dir found in config. therefore failed to locate pretrained parameters."
param_file = os.path.join(config.log_dir, config.saved_parameters)
else:
param_file = args.params
print(f"Loading {param_file} for image animation...")
nn.load_parameters(param_file)
bs, h, w, c = [1] + dataset_params.frame_shape
source = nn.Variable((bs, c, h, w))
driving_initial = nn.Variable((bs, c, h, w))
driving = nn.Variable((bs, c, h, w))
filename = args.driving
# process repeated until all the test data is used
driving_video = read_video(
filename, dataset_params.frame_shape) # (#frames, h, w, 3)
driving_video = np.transpose(driving_video,
(0, 3, 1, 2)) # (#frames, 3, h, w)
source_img = imread(args.source, channel_first=True,
size=(256, 256)) / 255.
source_img = source_img[:3]
source.d = np.expand_dims(source_img, 0)
driving_initial.d = driving_video[0][:3, ]
with nn.parameter_scope("kp_detector"):
kp_source = detect_keypoint(source,
**model_params.kp_detector_params,
**model_params.common_params,
test=True,
comm=False)
persistent_all(kp_source)
with nn.parameter_scope("kp_detector"):
kp_driving_initial = detect_keypoint(driving_initial,
**model_params.kp_detector_params,
**model_params.common_params,
test=True,
comm=False)
persistent_all(kp_driving_initial)
with nn.parameter_scope("kp_detector"):
kp_driving = detect_keypoint(driving,
**model_params.kp_detector_params,
**model_params.common_params,
test=True,
comm=False)
persistent_all(kp_driving)
if args.adapt_movement_scale:
nn.forward_all([
kp_source["value"], kp_source["jacobian"],
kp_driving_initial["value"], kp_driving_initial["jacobian"]
])
source_area = ConvexHull(kp_source['value'].d[0]).volume
driving_area = ConvexHull(kp_driving_initial['value'].d[0]).volume
adapt_movement_scale = np.sqrt(source_area) / np.sqrt(driving_area)
else:
adapt_movement_scale = 1
kp_norm = adjust_kp(kp_source=unlink_all(kp_source),
kp_driving=kp_driving,
kp_driving_initial=unlink_all(kp_driving_initial),
adapt_movement_scale=adapt_movement_scale,
use_relative_movement=args.unuse_relative_movement,
use_relative_jacobian=args.unuse_relative_jacobian)
persistent_all(kp_norm)
with nn.parameter_scope("generator"):
generated = occlusion_aware_generator(source,
kp_source=unlink_all(kp_source),
kp_driving=kp_norm,
**model_params.generator_params,
**model_params.common_params,
test=True,
comm=False)
if not args.full and 'sparse_deformed' in generated:
del generated['sparse_deformed'] # remove needless info
persistent_all(generated)
generated['kp_driving'] = kp_driving
generated['kp_source'] = kp_source
generated['kp_norm'] = kp_norm
# generated contains these values;
# 'mask': <Variable((bs, num_kp+1, h/4, w/4)) when scale_factor=0.25
# 'sparse_deformed': <Variable((bs, num_kp+1, num_channel, h/4, w/4)) # (bs, num_kp + 1, c, h, w)
# 'occlusion_map': <Variable((bs, 1, h/4, w/4))
# 'deformed': <Variable((bs, c, h, w))
# 'prediction': <Variable((bs, c, h, w))
mode = "arbitrary"
if "log_dir" in config:
result_dir = os.path.join(args.out_dir,
os.path.basename(config.log_dir), f"{mode}")
else:
result_dir = os.path.join(args.out_dir, "test_result", f"{mode}")
# create an empty directory to save generated results
_ = nm.Monitor(result_dir)
# load the header images.
header = imread("imgs/header_combined.png", channel_first=True)
generated_images = list()
# compute these in advance and reuse
nn.forward_all([kp_source["value"], kp_source["jacobian"]],
clear_buffer=True)
nn.forward_all(
[kp_driving_initial["value"], kp_driving_initial["jacobian"]],
clear_buffer=True)
num_of_driving_frames = driving_video.shape[0]
for frame_idx in tqdm(range(num_of_driving_frames)):
driving.d = driving_video[frame_idx][:3, ]
nn.forward_all([generated["prediction"], generated["deformed"]],
clear_buffer=True)
if args.detailed:
# visualize source w/kp, driving w/kp, deformed source, generated w/kp, generated image, occlusion map
visualization = visualizer.visualize(source=source.d,
driving=driving.d,
out=generated)
if args.full:
visualization = reshape_result(visualization) # (H, W, C)
combined_image = visualization.transpose(2, 0, 1) # (C, H, W)
elif args.only_generated:
combined_image = np.clip(generated["prediction"].d[0], 0.0, 1.0)
combined_image = (255 * combined_image).astype(
np.uint8) # (C, H, W)
else:
# visualize source, driving, and generated image
driving_fake = np.concatenate([
np.clip(driving.d[0], 0.0, 1.0),
np.clip(generated["prediction"].d[0], 0.0, 1.0)
],
axis=2)
header_source = np.concatenate([
np.clip(header / 255., 0.0, 1.0),
np.clip(source.d[0], 0.0, 1.0)
],
axis=2)
combined_image = np.concatenate([header_source, driving_fake],
axis=1)
combined_image = (255 * combined_image).astype(np.uint8)
generated_images.append(combined_image)
# once each video is generated, save it.
output_filename = f"{os.path.splitext(os.path.basename(filename))[0]}.mp4"
output_filename = f"{os.path.basename(args.source)}_by_{output_filename}"
output_filename = output_filename.replace("#", "_")
if args.output_png:
monitor_vis = nm.MonitorImage(output_filename,
nm.Monitor(result_dir),
interval=1,
num_images=1,
normalize_method=lambda x: x)
for frame_idx, img in enumerate(generated_images):
monitor_vis.add(frame_idx, img)
else:
generated_images = [_.transpose(1, 2, 0) for _ in generated_images]
# you might need to change ffmpeg_params according to your environment.
mimsave(f'{os.path.join(result_dir, output_filename)}',
generated_images,
fps=args.fps,
ffmpeg_params=[
"-pix_fmt", "yuv420p", "-vcodec", "libx264", "-f", "mp4",
"-q", "0"
])
return
parser.add_argument('--model',
'-m',
type=str,
default='vqvae',
help='Model Name to Train')
# Add more options for inference
return parser
if __name__ == '__main__':
parser = get_parser()
args = parser.parse_args()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
config = read_yaml(args.config_path)
if args.model == 'vqvae':
model = VQVAE(config).to(device)
trainer = VQVAETrainer(model, config)
elif args.model == 'glow':
model = Glow(config).to(device)
trainer = GlowTrainer(model, config)
elif args.model == 'factorvae':
vae_model = FactorVAE(config, device).to(device)
discriminator = Discriminator(config).to(device)
trainer = FactorVAETrainer(vae_model, discriminator, config, device)
elif args.model == 'classifiervae':
model = ClassifierVAE(config, device).to(device)
classifier = Classifier(config).to(device)
trainer = ClassifierVAETrainer(model, classifier, config, device)
def migrate_repo(path):
"""Perform migration to repo on given path."""
click.secho(f"\n>>> Migrating {path}...", fg="green")
repo = path.split("/")[-1]
repo_underscores = repo.replace("-", "_")
# TODO: add the trailing slash only if needed
path = path + "/"
# Reference: https://codimd.web.cern.ch/TOOkF5yhSAKJq3TiY0L42A?view
travis = read_yaml(path + ".travis.yml")
try:
if travis["deploy"]["provider"] == "pypi":
# Download pypi-publish.yml template
download_file(
GA_PYPI_PUBLISH_YAML_URL,
path + ".github/workflows/pypi-publish.yml",
)
except Exception as e:
logging.info(f"Couldn't find deploy key in .travis.yml")
# .editorconfig
replace_simple(path + ".travis.yml", ".github/workflows/*.yml",
".editorconfig")
# README.rst
replace_regex(
r"https:\/\/img\.shields\.io\/travis\/([a-z]*\/[a-z-]*)\.svg",
"https://github.com/\\1/workflows/CI/badge.svg",
path + "README.rst",
)
replace_regex(
r"https:\/\/travis-ci\.org\/([a-z]*\/[a-z-]*)",
"https://github.com/\\1/actions?query=workflow%3ACI",
path + "README.rst",
)
# CONTRIBUTING.rst
replace_regex(
r"https:\/\/travis-ci\.(org|com)\/([a-z]*\/[a-z-]*)\/pull_requests",
"https://github.com/\\2/actions?query=event%3Apull_request",
path + "CONTRIBUTING.rst",
)
# tests.yaml
build_template(repo, "tests.yml", path=f"{path}/.github/workflows")
# run-tests.sh
build_template(repo, "run-tests.sh", path=path)
# pytest.ini
delete_line("pep8ignore", path + "pytest.ini")
replace_regex(
"(addopts =).*",
f'\\1 --isort --pydocstyle --pycodestyle --doctest-glob="*.rst" --doctest-modules --cov={repo_underscores} --cov-report=term-missing',
path + "pytest.ini",
)
if not file_contains("testpaths", path + "pytest.ini"):
append_to_file(f"testpaths = tests {repo_underscores}",
path + "pytest.ini")
# Add .github/workflows *.yml to MANIFEST.in
add_line("recursive-include .github/workflows *.yml\n",
path + "MANIFEST.in")
# Delete travis file
delete_file(path + ".travis.yml")
# Upgrade Sphinx 1 to 3 in setup.py
replace_regex(
r"Sphinx>=1.[0-9].[0-9]",
"Sphinx>=3",
path + "setup.py",
)
# Simplify setup.py test requirements replacing them with pytest-invenio
replace_list(
path + "setup.py",
r"tests_require = (['\"\'[\s*\"(a-z-A-Z><=0-9.\[\]),]*])",
[
# Remove packages already installed by pytest-invenio
"check-manifest",
"coverage",
"docker-services-cli",
"pytest-celery",
"pytest-cov",
"pytest-flask",
"pytest-isort",
"pytest-pycodestyle",
"pytest-pydocstyle",
"pydocstyle",
"pytest",
"selenium",
# pytest-pep8 is replaced by pytest-pycodestyle
"pytest-pep8",
# pytest-pep8 is replaced by pytest-isort
"isort",
],
["pytest-invenio>=1.4.0"],
"tests_require",
)
# Remove bak files
delete_file(path + "*.bak")
def read_yaml(self, file_subpath):
return read_yaml(self.to_full_path(file_subpath))
def reconstruct(args):
# get context
ctx = get_extension_context(args.context)
nn.set_default_context(ctx)
logger.setLevel(logging.ERROR) # to supress minor messages
config = read_yaml(args.config)
dataset_params = config.dataset_params
model_params = config.model_params
if args.detailed:
vis_params = config.visualizer_params
visualizer = Visualizer(**vis_params)
if not args.params:
assert "log_dir" in config, "no log_dir found in config. therefore failed to locate pretrained parameters."
param_file = os.path.join(
config.log_dir, config.saved_parameters)
else:
param_file = args.params
nn.load_parameters(param_file)
bs, h, w, c = [1] + dataset_params.frame_shape
source = nn.Variable((bs, c, h, w))
driving_initial = nn.Variable((bs, c, h, w))
driving = nn.Variable((bs, c, h, w))
with nn.parameter_scope("kp_detector"):
kp_source = detect_keypoint(source,
**model_params.kp_detector_params,
**model_params.common_params,
test=True, comm=False)
persistent_all(kp_source)
with nn.parameter_scope("kp_detector"):
kp_driving = detect_keypoint(driving,
**model_params.kp_detector_params,
**model_params.common_params,
test=True, comm=False)
persistent_all(kp_driving)
with nn.parameter_scope("generator"):
generated = occlusion_aware_generator(source,
kp_source=unlink_all(kp_source),
kp_driving=kp_driving,
**model_params.generator_params,
**model_params.common_params,
test=True, comm=False)
if not args.full and 'sparse_deformed' in generated:
del generated['sparse_deformed'] # remove needless info
persistent_all(generated)
generated['kp_driving'] = kp_driving
generated['kp_source'] = kp_source
# generated contains these values;
# 'mask': <Variable((bs, num_kp+1, h/4, w/4)) when scale_factor=0.25
# 'sparse_deformed': <Variable((bs, num_kp+1, num_channel, h/4, w/4)) # (bs, num_kp + 1, c, h, w)
# 'occlusion_map': <Variable((bs, 1, h/4, w/4))
# 'deformed': <Variable((bs, c, h, w))
# 'prediction': <Variable((bs, c, h, w))
mode = "reconstruction"
if "log_dir" in config:
result_dir = os.path.join(args.out_dir, os.path.basename(config.log_dir), f"{mode}")
else:
result_dir = os.path.join(args.out_dir, "test_result", f"{mode}")
# create an empty directory to save generated results
_ = nm.Monitor(result_dir)
if args.eval:
os.makedirs(os.path.join(result_dir, "png"), exist_ok=True)
# load the header images.
header = imread("imgs/header_combined.png", channel_first=True)
filenames = sorted(glob.glob(os.path.join(
dataset_params.root_dir, "test", "*")))
recon_loss_list = list()
for filename in tqdm(filenames):
# process repeated until all the test data is used
driving_video = read_video(
filename, dataset_params.frame_shape) # (#frames, h, w, 3)
driving_video = np.transpose(
driving_video, (0, 3, 1, 2)) # (#frames, 3, h, w)
generated_images = list()
source_img = driving_video[0]
source.d = np.expand_dims(source_img, 0)
driving_initial.d = driving_video[0]
# compute these in advance and reuse
nn.forward_all(
[kp_source["value"], kp_source["jacobian"]], clear_buffer=True)
num_of_driving_frames = driving_video.shape[0]
for frame_idx in tqdm(range(num_of_driving_frames)):
driving.d = driving_video[frame_idx]
nn.forward_all([generated["prediction"],
generated["deformed"]], clear_buffer=True)
if args.detailed:
# visualize source w/kp, driving w/kp, deformed source, generated w/kp, generated image, occlusion map
visualization = visualizer.visualize(
source=source.d, driving=driving.d, out=generated)
if args.full:
visualization = reshape_result(visualization) # (H, W, C)
combined_image = visualization.transpose(2, 0, 1) # (C, H, W)
elif args.only_generated:
combined_image = np.clip(
generated["prediction"].d[0], 0.0, 1.0)
combined_image = (
255*combined_image).astype(np.uint8) # (C, H, W)
else:
# visualize source, driving, and generated image
driving_fake = np.concatenate([np.clip(driving.d[0], 0.0, 1.0),
np.clip(generated["prediction"].d[0], 0.0, 1.0)], axis=2)
header_source = np.concatenate([np.clip(header / 255., 0.0, 1.0),
np.clip(source.d[0], 0.0, 1.0)], axis=2)
combined_image = np.concatenate(
[header_source, driving_fake], axis=1)
combined_image = (255*combined_image).astype(np.uint8)
generated_images.append(combined_image)
# compute L1 distance per frame.
recon_loss_list.append(
np.mean(np.abs(generated["prediction"].d[0] - driving.d[0])))
# post process only for reconstruction evaluation.
if args.eval:
# crop generated images region only.
if args.only_generated:
eval_images = generated_images
elif args.full:
eval_images = [_[:, :h, 4*w:5*w] for _ in generated_images]
elif args.detailed:
assert generated_images[0].shape == (c, h, 5*w)
eval_images = [_[:, :, 3*w:4*w] for _ in generated_images]
else:
eval_images = [_[:, h:, w:] for _ in generated_images]
# place them horizontally and save for evaluation.
image_for_eval = np.concatenate(
eval_images, axis=2).transpose(1, 2, 0)
imsave(os.path.join(result_dir, "png", f"{os.path.basename(filename)}.png"),
image_for_eval)
# once each video is generated, save it.
output_filename = f"{os.path.splitext(os.path.basename(filename))[0]}.mp4"
if args.output_png:
monitor_vis = nm.MonitorImage(output_filename, nm.Monitor(result_dir),
interval=1, num_images=1,
normalize_method=lambda x: x)
for frame_idx, img in enumerate(generated_images):
monitor_vis.add(frame_idx, img)
else:
generated_images = [_.transpose(1, 2, 0) for _ in generated_images]
# you might need to change ffmpeg_params according to your environment.
mimsave(f'{os.path.join(result_dir, output_filename)}', generated_images,
fps=args.fps,
ffmpeg_params=["-pix_fmt", "yuv420p",
"-vcodec", "libx264",
"-f", "mp4",
"-q", "0"])
print(f"Reconstruction loss: {np.mean(recon_loss_list)}")
return
import sys
from create_api import *
from utils import read_yaml
from configs import conf
if __name__ == '__main__':
print("arguments :", sys.argv)
api = read_yaml(conf('docs_main_path'), 'apis.yaml')[sys.argv[1]]
api_executor(api)
# make log dir
logdir = os.path.join(outputdir, "log")
if not os.path.exists(logdir):
os.makedirs(logdir)
def prepare_adjoint(config):
# mkdir outputbase
outputbase = config["outputdir"]
clean_outputdir(outputbase)
eventlist_file = config["eventlist"]
eventlist = read_txt(eventlist_file)
# split the eventlist and dump into separate files
nevents_per_job = config["nevents_per_job"]
cmtlist_per_job = split_job(eventlist, nevents_per_job)
print("-"*10 + "\nJob list:\n%s" % cmtlist_per_job)
print("="*20 + "\nPreparing jobs...")
for job_id, cmtlist in cmtlist_per_job.iteritems():
prepare_one_job(job_id, cmtlist, config)
if __name__ == "__main__":
config = read_yaml("config.yaml")
prepare_adjoint(config)
def run(exp_name):
nilmtk_models = ["CO", "AFHMM"]
nilmtk_appliances = [
"fridge", "microwave", "washing machine", "television",
"electric heating element", "cooker"
]
experiment = utils.read_yaml(f"experiment_configs/{exp_name}.yaml")
pass
devices = experiment["devices"]
interval = experiment["interval"]
use_weather = experiment["use_weather"]
use_occupancy = experiment["use_occupancy"]
times = {"train_times": {}, "test_times": {}}
for model in experiment["models"].values():
network_type = model["network_type"]
window_length = model["window_length"]
output_length = model["output_length"]
epochs = model["epochs"]
learning_rate = model["learning_rate"]
batch_size = model["batch_size"]
early_stopping = model["early_stopping"]
patience = model["patience"]
restore_weights = model["restore_weights"]
times["train_times"][network_type] = {}
times["test_times"][network_type] = {}
if network_type in nilmtk_models:
if use_weather or use_occupancy:
continue
else:
train_time_agg = 0
test_time_agg = 0
eval_counter = 0
for test_set in experiment["evaluate"]:
train_time, test_time = nilmtk_run.run_model(
model_type=network_type,
appliances=nilmtk_appliances,
interval=interval,
test_dataset=test_set,
experiment_name=exp_name,
return_time=True,
export_predictions=True,
verbose=False)
train_time_agg += train_time
test_time_agg += test_time
eval_counter += 1
times["train_times"][
network_type] = train_time_agg / eval_counter
times["test_times"][
network_type] = test_time_agg / eval_counter
else:
for device in devices:
training_directory = f"../data/appliances/{device}/{device}_train_{interval}_.csv"
validation_directory = f"../data/appliances/{device}/{device}_val_{interval}_.csv"
save_model_dir = utils.get_model_save_path(
network_type=network_type,
interval=interval,
device=device,
window_length=window_length,
use_weather=use_weather,
use_occupancy=use_occupancy)
trainer = Trainer(appliance=device,
batch_size=batch_size,
crop=None,
network_type=network_type,
use_weather=use_weather,
use_occupancy=use_occupancy,
learning_rate=learning_rate,
epochs=epochs,
early_stopping=early_stopping,
patience=patience,
restore_weights=restore_weights,
training_directory=training_directory,
validation_directory=validation_directory,
save_model_dir=save_model_dir,
input_window_length=window_length,
output_length=output_length,
validation_frequency=1,
return_time=True,
plot_training=False)
train_time = trainer.train_model()
times["train_times"][network_type][device] = train_time
# Average test_times across all datasets
test_time_agg = 0
eval_counter = 0
for test_set in experiment["evaluate"]:
test_directory = f"../data/appliances/{device}/{device}_{test_set}_{interval}_.csv"
if os.path.isfile(test_directory):
# The logs including results will be recorded to this log file
utils.check_dir(f"outputs/logs/{exp_name}/")
log_file_dir = f"outputs/logs/{exp_name}/{network_type}_{device}.log"
tester = Tester(appliance=device,
algorithm=network_type,
crop=None,
batch_size=batch_size,
network_type=network_type,
test_directory=test_directory,
saved_model_dir=save_model_dir,
log_file_dir=log_file_dir,
input_window_length=window_length,
output_length=output_length,
use_weather=use_weather,
use_occupancy=use_occupancy,
plot_first=-1,
dataset=test_set,
return_time=True,
return_predictions=True)
pred, test_time = tester.test_model()
test_time_agg += test_time
eval_counter += 1
utils.check_dir(
f"outputs/model_predictions/{exp_name}/")
results_path = f"outputs/model_predictions/{exp_name}/{network_type}_{device}_{test_set}.csv"
pd.DataFrame(pred).to_csv(results_path, sep=";")
times["test_times"][network_type][
device] = test_time_agg / eval_counter
utils.save_dict(times,
f"outputs/experiment_logs/{exp_name}_training_times")
def main():
conf = utils.read_yaml(utils.get_arguments()["unknown"][0])
os.remove(utils.get_arguments()["unknown"][0])
# Clone repositories. Using os.getcwd() because the drive letter needs to
# be respected on Windows.
repositories_path = os.path.abspath(
os.path.join(os.getcwd(), "repositories", conf["name"]))
os.environ["CONDA_ENVIRONMENT_REPOSITORIES"] = repositories_path
# Kept for backwards compatibility
os.environ["CONDA_GIT_REPOSITORY"] = repositories_path
repositories = []
for item in conf["dependencies"]:
if "git" in item:
for repo in item["git"]:
repo_path = ""
if isinstance(repo, str):
repo_path = repo
if isinstance(repo, dict):
repo_path = repo.keys()[0]
data = {"url": repo_path}
name = repo_path.split("/")[-1].replace(".git", "")
if not name:
name = repo_path.split("/")[-2]
if "@" in name:
name = name.split("@")[0]
repo_path = repo_path.split("@")[0]
data["name"] = name
if not os.path.exists(repositories_path):
os.makedirs(repositories_path)
if name not in os.listdir(repositories_path):
subprocess.call(["git", "clone", repo_path],
cwd=repositories_path)
data["path"] = os.path.join(repositories_path, name)
data["commands"] = {
"on_launch": [],
"on_environment_update": []
}
if isinstance(repo, dict):
for item in repo[repo.keys()[0]]:
if isinstance(item, dict):
for event, commands in item.iteritems():
data["commands"][event].extend(commands)
else:
data["commands"]["on_launch"].append(item)
repositories.append(data)
# Update repositories.
if utils.get_arguments()["update-repositories"]:
for repo in repositories:
print repo["name"]
# Updating origin url
subprocess.call([
"git", "remote", "set-url", "origin", repo["url"].split("@")[0]
],
cwd=repo["path"])
# Update git repository
subprocess.call(["git", "checkout", "master"], cwd=repo["path"])
subprocess.call(["git", "pull"], cwd=repo["path"])
subprocess.call(
["git", "submodule", "update", "--init", "--recursive"],
cwd=repo["path"])
subprocess.call(["git", "submodule", "update", "--recursive"],
cwd=repo["path"])
# Checkout any commits/tags if there are newly cloned repositories or
# updating the repositories.
if utils.get_arguments()["update-repositories"]:
for repo in repositories:
if "@" in repo["url"]:
tag = repo["url"].split("@")[1]
if tag:
print repo["name"]
subprocess.call(["git", "checkout", tag], cwd=repo["path"])
# Checkout environment repository
environment_path = utils.get_environment()
if not os.path.exists(environment_path):
# Determine environment repositories by matching passed environment
# with repositories
environment_repo = None
match = 0.0
for repo in repositories:
sequence_match = SequenceMatcher(None, repo["url"],
environment_path).ratio()
if match < sequence_match:
environment_repo = repo
print environment_repo["name"]
branch = environment_path.split("/")[-2]
subprocess.call(["git", "checkout", branch],
cwd=environment_repo["path"])
# Install any setup.py if we are updating
if (utils.get_arguments()["update-repositories"]
or utils.get_arguments()["update-environment"]):
for repo in repositories:
if "setup.py" not in os.listdir(repo["path"]):
continue
args = ["python", "setup.py", "develop"]
subprocess.call(args, cwd=repo["path"])
# Add environment site packages to os.environ
prefix = ""
if platform.system().lower() == "windows":
prefix = os.environ["CONDA_PREFIX"]
else:
prefix = os.environ["CONDA_ENV_PATH"]
path = os.path.join(prefix, "lib", "site-packages")
os.environ["PYTHONPATH"] += os.pathsep + path
# Add sys.path to os.environ["PYTHONPATH"], because conda only modifies
# sys.path which gets lost when launching any detached subprocesses.
# This get a little complicated due to being in a process that hasn"t
# picked up on the changes, hence going through a subprocess.
python_file = os.path.join(os.path.dirname(__file__), "write_sys_path.py")
data_file = os.path.join(tempfile.gettempdir(),
"data_%s.yml" % os.getpid())
subprocess.call(["python", python_file, data_file])
paths = []
with open(data_file, "r") as f:
paths += utils.read_yaml(f.read())
os.remove(data_file)
for path in paths:
if path.lower().startswith(repositories_path.lower()):
os.environ["PYTHONPATH"] += os.pathsep + path
if path.endswith(".egg"):
os.environ["PYTHONPATH"] += os.pathsep + path
# Clean up any existing environment file
if os.path.exists(utils.get_environment_path()):
os.remove(utils.get_environment_path())
# Ensure subprocess is detached so closing connect will not also
# close launched applications.
options = {}
if not utils.get_arguments()["attached"]:
if sys.platform == "win32":
options["creationflags"] = subprocess.CREATE_NEW_CONSOLE
else:
options["preexec_fn"] = os.setsid
# Setting update mode environment variable
update_modes = []
if utils.get_arguments()["update-environment"]:
update_modes.append("environment")
if utils.get_arguments()["update-repositories"]:
update_modes.append("repositories")
os.environ["CONDA_GIT_UPDATE"] = ""
for mode in update_modes:
os.environ["CONDA_GIT_UPDATE"] += mode + os.pathsep
# Execute environment update commands.
if utils.get_arguments()["update-environment"]:
for repo in repositories:
if "commands" in repo.keys():
for cmd in repo["commands"]["on_environment_update"]:
os.environ.update(utils.read_environment())
cmd = cmd.replace("$REPO_PATH", repo["path"])
print "Executing: " + cmd
subprocess.call(cmd,
shell=True,
cwd=repo["path"],
**options)
# Execute launch commands.
for repo in repositories:
if "commands" in repo.keys():
for cmd in repo["commands"]["on_launch"]:
os.environ.update(utils.read_environment())
cmd = cmd.replace("$REPO_PATH", repo["path"])
print "Executing: " + cmd
subprocess.call(cmd, shell=True, cwd=repo["path"], **options)
opt = parser.parse_args()
print('the parameter of your training ----->')
print(opt)
########################################################################################################################
os.environ["CUDA_VISIBLE_DEVICES"] = str(opt.GPU)
model_path = opt.pth_path + '//' + opt.denoise_model
# print(model_path)
model_list = list(os.walk(model_path, topdown=False))[-1][-1]
# print(model_list)
for i in range(len(model_list)):
aaa = model_list[i]
if '.yaml' in aaa:
yaml_name = model_list[i]
print(yaml_name)
read_yaml(opt, model_path + '//' + yaml_name)
# print(opt.datasets_folder)
name_list, noise_img, coordinate_list = test_preprocess_lessMemoryNoTail(opt)
# name_list, noise_img, coordinate_list = test_preprocess_lessMemory(opt)
# trainX = np.expand_dims(np.array(train_raw),4)
num_h = (math.floor((noise_img.shape[1] - opt.img_h) / opt.gap_h) + 1)
num_w = (math.floor((noise_img.shape[2] - opt.img_w) / opt.gap_w) + 1)
num_s = (math.floor((noise_img.shape[0] - opt.img_s) / opt.gap_s) + 1)
# print(num_h, num_w, num_s)
# print(coordinate_list)
if not os.path.exists(opt.output_dir):
os.mkdir(opt.output_dir)
current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M")
output_path1 = opt.output_dir + '//' + opt.datasets_folder + '_' + current_time + '_' + opt.denoise_model
import hashlib
import struct
import time
from math import ceil
from connection import Connection, MAX_DATA_SIZE
from utils import read_yaml, log
if __name__ == '__main__':
# Read configuration
config = read_yaml("../config.yaml")
my_ip, my_port = config["sender_ip"], config["port"]["acknowledgement"][
"target"]
remote_ip, remote_port = config["receiver_ip"], config["port"]["data"][
"source"]
file_name = config["file_name"]["sender"]
chunk_size = MAX_DATA_SIZE - 4
timeout = config["timeout"]
# Read the whole file as bytes
with open(file_name, "rb") as file:
file_bytes = file.read(-1)
# We must send the number of chunks to the server,
# so that they know how many messages to receive
chunks_count = ceil(len(file_bytes) / chunk_size)
md5 = hashlib.md5(file_bytes).digest()
with Connection(my_ip, my_port, remote_ip, remote_port,
scores_window.append(score) # save most recent score
scores.append(score) # save most recent score
avg_av = agent.evaluate_on_fixed_set(fixed_states)
average_action_values.append(avg_av)
print(f'Episode {i_episode}\tAverage Score: '
f'{round(np.mean(scores_window),4)}\tEpsilon: {round(eps, 4)}\t'
f'Average Q value: {round(avg_av, 4)}')
if i_episode % conf['save_every'] == 0 and i_episode > 0:
print(f'Saving model at iteration: {i_episode}')
save_model(conf, agent)
env.close()
return {
'scores': scores,
'epsilons': epsilons,
'avg_action_values': average_action_values
}
if __name__ == '__main__':
arguments = parse_args()
pc = arguments.path_config
exp_conf = read_yaml(pc)
stats = train(exp_conf)
save_scores(exp_conf, stats)
def test_read_yaml(self):
self.assertTrue(read_yaml(self.yamlfile))
def test_path2rootdir(self):
path_dicts = make_paths(read_yaml(self.yamlfile))
for path in path_dicts:
rfile, rdir = path2rootdir(path['path'])
self.assertTrue(not rfile.IsZombie())
self.assertTrue(rdir)
def train(fp_config):
"""
Performs a DINO training experiment based on a provided configuration
"""
# ================ CONFIGURATION ================
# Load the configuration file
config = utils.read_yaml(fp_config)
# Announce the configuration
print(f"Using the configuration at: {fp_config}")
for k in config.keys():
print(f" - {k}: {config[k]}")
# ================ DEVICE ================
device = utils.report_and_get_device_gpu_preferred()
# ================ DATA ================
print("Loading training data ... ", end="")
train_loader, test_loader = dw.get_mars32k_train_test_dataloaders(
config['fp_data_mars32k'], config['data_proportion'],
config['train_test_ratio'], config['batch_size'],
config['perform_shuffle'])
print("ready")
# ================ STUDENT AND TEACHER NETWORKS ================
# Create the student network
print("Building student network ... ", end="")
nn_student = utils.build_vision_transformer_from_config(config)
print("ready")
# Create the teacher network
print("Building teacher network ... ", end="")
nn_teacher = utils.build_vision_transformer_from_config(config)
# Copy the student weights over to the teacher (initially they are the same)
nn_teacher.load_state_dict(nn_student.state_dict())
# There is no backpropagation through the teacher
for param in nn_teacher.parameters():
param.requires_grad = False
# Move them both to the desired device
nn_student.to(device)
nn_teacher.to(device)
print("ready")
# ================ MULTICROPPER ================
# Build the multicropper module
print("Instantiating multicropper ... ", end="")
multicropper = MultiCropper(global_size=config['global_size'],
local_size=config['local_size'],
n_global_crops=config['n_global_crops'],
n_local_crops=config['n_local_crops'])
print("ready")
# ================ SCHEDULERS ================
# Create learning rate scheduler and weight decay scheduler for the optimiser
sch_lr = utils.LinearPiecewiseScheduler(config['lr_values'],
config['lr_epochs'])
sch_wd = utils.LinearPiecewiseScheduler(config['weight_decay_values'],
config['weight_decay_epochs'])
# Create temperature schedulers
sch_temp_student = utils.LinearPiecewiseScheduler(
config['temp_student_values'], config['temp_student_epochs'])
# Too high temperature at the start may cause the teacher's training
# to be unstable. Investigate the use of warm up as necessary
sch_temp_teacher = utils.LinearPiecewiseScheduler(
config['temp_teacher_values'], config['temp_teacher_epochs'])
# The centering rate parameter (usually in range [0.9, 0.999])
sch_cent_rate_m = utils.LinearPiecewiseScheduler(
config['cent_rate_m_values'], config['cent_rate_m_epochs'])
# The lambda weight transfer parameter (the amount of teacher network v. student
# network to include in the next iteration of the teacher network)
# TODO a cosine scheduler should be used if possible
sch_lambda_ema = utils.LinearPiecewiseScheduler(
config['lambda_ema_values'], config['lambda_ema_epochs'])
# ================ OPTIMISER ================
# Preparing the optimiser
print("Preparing optimiser ... ", end="")
# Note that it is recommended to only construct this after the
# models are on the GPU (if using a GPU)
optimiser = torch.optim.AdamW(nn_student.parameters(),
lr=sch_lr.get_value(0),
weight_decay=sch_wd.get_value(0))
print("ready")
# ================ DINO LOSS ================
# Prepare the loss module
print("Preparing DINO loss module ... ", end="")
loss_computer = LossComputerDINO(output_dim=config['embed_dim'])
loss_computer.to(device)
print("ready")
# ================ TRAINING ================
print("Preparing training procedure ... ", end="")
fp_save = config['fp_save']
# Create a scalar for mixed precision as
# desired (no op if flag is false)
scaler = torch.cuda.amp.GradScaler(enabled=config['mixed_precision'])
# Synchronise
gc.collect()
torch.cuda.empty_cache()
torch.cuda.reset_max_memory_allocated()
torch.cuda.synchronize()
print("ready")
time.sleep(0.5)
print("I'm your huckleberry!")
# Perform as many epochs of training as required
metrics = []
n_epochs = config['n_epochs']
for epoch_idx in range(n_epochs):
# Get the schedule values
learning_rate = sch_lr.get_value(epoch_idx)
weight_decay = sch_wd.get_value(epoch_idx)
temp_student = sch_temp_student.get_value(epoch_idx)
temp_teacher = sch_temp_teacher.get_value(epoch_idx)
cent_rate_m = sch_cent_rate_m.get_value(epoch_idx)
lambda_ema = sch_lambda_ema.get_value(epoch_idx)
# And train a single epoch
epoch_metrics = train_single_epoch(
epoch_idx=epoch_idx,
n_epochs=n_epochs,
device=device,
scaler=scaler,
mixed_precision=config['mixed_precision'],
optimiser=optimiser,
nn_student=nn_student,
nn_teacher=nn_teacher,
loss_computer=loss_computer,
train_loader=train_loader,
multicropper=multicropper,
learning_rate=learning_rate,
weight_decay=weight_decay,
temp_student=temp_student,
temp_teacher=temp_teacher,
cent_rate_m=cent_rate_m,
lambda_ema=lambda_ema)
metrics.append(epoch_metrics)
# ================ SAVING ================
print("Saving experiment ... ", end="")
utils.save_dino_experiment(fp_save=fp_save,
nn_student=nn_student,
nn_teacher=nn_teacher,
optimiser=optimiser,
loss_computer=loss_computer,
config=config,
metrics=metrics)
print("complete")
def test_make_paths(self):
path_dicts = make_paths(read_yaml(self.yamlfile))
# test 1 generic metadata field
self.assertTrue(path_dicts[0]['title'])
for path in path_dicts:
self.assertTrue(path['path'])
def get_config():
"""
Get command line arguments.
Arguments set the default values of command line arguments.
"""
parser = argparse.ArgumentParser(description='TecoGAN')
parser.add_argument('--cfg', default="./config.yaml")
args, _ = parser.parse_known_args()
conf = read_yaml(args.cfg)
parser.add_argument(
'--input_video_dir',
default=conf.data.input_video_dir,
help='The directory of the video input data, for training')
parser.add_argument('--output_dir',
default=conf.data.output_dir,
help='The output directory of the checkpoint')
parser.add_argument(
'--num_resblock',
type=int,
default=conf.train.num_resblock,
help='How many residual blocks are there in the generator')
parser.add_argument('--max_iter',
type=int,
default=conf.train.max_iter,
help='max iteration for training')
parser.add_argument(
'--pre_trained_frvsr_weights',
type=str,
default=conf.train.pre_trained_frvsr_weights,
help='the weight of frvsr generator will be loaded as an initial point'
)
parser.add_argument('--vgg_pre_trained_weights',
type=str,
default=conf.train.vgg_pre_trained_weights,
help='path to pre-trained weights for the vgg19')
parser.add_argument(
'--tecogan',
type=bool,
default=conf.train.tecogan,
help='True for Tecogan training False for FRVSR training')
parser.add_argument('--checkpoint',
type=str,
default=conf.train.checkpoint,
help='path to checkpoint file')
args = parser.parse_args()
# refine config
conf.data.input_video_dir = args.input_video_dir
conf.train.max_iter = conf.train.max_iter
conf.data.output_dir = args.output_dir
conf.train.num_resblock = args.num_resblock
conf.train.pre_trained_frvsr_weights = args.pre_trained_frvsr_weights
conf.train.vgg_pre_trained_weights = args.vgg_pre_trained_weights
conf.train.tecogan = args.tecogan
conf.train.checkpoint = args.checkpoint
return conf
def __init__(self, **cli_args):
logger.info(f"Entered CLI args: {cli_args}")
logger.info(f"Executing command: {cli_args.get('cmd')} ...")
self.data_path: str = str(
cli_args.get("data_path")) # path to the dataset
logger.info(f"reading data from {self.data_path}")
self.command = cli_args.get("cmd", None)
if not self.command or self.command not in self.available_commands:
raise Exception(f"You must enter a valid command.\n"
f"available commands: {self.available_commands}")
if self.command == "fit":
self.yml_path = str(cli_args.get("yaml_path"))
file_ext = self.yml_path.split(".")[-1]
logger.info(f"You passed the configurations as a {file_ext} file.")
self.yaml_configs = (read_yaml(self.yml_path) if file_ext == "yaml"
else read_json(self.yml_path))
logger.info(f"your chosen configuration: {self.yaml_configs}")
# dataset options given by the user
self.dataset_props: dict = self.yaml_configs.get(
"dataset", self.default_dataset_props)
# model options given by the user
self.model_props: dict = self.yaml_configs.get(
"model", self.default_model_props)
# list of target(s) to predict
self.target: list = self.yaml_configs.get("target")
self.model_type: str = self.model_props.get("type")
logger.info(f"dataset_props: {self.dataset_props} \n"
f"model_props: {self.model_props} \n "
f"target: {self.target} \n")
# handle random numbers generation
random_num_options = self.dataset_props.get("random_numbers", None)
if random_num_options:
generate_reproducible = random_num_options.get(
"generate_reproducible", None)
if generate_reproducible:
logger.info(
"You provided the generate reproducible results option."
)
seed = random_num_options.get("seed", 42)
np.random.seed(seed)
logger.info(
f"Setting a seed = {seed} to generate same random numbers on each experiment.."
)
# if entered command is evaluate or predict, then the pre-fitted model needs to be loaded and used
else:
self.model_path = cli_args.get("model_path",
self.default_model_path)
logger.info(f"path of the pre-fitted model => {self.model_path}")
self.prediction_file = cli_args.get("prediction_file",
self.prediction_file)
# set description.json if provided:
self.description_file = cli_args.get("description_file",
self.description_file)
# load description file to read stored training parameters
with open(self.description_file) as f:
dic = json.load(f)
self.target: list = dic.get(
"target") # target to predict as a list
self.model_type: str = dic.get(
"type"
) # type of the model -> regression, classification or clustering
self.dataset_props: dict = dic.get(
"dataset_props") # dataset props entered while fitting
getattr(self, self.command)()
unidades_economicas_pequenas_distancia_km * {costo_eco_pequenas} +
unidades_economicas_medianas_distancia_km * {costo_eco_medianas} +
carretera_distancia_km * {costo_transporte} AS costo_construccion,
CASE WHEN zona_urbana_distancia_min = 0
THEN 0 ELSE 1 END AS dummy_urbano
FROM features.features_{grid_size}
) SELECT cell_id,
year,
0 as CTN,
costo_construccion * dummy_urbano AS CFP,
costo_construccion * dummy_urbano * {antes} AS CTP,
costo_construccion * dummy_urbano * {despues} AS CFN
FROM costos_construccion)""".format(grid_size=grid_size,
costo_calles=costo_calles,
costo_eco_pequenas=costo_eco_pequenas,
costo_eco_medianas=costo_eco_medianas,
costo_transporte=costo_transporte,
antes=antes,
despues=despues))
db_conn = utils.get_connection().raw_connection()
cur = db_conn.cursor()
cur.execute(DROP)
db_conn.commit()
cur.execute(QUERY)
db_conn.commit()
if __name__ == '__main__':
experiment_path = '../experiment.yaml'
experiment = utils.read_yaml(experiment_path)
gen_cost_matrix(experiment['costos'], experiment['grid_size'])
import os
from attrdict import AttrDict
from deepsense import neptune
from utils import read_yaml
ctx = neptune.Context()
params = ctx.params
if params.__class__.__name__ == 'OfflineContextParams':
neptune_config = read_yaml('neptune_config.yaml')
params = neptune_config.parameters
X_COLUMNS = ['comment_text']
Y_COLUMNS = [
'toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate'
]
SOLUTION_CONFIG = AttrDict({
'env': {
'cache_dirpath': params.experiment_dir
},
'xy_splitter': {
'x_columns': X_COLUMNS,
'y_columns': Y_COLUMNS
},
'text_cleaner': {
'drop_punctuation': bool(params.drop_punctuation),
'drop_newline': bool(params.drop_newline),
'drop_multispaces': bool(params.drop_multispaces),
def create(ctx, number, existing, proposed, load, path=default_path, **kwargs):
""" Main function that creates all the inputs 🔥"""
# if ctx.obj['DEBUG']:
# def debug(type, value, tb):
# import traceback, pdb
# traceback.print_exception(type, value, tb)
# pdb.pm()
# sys.excepthook = debug
click.echo('Starting app')
if existing and proposed:
click.echo('Creating generation project info')
gen_project_legacy = pd.read_csv(os.path.join(
default_path, 'generation_projects_info.tab'),
sep='\t')
gen_project_proposed = init_scenario()
gen_project = pd.concat([gen_project_legacy, gen_project_proposed])
gen_legacy = gen_build_predetermined(existing)
create_gen_build_cost(gen_project, gen_legacy)
else:
click.echo('Oops I do not know what to do yet')
sys.exit(1)
# Finally
gen_project.to_csv(os.path.join(output_path,
'generation_projects_info.tab'),
sep='\t',
index=False)
click.echo(f'Number of timepoints selected: {number}')
click.echo(f'Reading load data')
load_data = get_load_data(filename=load)
click.echo(f'Reading periods data')
periods = read_yaml(path, 'periods.yaml')
d = OrderedDict(periods)
periods_tab = pd.DataFrame(d)
periods_tab = periods_tab.set_index('INVESTMENT_PERIOD')
click.echo(f'Creating timeseries')
timeseries, timeseries_dict = [], {}
for periods, row in periods_tab.iterrows():
timeseries_dict[periods] = []
scale_to_period = row[1] - row[0]
peak_data = get_peak_day(load_data[str(periods)]['total'],
number,
freq='1MS',
**kwargs)
median_data = get_median_day(load_data[str(periods)]['total'],
number,
freq='1MS',
**kwargs)
timeseries_dict[periods].append(
create_strings(peak_data, scale_to_period))
timeseries_dict[periods].append(
create_strings(median_data, scale_to_period, identifier='M'))
timeseries.append(create_strings(peak_data, scale_to_period))
timeseries.append(
create_strings(median_data, scale_to_period, identifier='M'))
click.echo(f'Creating investment period')
create_investment_period()
# create_gen_build_cost_new(peak_data)
create_timeseries(timeseries, number, **kwargs)
create_timepoints(timeseries)
click.echo(f'Creating variable capacity factor')
create_variablecp(gen_project, timeseries, timeseries_dict)
click.echo(f'Creating loads')
create_loads(load_data, timeseries)
rps_file, ext = look_for_file('rps_targets', default_path)
click.echo(f'Creating fuel loads')
create_fuel_cost()
if rps_file:
click.echo(f'Creating rps')
create_rps(filename=rps_file, ext=ext)
click.echo(f'App ended')
def generate_bodies():
assert args.seed_bodies<100, "The way we combining real seeds only allow seed_bodies to be smaller than 100."
assert args.body_variation_range%10==0, "The way we combining real seeds only allow body_variation_range to be multiplication of 10, e.g. 10, 20, ... 90."
assert args.num_bodies%10==0, "The way we combining real seeds only allow num_bodies to be multiplication of 10, e.g. 10, 20, 30 ..."
real_seed = args.num_bodies*1000 + args.body_variation_range * 100 + args.seed_bodies # if I only apply seed_bodies, the first 20 of walker2d_30_10-v0 will be the same of walker2d_20_10-v0.
random.seed(real_seed)
output(f"Start generating bodies {args.num_bodies} with seed {real_seed}", 1)
# 1. Check templates
template_files = check_templates()
# 2. Create Folders
dataset_path = create_folder()
# 3. Generate variations and write body, param files
body_xml = read_template(template_files[0])
body_yaml = read_yaml(template_files[1])
file_list, param_list = [], []
for i in range(args.num_bodies):
data = {}
for key in body_yaml['variable']:
data[key] = body_yaml['variable'][key] * ((random.random() * 2 - 1) * args.body_variation_range / 100 + 1.0)
data[key] = significant_digits(data[key], 4)
for key in body_yaml['fixed']:
data[key] = body_yaml['fixed'][key]
for key in body_yaml['combination']:
data[key] = 0
for key1 in body_yaml['combination'][key]:
data[key] += data[key1]
data[key] = significant_digits(data[key], 4)
# Volume calculation
for part in body_yaml['part']:
data[f"volume_{part}"] = data[f"length_{part}"] * 3.14 * data[f"weight_{part}"] * data[f"weight_{part}"]
write_xml(f"{dataset_path}/bodies/{i}.xml", data, body_xml)
write_yaml(f"{dataset_path}/params/{i}.yaml", data)
file_list.append(f"bodies/{i}.xml")
param_list.append(f"params/{i}.yaml")
# 4. Write config file
env_id = f"{args.template_body}_{args.num_bodies}_{args.body_variation_range}-v{args.seed_bodies}"
config_yaml = {
"dataset_name": args.template_body,
"real_seed": real_seed,
"bodies": {
"total": len(file_list),
"files": file_list,
"params": param_list,
},
"gym_env": {
"env_id": env_id,
"filename": f"{args.template_body}.py",
"class": f"{args.template_body.capitalize()}Env",
}
}
write_yaml(f"{dataset_path}/config.yaml", config_yaml)
# 5. Copy over Gym Env Python file
shutil.copyfile(template_files[2], f"{dataset_path}/{args.template_body}.py")
return env_id, dataset_path
from utils import read_yaml, make_paths, path2rootdir
from pprint import pprint
d = read_yaml('dsk_train_out.yaml')
allp = make_paths(d)
pprint(allp)
from ROOT import TFile
dirs = []
for path in allp:
rfile, rdir = path2rootdir(path['path'])
dirs.append(rdir)
print dirs
# for d in dirs:
# d.ls()
# t = path2rootdir('dsk_train_out.root')
def read_yaml(self, file_subpath):
return read_yaml(self.to_full_path(file_subpath))
def _read_yaml(params_file):
""" Get params from yaml """
return utils.read_yaml(params_file)
import yaml
from base64 import b64encode
from datetime import datetime
import logging as log
import paho.mqtt.client as mqtt
# import paho.mqtt.publish as publish
import utils
# def read_yaml(file_path):
# with open(file_path, "r") as f:
# return yaml.safe_load(f)
creds = utils.read_yaml("credentials.yml")
camera_config = utils.read_yaml("camera_config.yml")
# MQTT server environment variables
MQTT_HOST =creds['broker']
MQTT_PORT = creds['port']
MQTT_KEEPALIVE_INTERVAL = creds['keep_alive']
MQTT_USER = creds['user']
MQTT_PW = creds['password']
client_id = camera_config['camera_id']
QOS = creds['qos'] # quality of service
def create_timestamp():
now = datetime.now()
date_time = now.strftime("%Y%m%d_%H%M%S")
#print(date_time)
def update_numbering(filename):
'''Update global numbering from saved data.'''
global NUMBERING
saved = utils.read_yaml(filename)
for entry in saved['entries']:
NUMBERING[entry['slug']] = entry['kind']
########################################################################################################
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description="Fine-tune a BERT model for a specific NLP task.")
parser.add_argument(
'--yaml_file',
type=str,
help='''Path to the yaml file containing additional information on how
the dataset is structured.'''
)
args = parser.parse_args()
# Fetch parameters
parameters = read_yaml(args.yaml_file)
check_folder(parameters['output_dir'])
save_yaml(parameters, os.path.join(parameters['output_dir'], 'config.yml'))
logging.basicConfig(filename=os.path.join(parameters['output_dir'],
parameters['log_file']),
filemode='w+',
level=logging.INFO)
logging.info("Parameters fetched.")
logging.info("Setting seed for reproductibility...")
set_seed(parameters['seed'])
logging.info("\tDone.")
logging.info("Set and retrieve the device on which to run...")
device = get_device()
task = parameters['task'].lower()
def test(args):
tf.reset_default_graph()
model_path = args.CBSDeep_model_folder + '//' + args.denoise_model
# print(list(os.walk(model_path, topdown=False))[-1])
# print(list(os.walk(model_path, topdown=False))[-1][-1][0])
# print(list(os.walk(model_path, topdown=False))[-1][-2])
model_list = list(os.walk(model_path, topdown=False))[-1][-2]
yaml_name = list(os.walk(model_path, topdown=False))[-1][-1][0]
print(yaml_name)
read_yaml(args, model_path + '//' + yaml_name)
print('hhhhh ----->', args)
name_list, noise_img, coordinate_list = test_preprocess_lessMemory(args)
num_h = (math.floor((noise_img.shape[1] - args.img_h) / args.gap_h) + 1)
num_w = (math.floor((noise_img.shape[2] - args.img_w) / args.gap_w) + 1)
num_s = (math.floor((noise_img.shape[0] - args.img_s) / args.gap_s) + 1)
TIME = args.datasets_folder + '_' + args.denoise_model #+'_'+args.model_name #+'_'+datetime.datetime.now().strftime("%Y%m%d-%H%M")
results_path = args.results_folder + '//' + 'unet3d_' + TIME + '//'
if not os.path.exists(args.results_folder):
os.mkdir(args.results_folder)
if not os.path.exists(results_path):
os.mkdir(results_path)
model_name = args.model_name
print('model_name -----> ', model_name)
output_path = results_path + '//' + model_name
if not os.path.exists(output_path):
os.mkdir(output_path)
output_graph_path = args.CBSDeep_model_folder + '//' + args.denoise_model + '//' + model_name + '//'
print('output_graph_path -----> ', output_graph_path)
start_time = time.time()
sess = tf.Session()
with sess.as_default():
# sess.run(tf.global_variables_initializer())
meta_graph_def = tf.saved_model.loader.load(sess, ['3D_N2N'],
output_graph_path)
signature = meta_graph_def.signature_def
in_tensor_name = signature['my_signature'].inputs['input0'].name
out_tensor_name = signature['my_signature'].outputs['output0'].name
input = sess.graph.get_tensor_by_name(in_tensor_name)
output = sess.graph.get_tensor_by_name(out_tensor_name)
# sess.run(tf.global_variables_initializer())
'''
variable_names = [v.name for v in tf.trainable_variables()]
values = sess.run(variable_names)
for k,v in zip(variable_names, values):
if len(v.shape)==5:
print("Variable: ", k, "Shape: ", v.shape,"value: ",v[0][0][0][0][0])
if len(v.shape)==1:
print("Variable: ", k, "Shape: ", v.shape,"value: ",v[0])
'''
denoise_img = np.zeros(noise_img.shape)
input_img = np.zeros(noise_img.shape)
for index in range(len(name_list)):
input_name = name_list[index]
single_coordinate = coordinate_list[name_list[index]]
init_h = single_coordinate['init_h']
end_h = single_coordinate['end_h']
init_w = single_coordinate['init_w']
end_w = single_coordinate['end_w']
init_s = single_coordinate['init_s']
end_s = single_coordinate['end_s']
noise_patch1 = noise_img[init_s:end_s, init_h:end_h, init_w:end_w]
train_input = np.expand_dims(
np.expand_dims(noise_patch1.transpose(1, 2, 0), 3), 0)
# print('train_input -----> ',train_input.shape)
data_name = name_list[index]
train_output = sess.run(output, feed_dict={input: train_input})
train_input = np.squeeze(train_input).transpose(2, 0, 1)
train_output = np.squeeze(train_output).transpose(2, 0, 1)
stack_start_w ,stack_end_w ,patch_start_w ,patch_end_w ,\
stack_start_h ,stack_end_h ,patch_start_h ,patch_end_h ,\
stack_start_s ,stack_end_s ,patch_start_s ,patch_end_s = name2index(args, input_name, num_h, num_w, num_s)
aaaa = train_output[patch_start_s:patch_end_s,
patch_start_h:patch_end_h,
patch_start_w:patch_end_w]
bbbb = train_input[patch_start_s:patch_end_s,
patch_start_h:patch_end_h,
patch_start_w:patch_end_w]
denoise_img[stack_start_s:stack_end_s, stack_start_w:stack_end_w, stack_start_h:stack_end_h] \
= train_output[patch_start_s:patch_end_s, patch_start_w:patch_end_w, patch_start_h:patch_end_h]*(np.sum(bbbb)/np.sum(aaaa))**0.5
input_img[stack_start_s:stack_end_s, stack_start_w:stack_end_w, stack_start_h:stack_end_h] \
= train_input[patch_start_s:patch_end_s, patch_start_w:patch_end_w, patch_start_h:patch_end_h]
# print('output_img shape -----> ',output_img.shape)
'''
output_img = denoise_img.squeeze().astype(np.float32)*args.normalize_factor
output_img = np.clip(output_img, 0, 65535).astype('uint16')
result_name = results_path+str(i)+'_'+str(index)+'_'+data_name+'_output.tif'
io.imsave(result_name, output_img.transpose(2,0,1))
'''
# if index % 100 == 0:
# print('denoise_img ---> ',denoise_img.max(),'---> ',denoise_img.min())
# print('input_img ---> ',input_img.max(),'---> ',input_img.min())
output_img = denoise_img.squeeze().astype(
np.float32) * args.normalize_factor
output_img = output_img - output_img.min()
output_img = output_img / output_img.max() * 65535
output_img = np.clip(output_img, 0, 65535).astype('uint16')
output_img = output_img - output_img.min()
input_img = input_img.squeeze().astype(
np.float32) * args.normalize_factor
input_img = np.clip(input_img, 0, 65535).astype('uint16')
result_name = output_path + '//' + 'output_' + model_name + '.tif'
input_name = output_path + '//' + 'input_' + model_name + '.tif'
io.imsave(result_name, output_img)
io.imsave(input_name, input_img)
def test_make_paths(self):
path_dicts = make_paths(read_yaml(self.yamlfile))
# test 1 generic metadata field
self.assertTrue(path_dicts[0]['title'])
for path in path_dicts:
self.assertTrue(path['path'])
if args.y:
args.ymin, args.ymax, args.ynum = [
float(a) for a in args.y.split(':')
]
assert args.ymin and args.ymax and args.ynum, \
'You specified some arguments for the y axis, but not all'
except:
raise Exception(
'Improper format for x- or y-coordinates. Try something like -1:1:51'
)
#--------------------------------------------------------------------------
# Load models and extract parameters
#--------------------------------------------------------------------------
# net = model_loader.load(args.dataset, args.model, args.model_file)
config = read_yaml(args.model_config)
config['use_cuda'] = True
config['network_type'] = args.model
config['dataset']['artificial_and'] = config['artificial_and']
dataloader = ShapeWorldLoader(config=config['dataset'])
config['vocab'] = dataloader.vocab
trainer = Trainer(config)
trainer.model.load_model(args.model_file)
net = trainer.model
net.cpu()
w = net_plotter.get_weights(net) # initial parameters
s = copy.deepcopy(
net.state_dict()) # deepcopy since state_dict are references
if args.ngpu > 1:
default=False, help='Batch mode')
optparser.add_argument('-c', dest='clnameglob', metavar='classifier',
default=None, help='Only plot matching classifiers (globs allowed)')
optparser.add_argument('--rarity', action='store_true',
default=False, help='Plot rarity distributions')
optparser.add_argument('--probab', action='store_true',
default=False, help='Plot classifier probability distributions')
options = optparser.parse_args()
locals().update(_import_args(options))
from pprint import pprint
import sys
from utils import read_yaml, get_rpaths
conf = read_yaml(yamlfile)
if isinstance(conf, list):
for entry in conf:
print entry['file']
if entry['file'] != 'TMVA.root': continue
rfiles = get_rpaths(files, entry)
else:
if conf['file'] == 'TMVA.root':
rfiles = get_rpaths(files, conf)
if not rfiles: sys.exit('Config parsing error.')
from rplot.rdir import Rdir
fnames = [ rfile[0]['file'] for rfile in rfiles ]
rpath_tool = Rdir(fnames)
def execute(cls, unmatched_products_df, product_df):
config_file = utils.read_yaml('config.yaml')
precision = config_file['matching_text']['precision']
text_match_products_df = cls.match_by_text(unmatched_products_df,
product_df, precision)
return text_match_products_df
def test_read_yaml(self):
self.assertTrue(read_yaml(self.yamlfile))
#!/usr/bin/env python3
import argparse
import os
import shutil
import sys
import utils
import yaml
from importlib import import_module
from multiprocessing import Queue
from os.path import join as pjoin
from time import sleep
# module alias (case-insensitive): base filename
cgui_modules = utils.read_yaml('modules.yml')
def log_exception(case_info, step_num, exc_info):
global LOGFILE
templ = 'Job "{}" ({}) encountered an exception on step {}:\n{}\n'
if not 'jobid' in case_info:
case_info['jobid'] = '-1'
jobid = case_info['jobid']
label = case_info['label']
with open(LOGFILE, 'a') as fh:
label = case_info['label']
fh.write(templ.format(label, jobid, step_num, exc_info))
def log_failure(case_info, step, elapsed_time):
global LOGFILE
templ = 'Job "{}" ({}) failed on step {} after {:.2f} seconds\n'
def test_path2rootdir(self):
path_dicts = make_paths(read_yaml(self.yamlfile))
for path in path_dicts:
rfile, rdir = path2rootdir(path['path'])
self.assertTrue(not rfile.IsZombie())
self.assertTrue(rdir)
def convert(args):
"""
Converts files into JSON format for BRITS models
:param yaml_file: configuration file with the following information
- data_folder: path to folder storing data files to be converted
- file_extension: the file extension of the data files to be converted
- only the file types in SUPPORTED_FILE_TYPES (see constants at top)
are supported
- file_regex: regex to specify file names to be converted
- if file_regex is None, then method will convert all files of
type file_extension
- train_split: proportion of files to use as training set
- if None, then JSON data will be saved into one file
- otherwise, will be saved into two files, one for train and one for test
- val_split: proportion of training sequences to use as validation set
- if None, then no splitting will be done
- otherwise, validation data will saved into separate file
- output_folder: path to folder to save JSON data
- output_file_name: file name to save JSON output file
- if train_split is not None, then JSON data will be saved into two files
- train set will be prefixed by "train_", test set prefixed by "test_"
- max_len: maximum sequence length
- if sequence is longer than max_len, gets split
- can be None for no splitting
- seed: random seed
- current randomness comes from random train/test split and random removal
of values for imputation
:param validate_after: boolean determining whether to run validation at the end
- validation can always be run from the command line
:param info_after: boolean determining whether to print info about JSON data
- info can always be run from the command line
"""
print_banner()
print("RUNNING CONVERSION\n")
yaml_data = read_yaml(args["--yaml-file"])
data_folder = yaml_data["data_folder"]
file_extension = yaml_data["file_extension"]
file_regex = yaml_data["file_regex"]
output_file_name = yaml_data["output_file_name"]
output_folder = yaml_data["output_folder"]
train_split = yaml_data["train_split"]
val_split = yaml_data["val_split"]
max_len = yaml_data["max_len"]
seed = yaml_data["seed"]
if file_extension not in SUPPORTED_FILE_TYPES:
print(".{} is not currently a supported file type")
return
if seed is not None:
print("Seeding random generator with seed {}".format(seed))
np.random.seed(seed)
else:
print("WARNING: The random number generator has not been seeded.")
print(
"You are encouraged to run again with a random seed for reproducibility!"
)
file_regex = file_regex if file_regex is not None else ".*"
file_regex += r"\." + file_extension
json_data = extract_json(data_folder, file_regex, file_extension, max_len)
output_files = []
train_data = json_data
test_data = None
val_data = None
if train_split is not None:
train_data, test_data = random_split(train_data, train_split)
save_path = save_json(test_data, output_folder,
"test_" + output_file_name)
output_files.append(save_path)
if val_split is not None:
train_data, val_data = random_split(train_data, 1 - val_split)
save_path = save_json(val_data, output_folder,
"val_" + output_file_name)
output_files.append(save_path)
print("Using {} sequences for training".format(len(train_data)))
save_path = save_json(train_data, output_folder,
"train_" + output_file_name)
output_files.append(save_path)
print_banner()
if args["--validate"]: # run validation on newly saved JSON
for output_file in output_files:
args["--json-file"] = output_file
validate(args)
if args["--info"]: # print info about newly saved JSON
for output_file in output_files:
print_info(output_file)
