def setup_substrate(): """ Sets up the substrate """ #Constants SUBSTRATE_SPEC = './substrate.yaml' IMAGE="TBonTB_C4" template = read_yaml(template_file=SUBSTRATE_SPEC) server_manager = ServerManager(os.environ["OS_USERNAME"], os.environ["OS_PASSWORD"], os.environ["OS_REGION_NAME"], os.environ["OS_TENANT_NAME"]) #First boot the controller contr_flavor = template['Cloud'].get('controller_flavor', 'm1.medium') contr_id = server_manager.create_server('span-contr-1', IMAGE, contr_flavor, key_name='key_spandan', secgroups=['default','spandantb']) #Next boot the agents agent_flavor = template['Cloud'].get('agent_flavor', 'm1.medium') agent_count = template['Cloud'].get('agent_count', 1) agent_ids = [] for i in range(agent_count): agent_ids.append(server_manager.create_server('span-agent-1', IMAGE, agent_flavor, key_name='key_spandan', secgroups=['default','spandantb'])) contr_ip = server_manager.wait_until_sshable(contr_id) agent_ips = [server_manager.wait_until_sshable(agent_id) for agent_id in agent_ids ] return {'contr_ip':contr_ip, 'agents_ips':agent_ip}
def __init__(self, path, settings): super(TravisRepoDispatcher, self).__init__(path, settings) self._travisyml_path = os.path.join(self.repo_path, '.travis.yml') if not os.path.exists(self._travisyml_path): raise IOError('The file .travis.yml does not exist in the directory %s' % self.repo_path) self._travisyml = utils.read_yaml(self._travisyml_path)
def animate(args): # get context ctx = get_extension_context(args.context) nn.set_default_context(ctx) logger.setLevel(logging.ERROR) # to supress minor messages if not args.config: assert not args.params, "pretrained weights file is given, but corresponding config file is not. Please give both." download_provided_file( "https://nnabla.org/pretrained-models/nnabla-examples/GANs/first-order-model/voxceleb_trained_info.yaml" ) args.config = 'voxceleb_trained_info.yaml' download_provided_file( "https://nnabla.org/pretrained-models/nnabla-examples/GANs/first-order-model/pretrained_fomm_params.h5" ) config = read_yaml(args.config) dataset_params = config.dataset_params model_params = config.model_params if args.detailed: vis_params = config.visualizer_params visualizer = Visualizer(**vis_params) if not args.params: assert "log_dir" in config, "no log_dir found in config. therefore failed to locate pretrained parameters." param_file = os.path.join(config.log_dir, config.saved_parameters) else: param_file = args.params print(f"Loading {param_file} for image animation...") nn.load_parameters(param_file) bs, h, w, c = [1] + dataset_params.frame_shape source = nn.Variable((bs, c, h, w)) driving_initial = nn.Variable((bs, c, h, w)) driving = nn.Variable((bs, c, h, w)) filename = args.driving # process repeated until all the test data is used driving_video = read_video( filename, dataset_params.frame_shape) # (#frames, h, w, 3) driving_video = np.transpose(driving_video, (0, 3, 1, 2)) # (#frames, 3, h, w) source_img = imread(args.source, channel_first=True, size=(256, 256)) / 255. source_img = source_img[:3] source.d = np.expand_dims(source_img, 0) driving_initial.d = driving_video[0][:3, ] with nn.parameter_scope("kp_detector"): kp_source = detect_keypoint(source, **model_params.kp_detector_params, **model_params.common_params, test=True, comm=False) persistent_all(kp_source) with nn.parameter_scope("kp_detector"): kp_driving_initial = detect_keypoint(driving_initial, **model_params.kp_detector_params, **model_params.common_params, test=True, comm=False) persistent_all(kp_driving_initial) with nn.parameter_scope("kp_detector"): kp_driving = detect_keypoint(driving, **model_params.kp_detector_params, **model_params.common_params, test=True, comm=False) persistent_all(kp_driving) if args.adapt_movement_scale: nn.forward_all([ kp_source["value"], kp_source["jacobian"], kp_driving_initial["value"], kp_driving_initial["jacobian"] ]) source_area = ConvexHull(kp_source['value'].d[0]).volume driving_area = ConvexHull(kp_driving_initial['value'].d[0]).volume adapt_movement_scale = np.sqrt(source_area) / np.sqrt(driving_area) else: adapt_movement_scale = 1 kp_norm = adjust_kp(kp_source=unlink_all(kp_source), kp_driving=kp_driving, kp_driving_initial=unlink_all(kp_driving_initial), adapt_movement_scale=adapt_movement_scale, use_relative_movement=args.unuse_relative_movement, use_relative_jacobian=args.unuse_relative_jacobian) persistent_all(kp_norm) with nn.parameter_scope("generator"): generated = occlusion_aware_generator(source, kp_source=unlink_all(kp_source), kp_driving=kp_norm, **model_params.generator_params, **model_params.common_params, test=True, comm=False) if not args.full and 'sparse_deformed' in generated: del generated['sparse_deformed'] # remove needless info persistent_all(generated) generated['kp_driving'] = kp_driving generated['kp_source'] = kp_source generated['kp_norm'] = kp_norm # generated contains these values; # 'mask': <Variable((bs, num_kp+1, h/4, w/4)) when scale_factor=0.25 # 'sparse_deformed': <Variable((bs, num_kp+1, num_channel, h/4, w/4)) # (bs, num_kp + 1, c, h, w) # 'occlusion_map': <Variable((bs, 1, h/4, w/4)) # 'deformed': <Variable((bs, c, h, w)) # 'prediction': <Variable((bs, c, h, w)) mode = "arbitrary" if "log_dir" in config: result_dir = os.path.join(args.out_dir, os.path.basename(config.log_dir), f"{mode}") else: result_dir = os.path.join(args.out_dir, "test_result", f"{mode}") # create an empty directory to save generated results _ = nm.Monitor(result_dir) # load the header images. header = imread("imgs/header_combined.png", channel_first=True) generated_images = list() # compute these in advance and reuse nn.forward_all([kp_source["value"], kp_source["jacobian"]], clear_buffer=True) nn.forward_all( [kp_driving_initial["value"], kp_driving_initial["jacobian"]], clear_buffer=True) num_of_driving_frames = driving_video.shape[0] for frame_idx in tqdm(range(num_of_driving_frames)): driving.d = driving_video[frame_idx][:3, ] nn.forward_all([generated["prediction"], generated["deformed"]], clear_buffer=True) if args.detailed: # visualize source w/kp, driving w/kp, deformed source, generated w/kp, generated image, occlusion map visualization = visualizer.visualize(source=source.d, driving=driving.d, out=generated) if args.full: visualization = reshape_result(visualization) # (H, W, C) combined_image = visualization.transpose(2, 0, 1) # (C, H, W) elif args.only_generated: combined_image = np.clip(generated["prediction"].d[0], 0.0, 1.0) combined_image = (255 * combined_image).astype( np.uint8) # (C, H, W) else: # visualize source, driving, and generated image driving_fake = np.concatenate([ np.clip(driving.d[0], 0.0, 1.0), np.clip(generated["prediction"].d[0], 0.0, 1.0) ], axis=2) header_source = np.concatenate([ np.clip(header / 255., 0.0, 1.0), np.clip(source.d[0], 0.0, 1.0) ], axis=2) combined_image = np.concatenate([header_source, driving_fake], axis=1) combined_image = (255 * combined_image).astype(np.uint8) generated_images.append(combined_image) # once each video is generated, save it. output_filename = f"{os.path.splitext(os.path.basename(filename))[0]}.mp4" output_filename = f"{os.path.basename(args.source)}_by_{output_filename}" output_filename = output_filename.replace("#", "_") if args.output_png: monitor_vis = nm.MonitorImage(output_filename, nm.Monitor(result_dir), interval=1, num_images=1, normalize_method=lambda x: x) for frame_idx, img in enumerate(generated_images): monitor_vis.add(frame_idx, img) else: generated_images = [_.transpose(1, 2, 0) for _ in generated_images] # you might need to change ffmpeg_params according to your environment. mimsave(f'{os.path.join(result_dir, output_filename)}', generated_images, fps=args.fps, ffmpeg_params=[ "-pix_fmt", "yuv420p", "-vcodec", "libx264", "-f", "mp4", "-q", "0" ]) return
parser.add_argument('--model', '-m', type=str, default='vqvae', help='Model Name to Train') # Add more options for inference return parser if __name__ == '__main__': parser = get_parser() args = parser.parse_args() device = torch.device("cuda" if torch.cuda.is_available() else "cpu") config = read_yaml(args.config_path) if args.model == 'vqvae': model = VQVAE(config).to(device) trainer = VQVAETrainer(model, config) elif args.model == 'glow': model = Glow(config).to(device) trainer = GlowTrainer(model, config) elif args.model == 'factorvae': vae_model = FactorVAE(config, device).to(device) discriminator = Discriminator(config).to(device) trainer = FactorVAETrainer(vae_model, discriminator, config, device) elif args.model == 'classifiervae': model = ClassifierVAE(config, device).to(device) classifier = Classifier(config).to(device) trainer = ClassifierVAETrainer(model, classifier, config, device)
def migrate_repo(path): """Perform migration to repo on given path.""" click.secho(f"\n>>> Migrating {path}...", fg="green") repo = path.split("/")[-1] repo_underscores = repo.replace("-", "_") # TODO: add the trailing slash only if needed path = path + "/" # Reference: https://codimd.web.cern.ch/TOOkF5yhSAKJq3TiY0L42A?view travis = read_yaml(path + ".travis.yml") try: if travis["deploy"]["provider"] == "pypi": # Download pypi-publish.yml template download_file( GA_PYPI_PUBLISH_YAML_URL, path + ".github/workflows/pypi-publish.yml", ) except Exception as e: logging.info(f"Couldn't find deploy key in .travis.yml") # .editorconfig replace_simple(path + ".travis.yml", ".github/workflows/*.yml", ".editorconfig") # README.rst replace_regex( r"https:\/\/img\.shields\.io\/travis\/([a-z]*\/[a-z-]*)\.svg", "https://github.com/\\1/workflows/CI/badge.svg", path + "README.rst", ) replace_regex( r"https:\/\/travis-ci\.org\/([a-z]*\/[a-z-]*)", "https://github.com/\\1/actions?query=workflow%3ACI", path + "README.rst", ) # CONTRIBUTING.rst replace_regex( r"https:\/\/travis-ci\.(org|com)\/([a-z]*\/[a-z-]*)\/pull_requests", "https://github.com/\\2/actions?query=event%3Apull_request", path + "CONTRIBUTING.rst", ) # tests.yaml build_template(repo, "tests.yml", path=f"{path}/.github/workflows") # run-tests.sh build_template(repo, "run-tests.sh", path=path) # pytest.ini delete_line("pep8ignore", path + "pytest.ini") replace_regex( "(addopts =).*", f'\\1 --isort --pydocstyle --pycodestyle --doctest-glob="*.rst" --doctest-modules --cov={repo_underscores} --cov-report=term-missing', path + "pytest.ini", ) if not file_contains("testpaths", path + "pytest.ini"): append_to_file(f"testpaths = tests {repo_underscores}", path + "pytest.ini") # Add .github/workflows *.yml to MANIFEST.in add_line("recursive-include .github/workflows *.yml\n", path + "MANIFEST.in") # Delete travis file delete_file(path + ".travis.yml") # Upgrade Sphinx 1 to 3 in setup.py replace_regex( r"Sphinx>=1.[0-9].[0-9]", "Sphinx>=3", path + "setup.py", ) # Simplify setup.py test requirements replacing them with pytest-invenio replace_list( path + "setup.py", r"tests_require = (['\"\'[\s*\"(a-z-A-Z><=0-9.\[\]),]*])", [ # Remove packages already installed by pytest-invenio "check-manifest", "coverage", "docker-services-cli", "pytest-celery", "pytest-cov", "pytest-flask", "pytest-isort", "pytest-pycodestyle", "pytest-pydocstyle", "pydocstyle", "pytest", "selenium", # pytest-pep8 is replaced by pytest-pycodestyle "pytest-pep8", # pytest-pep8 is replaced by pytest-isort "isort", ], ["pytest-invenio>=1.4.0"], "tests_require", ) # Remove bak files delete_file(path + "*.bak")
def read_yaml(self, file_subpath): return read_yaml(self.to_full_path(file_subpath))
def reconstruct(args): # get context ctx = get_extension_context(args.context) nn.set_default_context(ctx) logger.setLevel(logging.ERROR) # to supress minor messages config = read_yaml(args.config) dataset_params = config.dataset_params model_params = config.model_params if args.detailed: vis_params = config.visualizer_params visualizer = Visualizer(**vis_params) if not args.params: assert "log_dir" in config, "no log_dir found in config. therefore failed to locate pretrained parameters." param_file = os.path.join( config.log_dir, config.saved_parameters) else: param_file = args.params nn.load_parameters(param_file) bs, h, w, c = [1] + dataset_params.frame_shape source = nn.Variable((bs, c, h, w)) driving_initial = nn.Variable((bs, c, h, w)) driving = nn.Variable((bs, c, h, w)) with nn.parameter_scope("kp_detector"): kp_source = detect_keypoint(source, **model_params.kp_detector_params, **model_params.common_params, test=True, comm=False) persistent_all(kp_source) with nn.parameter_scope("kp_detector"): kp_driving = detect_keypoint(driving, **model_params.kp_detector_params, **model_params.common_params, test=True, comm=False) persistent_all(kp_driving) with nn.parameter_scope("generator"): generated = occlusion_aware_generator(source, kp_source=unlink_all(kp_source), kp_driving=kp_driving, **model_params.generator_params, **model_params.common_params, test=True, comm=False) if not args.full and 'sparse_deformed' in generated: del generated['sparse_deformed'] # remove needless info persistent_all(generated) generated['kp_driving'] = kp_driving generated['kp_source'] = kp_source # generated contains these values; # 'mask': <Variable((bs, num_kp+1, h/4, w/4)) when scale_factor=0.25 # 'sparse_deformed': <Variable((bs, num_kp+1, num_channel, h/4, w/4)) # (bs, num_kp + 1, c, h, w) # 'occlusion_map': <Variable((bs, 1, h/4, w/4)) # 'deformed': <Variable((bs, c, h, w)) # 'prediction': <Variable((bs, c, h, w)) mode = "reconstruction" if "log_dir" in config: result_dir = os.path.join(args.out_dir, os.path.basename(config.log_dir), f"{mode}") else: result_dir = os.path.join(args.out_dir, "test_result", f"{mode}") # create an empty directory to save generated results _ = nm.Monitor(result_dir) if args.eval: os.makedirs(os.path.join(result_dir, "png"), exist_ok=True) # load the header images. header = imread("imgs/header_combined.png", channel_first=True) filenames = sorted(glob.glob(os.path.join( dataset_params.root_dir, "test", "*"))) recon_loss_list = list() for filename in tqdm(filenames): # process repeated until all the test data is used driving_video = read_video( filename, dataset_params.frame_shape) # (#frames, h, w, 3) driving_video = np.transpose( driving_video, (0, 3, 1, 2)) # (#frames, 3, h, w) generated_images = list() source_img = driving_video[0] source.d = np.expand_dims(source_img, 0) driving_initial.d = driving_video[0] # compute these in advance and reuse nn.forward_all( [kp_source["value"], kp_source["jacobian"]], clear_buffer=True) num_of_driving_frames = driving_video.shape[0] for frame_idx in tqdm(range(num_of_driving_frames)): driving.d = driving_video[frame_idx] nn.forward_all([generated["prediction"], generated["deformed"]], clear_buffer=True) if args.detailed: # visualize source w/kp, driving w/kp, deformed source, generated w/kp, generated image, occlusion map visualization = visualizer.visualize( source=source.d, driving=driving.d, out=generated) if args.full: visualization = reshape_result(visualization) # (H, W, C) combined_image = visualization.transpose(2, 0, 1) # (C, H, W) elif args.only_generated: combined_image = np.clip( generated["prediction"].d[0], 0.0, 1.0) combined_image = ( 255*combined_image).astype(np.uint8) # (C, H, W) else: # visualize source, driving, and generated image driving_fake = np.concatenate([np.clip(driving.d[0], 0.0, 1.0), np.clip(generated["prediction"].d[0], 0.0, 1.0)], axis=2) header_source = np.concatenate([np.clip(header / 255., 0.0, 1.0), np.clip(source.d[0], 0.0, 1.0)], axis=2) combined_image = np.concatenate( [header_source, driving_fake], axis=1) combined_image = (255*combined_image).astype(np.uint8) generated_images.append(combined_image) # compute L1 distance per frame. recon_loss_list.append( np.mean(np.abs(generated["prediction"].d[0] - driving.d[0]))) # post process only for reconstruction evaluation. if args.eval: # crop generated images region only. if args.only_generated: eval_images = generated_images elif args.full: eval_images = [_[:, :h, 4*w:5*w] for _ in generated_images] elif args.detailed: assert generated_images[0].shape == (c, h, 5*w) eval_images = [_[:, :, 3*w:4*w] for _ in generated_images] else: eval_images = [_[:, h:, w:] for _ in generated_images] # place them horizontally and save for evaluation. image_for_eval = np.concatenate( eval_images, axis=2).transpose(1, 2, 0) imsave(os.path.join(result_dir, "png", f"{os.path.basename(filename)}.png"), image_for_eval) # once each video is generated, save it. output_filename = f"{os.path.splitext(os.path.basename(filename))[0]}.mp4" if args.output_png: monitor_vis = nm.MonitorImage(output_filename, nm.Monitor(result_dir), interval=1, num_images=1, normalize_method=lambda x: x) for frame_idx, img in enumerate(generated_images): monitor_vis.add(frame_idx, img) else: generated_images = [_.transpose(1, 2, 0) for _ in generated_images] # you might need to change ffmpeg_params according to your environment. mimsave(f'{os.path.join(result_dir, output_filename)}', generated_images, fps=args.fps, ffmpeg_params=["-pix_fmt", "yuv420p", "-vcodec", "libx264", "-f", "mp4", "-q", "0"]) print(f"Reconstruction loss: {np.mean(recon_loss_list)}") return
import sys from create_api import * from utils import read_yaml from configs import conf if __name__ == '__main__': print("arguments :", sys.argv) api = read_yaml(conf('docs_main_path'), 'apis.yaml')[sys.argv[1]] api_executor(api)
# make log dir logdir = os.path.join(outputdir, "log") if not os.path.exists(logdir): os.makedirs(logdir) def prepare_adjoint(config): # mkdir outputbase outputbase = config["outputdir"] clean_outputdir(outputbase) eventlist_file = config["eventlist"] eventlist = read_txt(eventlist_file) # split the eventlist and dump into separate files nevents_per_job = config["nevents_per_job"] cmtlist_per_job = split_job(eventlist, nevents_per_job) print("-"*10 + "\nJob list:\n%s" % cmtlist_per_job) print("="*20 + "\nPreparing jobs...") for job_id, cmtlist in cmtlist_per_job.iteritems(): prepare_one_job(job_id, cmtlist, config) if __name__ == "__main__": config = read_yaml("config.yaml") prepare_adjoint(config)
def run(exp_name): nilmtk_models = ["CO", "AFHMM"] nilmtk_appliances = [ "fridge", "microwave", "washing machine", "television", "electric heating element", "cooker" ] experiment = utils.read_yaml(f"experiment_configs/{exp_name}.yaml") pass devices = experiment["devices"] interval = experiment["interval"] use_weather = experiment["use_weather"] use_occupancy = experiment["use_occupancy"] times = {"train_times": {}, "test_times": {}} for model in experiment["models"].values(): network_type = model["network_type"] window_length = model["window_length"] output_length = model["output_length"] epochs = model["epochs"] learning_rate = model["learning_rate"] batch_size = model["batch_size"] early_stopping = model["early_stopping"] patience = model["patience"] restore_weights = model["restore_weights"] times["train_times"][network_type] = {} times["test_times"][network_type] = {} if network_type in nilmtk_models: if use_weather or use_occupancy: continue else: train_time_agg = 0 test_time_agg = 0 eval_counter = 0 for test_set in experiment["evaluate"]: train_time, test_time = nilmtk_run.run_model( model_type=network_type, appliances=nilmtk_appliances, interval=interval, test_dataset=test_set, experiment_name=exp_name, return_time=True, export_predictions=True, verbose=False) train_time_agg += train_time test_time_agg += test_time eval_counter += 1 times["train_times"][ network_type] = train_time_agg / eval_counter times["test_times"][ network_type] = test_time_agg / eval_counter else: for device in devices: training_directory = f"../data/appliances/{device}/{device}_train_{interval}_.csv" validation_directory = f"../data/appliances/{device}/{device}_val_{interval}_.csv" save_model_dir = utils.get_model_save_path( network_type=network_type, interval=interval, device=device, window_length=window_length, use_weather=use_weather, use_occupancy=use_occupancy) trainer = Trainer(appliance=device, batch_size=batch_size, crop=None, network_type=network_type, use_weather=use_weather, use_occupancy=use_occupancy, learning_rate=learning_rate, epochs=epochs, early_stopping=early_stopping, patience=patience, restore_weights=restore_weights, training_directory=training_directory, validation_directory=validation_directory, save_model_dir=save_model_dir, input_window_length=window_length, output_length=output_length, validation_frequency=1, return_time=True, plot_training=False) train_time = trainer.train_model() times["train_times"][network_type][device] = train_time # Average test_times across all datasets test_time_agg = 0 eval_counter = 0 for test_set in experiment["evaluate"]: test_directory = f"../data/appliances/{device}/{device}_{test_set}_{interval}_.csv" if os.path.isfile(test_directory): # The logs including results will be recorded to this log file utils.check_dir(f"outputs/logs/{exp_name}/") log_file_dir = f"outputs/logs/{exp_name}/{network_type}_{device}.log" tester = Tester(appliance=device, algorithm=network_type, crop=None, batch_size=batch_size, network_type=network_type, test_directory=test_directory, saved_model_dir=save_model_dir, log_file_dir=log_file_dir, input_window_length=window_length, output_length=output_length, use_weather=use_weather, use_occupancy=use_occupancy, plot_first=-1, dataset=test_set, return_time=True, return_predictions=True) pred, test_time = tester.test_model() test_time_agg += test_time eval_counter += 1 utils.check_dir( f"outputs/model_predictions/{exp_name}/") results_path = f"outputs/model_predictions/{exp_name}/{network_type}_{device}_{test_set}.csv" pd.DataFrame(pred).to_csv(results_path, sep=";") times["test_times"][network_type][ device] = test_time_agg / eval_counter utils.save_dict(times, f"outputs/experiment_logs/{exp_name}_training_times")
def main(): conf = utils.read_yaml(utils.get_arguments()["unknown"][0]) os.remove(utils.get_arguments()["unknown"][0]) # Clone repositories. Using os.getcwd() because the drive letter needs to # be respected on Windows. repositories_path = os.path.abspath( os.path.join(os.getcwd(), "repositories", conf["name"])) os.environ["CONDA_ENVIRONMENT_REPOSITORIES"] = repositories_path # Kept for backwards compatibility os.environ["CONDA_GIT_REPOSITORY"] = repositories_path repositories = [] for item in conf["dependencies"]: if "git" in item: for repo in item["git"]: repo_path = "" if isinstance(repo, str): repo_path = repo if isinstance(repo, dict): repo_path = repo.keys()[0] data = {"url": repo_path} name = repo_path.split("/")[-1].replace(".git", "") if not name: name = repo_path.split("/")[-2] if "@" in name: name = name.split("@")[0] repo_path = repo_path.split("@")[0] data["name"] = name if not os.path.exists(repositories_path): os.makedirs(repositories_path) if name not in os.listdir(repositories_path): subprocess.call(["git", "clone", repo_path], cwd=repositories_path) data["path"] = os.path.join(repositories_path, name) data["commands"] = { "on_launch": [], "on_environment_update": [] } if isinstance(repo, dict): for item in repo[repo.keys()[0]]: if isinstance(item, dict): for event, commands in item.iteritems(): data["commands"][event].extend(commands) else: data["commands"]["on_launch"].append(item) repositories.append(data) # Update repositories. if utils.get_arguments()["update-repositories"]: for repo in repositories: print repo["name"] # Updating origin url subprocess.call([ "git", "remote", "set-url", "origin", repo["url"].split("@")[0] ], cwd=repo["path"]) # Update git repository subprocess.call(["git", "checkout", "master"], cwd=repo["path"]) subprocess.call(["git", "pull"], cwd=repo["path"]) subprocess.call( ["git", "submodule", "update", "--init", "--recursive"], cwd=repo["path"]) subprocess.call(["git", "submodule", "update", "--recursive"], cwd=repo["path"]) # Checkout any commits/tags if there are newly cloned repositories or # updating the repositories. if utils.get_arguments()["update-repositories"]: for repo in repositories: if "@" in repo["url"]: tag = repo["url"].split("@")[1] if tag: print repo["name"] subprocess.call(["git", "checkout", tag], cwd=repo["path"]) # Checkout environment repository environment_path = utils.get_environment() if not os.path.exists(environment_path): # Determine environment repositories by matching passed environment # with repositories environment_repo = None match = 0.0 for repo in repositories: sequence_match = SequenceMatcher(None, repo["url"], environment_path).ratio() if match < sequence_match: environment_repo = repo print environment_repo["name"] branch = environment_path.split("/")[-2] subprocess.call(["git", "checkout", branch], cwd=environment_repo["path"]) # Install any setup.py if we are updating if (utils.get_arguments()["update-repositories"] or utils.get_arguments()["update-environment"]): for repo in repositories: if "setup.py" not in os.listdir(repo["path"]): continue args = ["python", "setup.py", "develop"] subprocess.call(args, cwd=repo["path"]) # Add environment site packages to os.environ prefix = "" if platform.system().lower() == "windows": prefix = os.environ["CONDA_PREFIX"] else: prefix = os.environ["CONDA_ENV_PATH"] path = os.path.join(prefix, "lib", "site-packages") os.environ["PYTHONPATH"] += os.pathsep + path # Add sys.path to os.environ["PYTHONPATH"], because conda only modifies # sys.path which gets lost when launching any detached subprocesses. # This get a little complicated due to being in a process that hasn"t # picked up on the changes, hence going through a subprocess. python_file = os.path.join(os.path.dirname(__file__), "write_sys_path.py") data_file = os.path.join(tempfile.gettempdir(), "data_%s.yml" % os.getpid()) subprocess.call(["python", python_file, data_file]) paths = [] with open(data_file, "r") as f: paths += utils.read_yaml(f.read()) os.remove(data_file) for path in paths: if path.lower().startswith(repositories_path.lower()): os.environ["PYTHONPATH"] += os.pathsep + path if path.endswith(".egg"): os.environ["PYTHONPATH"] += os.pathsep + path # Clean up any existing environment file if os.path.exists(utils.get_environment_path()): os.remove(utils.get_environment_path()) # Ensure subprocess is detached so closing connect will not also # close launched applications. options = {} if not utils.get_arguments()["attached"]: if sys.platform == "win32": options["creationflags"] = subprocess.CREATE_NEW_CONSOLE else: options["preexec_fn"] = os.setsid # Setting update mode environment variable update_modes = [] if utils.get_arguments()["update-environment"]: update_modes.append("environment") if utils.get_arguments()["update-repositories"]: update_modes.append("repositories") os.environ["CONDA_GIT_UPDATE"] = "" for mode in update_modes: os.environ["CONDA_GIT_UPDATE"] += mode + os.pathsep # Execute environment update commands. if utils.get_arguments()["update-environment"]: for repo in repositories: if "commands" in repo.keys(): for cmd in repo["commands"]["on_environment_update"]: os.environ.update(utils.read_environment()) cmd = cmd.replace("$REPO_PATH", repo["path"]) print "Executing: " + cmd subprocess.call(cmd, shell=True, cwd=repo["path"], **options) # Execute launch commands. for repo in repositories: if "commands" in repo.keys(): for cmd in repo["commands"]["on_launch"]: os.environ.update(utils.read_environment()) cmd = cmd.replace("$REPO_PATH", repo["path"]) print "Executing: " + cmd subprocess.call(cmd, shell=True, cwd=repo["path"], **options)
opt = parser.parse_args() print('the parameter of your training ----->') print(opt) ######################################################################################################################## os.environ["CUDA_VISIBLE_DEVICES"] = str(opt.GPU) model_path = opt.pth_path + '//' + opt.denoise_model # print(model_path) model_list = list(os.walk(model_path, topdown=False))[-1][-1] # print(model_list) for i in range(len(model_list)): aaa = model_list[i] if '.yaml' in aaa: yaml_name = model_list[i] print(yaml_name) read_yaml(opt, model_path + '//' + yaml_name) # print(opt.datasets_folder) name_list, noise_img, coordinate_list = test_preprocess_lessMemoryNoTail(opt) # name_list, noise_img, coordinate_list = test_preprocess_lessMemory(opt) # trainX = np.expand_dims(np.array(train_raw),4) num_h = (math.floor((noise_img.shape[1] - opt.img_h) / opt.gap_h) + 1) num_w = (math.floor((noise_img.shape[2] - opt.img_w) / opt.gap_w) + 1) num_s = (math.floor((noise_img.shape[0] - opt.img_s) / opt.gap_s) + 1) # print(num_h, num_w, num_s) # print(coordinate_list) if not os.path.exists(opt.output_dir): os.mkdir(opt.output_dir) current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M") output_path1 = opt.output_dir + '//' + opt.datasets_folder + '_' + current_time + '_' + opt.denoise_model
import hashlib import struct import time from math import ceil from connection import Connection, MAX_DATA_SIZE from utils import read_yaml, log if __name__ == '__main__': # Read configuration config = read_yaml("../config.yaml") my_ip, my_port = config["sender_ip"], config["port"]["acknowledgement"][ "target"] remote_ip, remote_port = config["receiver_ip"], config["port"]["data"][ "source"] file_name = config["file_name"]["sender"] chunk_size = MAX_DATA_SIZE - 4 timeout = config["timeout"] # Read the whole file as bytes with open(file_name, "rb") as file: file_bytes = file.read(-1) # We must send the number of chunks to the server, # so that they know how many messages to receive chunks_count = ceil(len(file_bytes) / chunk_size) md5 = hashlib.md5(file_bytes).digest() with Connection(my_ip, my_port, remote_ip, remote_port,
scores_window.append(score) # save most recent score scores.append(score) # save most recent score avg_av = agent.evaluate_on_fixed_set(fixed_states) average_action_values.append(avg_av) print(f'Episode {i_episode}\tAverage Score: ' f'{round(np.mean(scores_window),4)}\tEpsilon: {round(eps, 4)}\t' f'Average Q value: {round(avg_av, 4)}') if i_episode % conf['save_every'] == 0 and i_episode > 0: print(f'Saving model at iteration: {i_episode}') save_model(conf, agent) env.close() return { 'scores': scores, 'epsilons': epsilons, 'avg_action_values': average_action_values } if __name__ == '__main__': arguments = parse_args() pc = arguments.path_config exp_conf = read_yaml(pc) stats = train(exp_conf) save_scores(exp_conf, stats)
def test_read_yaml(self): self.assertTrue(read_yaml(self.yamlfile))
def test_path2rootdir(self): path_dicts = make_paths(read_yaml(self.yamlfile)) for path in path_dicts: rfile, rdir = path2rootdir(path['path']) self.assertTrue(not rfile.IsZombie()) self.assertTrue(rdir)
def train(fp_config): """ Performs a DINO training experiment based on a provided configuration """ # ================ CONFIGURATION ================ # Load the configuration file config = utils.read_yaml(fp_config) # Announce the configuration print(f"Using the configuration at: {fp_config}") for k in config.keys(): print(f" - {k}: {config[k]}") # ================ DEVICE ================ device = utils.report_and_get_device_gpu_preferred() # ================ DATA ================ print("Loading training data ... ", end="") train_loader, test_loader = dw.get_mars32k_train_test_dataloaders( config['fp_data_mars32k'], config['data_proportion'], config['train_test_ratio'], config['batch_size'], config['perform_shuffle']) print("ready") # ================ STUDENT AND TEACHER NETWORKS ================ # Create the student network print("Building student network ... ", end="") nn_student = utils.build_vision_transformer_from_config(config) print("ready") # Create the teacher network print("Building teacher network ... ", end="") nn_teacher = utils.build_vision_transformer_from_config(config) # Copy the student weights over to the teacher (initially they are the same) nn_teacher.load_state_dict(nn_student.state_dict()) # There is no backpropagation through the teacher for param in nn_teacher.parameters(): param.requires_grad = False # Move them both to the desired device nn_student.to(device) nn_teacher.to(device) print("ready") # ================ MULTICROPPER ================ # Build the multicropper module print("Instantiating multicropper ... ", end="") multicropper = MultiCropper(global_size=config['global_size'], local_size=config['local_size'], n_global_crops=config['n_global_crops'], n_local_crops=config['n_local_crops']) print("ready") # ================ SCHEDULERS ================ # Create learning rate scheduler and weight decay scheduler for the optimiser sch_lr = utils.LinearPiecewiseScheduler(config['lr_values'], config['lr_epochs']) sch_wd = utils.LinearPiecewiseScheduler(config['weight_decay_values'], config['weight_decay_epochs']) # Create temperature schedulers sch_temp_student = utils.LinearPiecewiseScheduler( config['temp_student_values'], config['temp_student_epochs']) # Too high temperature at the start may cause the teacher's training # to be unstable. Investigate the use of warm up as necessary sch_temp_teacher = utils.LinearPiecewiseScheduler( config['temp_teacher_values'], config['temp_teacher_epochs']) # The centering rate parameter (usually in range [0.9, 0.999]) sch_cent_rate_m = utils.LinearPiecewiseScheduler( config['cent_rate_m_values'], config['cent_rate_m_epochs']) # The lambda weight transfer parameter (the amount of teacher network v. student # network to include in the next iteration of the teacher network) # TODO a cosine scheduler should be used if possible sch_lambda_ema = utils.LinearPiecewiseScheduler( config['lambda_ema_values'], config['lambda_ema_epochs']) # ================ OPTIMISER ================ # Preparing the optimiser print("Preparing optimiser ... ", end="") # Note that it is recommended to only construct this after the # models are on the GPU (if using a GPU) optimiser = torch.optim.AdamW(nn_student.parameters(), lr=sch_lr.get_value(0), weight_decay=sch_wd.get_value(0)) print("ready") # ================ DINO LOSS ================ # Prepare the loss module print("Preparing DINO loss module ... ", end="") loss_computer = LossComputerDINO(output_dim=config['embed_dim']) loss_computer.to(device) print("ready") # ================ TRAINING ================ print("Preparing training procedure ... ", end="") fp_save = config['fp_save'] # Create a scalar for mixed precision as # desired (no op if flag is false) scaler = torch.cuda.amp.GradScaler(enabled=config['mixed_precision']) # Synchronise gc.collect() torch.cuda.empty_cache() torch.cuda.reset_max_memory_allocated() torch.cuda.synchronize() print("ready") time.sleep(0.5) print("I'm your huckleberry!") # Perform as many epochs of training as required metrics = [] n_epochs = config['n_epochs'] for epoch_idx in range(n_epochs): # Get the schedule values learning_rate = sch_lr.get_value(epoch_idx) weight_decay = sch_wd.get_value(epoch_idx) temp_student = sch_temp_student.get_value(epoch_idx) temp_teacher = sch_temp_teacher.get_value(epoch_idx) cent_rate_m = sch_cent_rate_m.get_value(epoch_idx) lambda_ema = sch_lambda_ema.get_value(epoch_idx) # And train a single epoch epoch_metrics = train_single_epoch( epoch_idx=epoch_idx, n_epochs=n_epochs, device=device, scaler=scaler, mixed_precision=config['mixed_precision'], optimiser=optimiser, nn_student=nn_student, nn_teacher=nn_teacher, loss_computer=loss_computer, train_loader=train_loader, multicropper=multicropper, learning_rate=learning_rate, weight_decay=weight_decay, temp_student=temp_student, temp_teacher=temp_teacher, cent_rate_m=cent_rate_m, lambda_ema=lambda_ema) metrics.append(epoch_metrics) # ================ SAVING ================ print("Saving experiment ... ", end="") utils.save_dino_experiment(fp_save=fp_save, nn_student=nn_student, nn_teacher=nn_teacher, optimiser=optimiser, loss_computer=loss_computer, config=config, metrics=metrics) print("complete")
def test_make_paths(self): path_dicts = make_paths(read_yaml(self.yamlfile)) # test 1 generic metadata field self.assertTrue(path_dicts[0]['title']) for path in path_dicts: self.assertTrue(path['path'])
def get_config(): """ Get command line arguments. Arguments set the default values of command line arguments. """ parser = argparse.ArgumentParser(description='TecoGAN') parser.add_argument('--cfg', default="./config.yaml") args, _ = parser.parse_known_args() conf = read_yaml(args.cfg) parser.add_argument( '--input_video_dir', default=conf.data.input_video_dir, help='The directory of the video input data, for training') parser.add_argument('--output_dir', default=conf.data.output_dir, help='The output directory of the checkpoint') parser.add_argument( '--num_resblock', type=int, default=conf.train.num_resblock, help='How many residual blocks are there in the generator') parser.add_argument('--max_iter', type=int, default=conf.train.max_iter, help='max iteration for training') parser.add_argument( '--pre_trained_frvsr_weights', type=str, default=conf.train.pre_trained_frvsr_weights, help='the weight of frvsr generator will be loaded as an initial point' ) parser.add_argument('--vgg_pre_trained_weights', type=str, default=conf.train.vgg_pre_trained_weights, help='path to pre-trained weights for the vgg19') parser.add_argument( '--tecogan', type=bool, default=conf.train.tecogan, help='True for Tecogan training False for FRVSR training') parser.add_argument('--checkpoint', type=str, default=conf.train.checkpoint, help='path to checkpoint file') args = parser.parse_args() # refine config conf.data.input_video_dir = args.input_video_dir conf.train.max_iter = conf.train.max_iter conf.data.output_dir = args.output_dir conf.train.num_resblock = args.num_resblock conf.train.pre_trained_frvsr_weights = args.pre_trained_frvsr_weights conf.train.vgg_pre_trained_weights = args.vgg_pre_trained_weights conf.train.tecogan = args.tecogan conf.train.checkpoint = args.checkpoint return conf
def __init__(self, **cli_args): logger.info(f"Entered CLI args: {cli_args}") logger.info(f"Executing command: {cli_args.get('cmd')} ...") self.data_path: str = str( cli_args.get("data_path")) # path to the dataset logger.info(f"reading data from {self.data_path}") self.command = cli_args.get("cmd", None) if not self.command or self.command not in self.available_commands: raise Exception(f"You must enter a valid command.\n" f"available commands: {self.available_commands}") if self.command == "fit": self.yml_path = str(cli_args.get("yaml_path")) file_ext = self.yml_path.split(".")[-1] logger.info(f"You passed the configurations as a {file_ext} file.") self.yaml_configs = (read_yaml(self.yml_path) if file_ext == "yaml" else read_json(self.yml_path)) logger.info(f"your chosen configuration: {self.yaml_configs}") # dataset options given by the user self.dataset_props: dict = self.yaml_configs.get( "dataset", self.default_dataset_props) # model options given by the user self.model_props: dict = self.yaml_configs.get( "model", self.default_model_props) # list of target(s) to predict self.target: list = self.yaml_configs.get("target") self.model_type: str = self.model_props.get("type") logger.info(f"dataset_props: {self.dataset_props} \n" f"model_props: {self.model_props} \n " f"target: {self.target} \n") # handle random numbers generation random_num_options = self.dataset_props.get("random_numbers", None) if random_num_options: generate_reproducible = random_num_options.get( "generate_reproducible", None) if generate_reproducible: logger.info( "You provided the generate reproducible results option." ) seed = random_num_options.get("seed", 42) np.random.seed(seed) logger.info( f"Setting a seed = {seed} to generate same random numbers on each experiment.." ) # if entered command is evaluate or predict, then the pre-fitted model needs to be loaded and used else: self.model_path = cli_args.get("model_path", self.default_model_path) logger.info(f"path of the pre-fitted model => {self.model_path}") self.prediction_file = cli_args.get("prediction_file", self.prediction_file) # set description.json if provided: self.description_file = cli_args.get("description_file", self.description_file) # load description file to read stored training parameters with open(self.description_file) as f: dic = json.load(f) self.target: list = dic.get( "target") # target to predict as a list self.model_type: str = dic.get( "type" ) # type of the model -> regression, classification or clustering self.dataset_props: dict = dic.get( "dataset_props") # dataset props entered while fitting getattr(self, self.command)()
unidades_economicas_pequenas_distancia_km * {costo_eco_pequenas} + unidades_economicas_medianas_distancia_km * {costo_eco_medianas} + carretera_distancia_km * {costo_transporte} AS costo_construccion, CASE WHEN zona_urbana_distancia_min = 0 THEN 0 ELSE 1 END AS dummy_urbano FROM features.features_{grid_size} ) SELECT cell_id, year, 0 as CTN, costo_construccion * dummy_urbano AS CFP, costo_construccion * dummy_urbano * {antes} AS CTP, costo_construccion * dummy_urbano * {despues} AS CFN FROM costos_construccion)""".format(grid_size=grid_size, costo_calles=costo_calles, costo_eco_pequenas=costo_eco_pequenas, costo_eco_medianas=costo_eco_medianas, costo_transporte=costo_transporte, antes=antes, despues=despues)) db_conn = utils.get_connection().raw_connection() cur = db_conn.cursor() cur.execute(DROP) db_conn.commit() cur.execute(QUERY) db_conn.commit() if __name__ == '__main__': experiment_path = '../experiment.yaml' experiment = utils.read_yaml(experiment_path) gen_cost_matrix(experiment['costos'], experiment['grid_size'])
import os from attrdict import AttrDict from deepsense import neptune from utils import read_yaml ctx = neptune.Context() params = ctx.params if params.__class__.__name__ == 'OfflineContextParams': neptune_config = read_yaml('neptune_config.yaml') params = neptune_config.parameters X_COLUMNS = ['comment_text'] Y_COLUMNS = [ 'toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate' ] SOLUTION_CONFIG = AttrDict({ 'env': { 'cache_dirpath': params.experiment_dir }, 'xy_splitter': { 'x_columns': X_COLUMNS, 'y_columns': Y_COLUMNS }, 'text_cleaner': { 'drop_punctuation': bool(params.drop_punctuation), 'drop_newline': bool(params.drop_newline), 'drop_multispaces': bool(params.drop_multispaces),
def create(ctx, number, existing, proposed, load, path=default_path, **kwargs): """ Main function that creates all the inputs 🔥""" # if ctx.obj['DEBUG']: # def debug(type, value, tb): # import traceback, pdb # traceback.print_exception(type, value, tb) # pdb.pm() # sys.excepthook = debug click.echo('Starting app') if existing and proposed: click.echo('Creating generation project info') gen_project_legacy = pd.read_csv(os.path.join( default_path, 'generation_projects_info.tab'), sep='\t') gen_project_proposed = init_scenario() gen_project = pd.concat([gen_project_legacy, gen_project_proposed]) gen_legacy = gen_build_predetermined(existing) create_gen_build_cost(gen_project, gen_legacy) else: click.echo('Oops I do not know what to do yet') sys.exit(1) # Finally gen_project.to_csv(os.path.join(output_path, 'generation_projects_info.tab'), sep='\t', index=False) click.echo(f'Number of timepoints selected: {number}') click.echo(f'Reading load data') load_data = get_load_data(filename=load) click.echo(f'Reading periods data') periods = read_yaml(path, 'periods.yaml') d = OrderedDict(periods) periods_tab = pd.DataFrame(d) periods_tab = periods_tab.set_index('INVESTMENT_PERIOD') click.echo(f'Creating timeseries') timeseries, timeseries_dict = [], {} for periods, row in periods_tab.iterrows(): timeseries_dict[periods] = [] scale_to_period = row[1] - row[0] peak_data = get_peak_day(load_data[str(periods)]['total'], number, freq='1MS', **kwargs) median_data = get_median_day(load_data[str(periods)]['total'], number, freq='1MS', **kwargs) timeseries_dict[periods].append( create_strings(peak_data, scale_to_period)) timeseries_dict[periods].append( create_strings(median_data, scale_to_period, identifier='M')) timeseries.append(create_strings(peak_data, scale_to_period)) timeseries.append( create_strings(median_data, scale_to_period, identifier='M')) click.echo(f'Creating investment period') create_investment_period() # create_gen_build_cost_new(peak_data) create_timeseries(timeseries, number, **kwargs) create_timepoints(timeseries) click.echo(f'Creating variable capacity factor') create_variablecp(gen_project, timeseries, timeseries_dict) click.echo(f'Creating loads') create_loads(load_data, timeseries) rps_file, ext = look_for_file('rps_targets', default_path) click.echo(f'Creating fuel loads') create_fuel_cost() if rps_file: click.echo(f'Creating rps') create_rps(filename=rps_file, ext=ext) click.echo(f'App ended')
def generate_bodies(): assert args.seed_bodies<100, "The way we combining real seeds only allow seed_bodies to be smaller than 100." assert args.body_variation_range%10==0, "The way we combining real seeds only allow body_variation_range to be multiplication of 10, e.g. 10, 20, ... 90." assert args.num_bodies%10==0, "The way we combining real seeds only allow num_bodies to be multiplication of 10, e.g. 10, 20, 30 ..." real_seed = args.num_bodies*1000 + args.body_variation_range * 100 + args.seed_bodies # if I only apply seed_bodies, the first 20 of walker2d_30_10-v0 will be the same of walker2d_20_10-v0. random.seed(real_seed) output(f"Start generating bodies {args.num_bodies} with seed {real_seed}", 1) # 1. Check templates template_files = check_templates() # 2. Create Folders dataset_path = create_folder() # 3. Generate variations and write body, param files body_xml = read_template(template_files[0]) body_yaml = read_yaml(template_files[1]) file_list, param_list = [], [] for i in range(args.num_bodies): data = {} for key in body_yaml['variable']: data[key] = body_yaml['variable'][key] * ((random.random() * 2 - 1) * args.body_variation_range / 100 + 1.0) data[key] = significant_digits(data[key], 4) for key in body_yaml['fixed']: data[key] = body_yaml['fixed'][key] for key in body_yaml['combination']: data[key] = 0 for key1 in body_yaml['combination'][key]: data[key] += data[key1] data[key] = significant_digits(data[key], 4) # Volume calculation for part in body_yaml['part']: data[f"volume_{part}"] = data[f"length_{part}"] * 3.14 * data[f"weight_{part}"] * data[f"weight_{part}"] write_xml(f"{dataset_path}/bodies/{i}.xml", data, body_xml) write_yaml(f"{dataset_path}/params/{i}.yaml", data) file_list.append(f"bodies/{i}.xml") param_list.append(f"params/{i}.yaml") # 4. Write config file env_id = f"{args.template_body}_{args.num_bodies}_{args.body_variation_range}-v{args.seed_bodies}" config_yaml = { "dataset_name": args.template_body, "real_seed": real_seed, "bodies": { "total": len(file_list), "files": file_list, "params": param_list, }, "gym_env": { "env_id": env_id, "filename": f"{args.template_body}.py", "class": f"{args.template_body.capitalize()}Env", } } write_yaml(f"{dataset_path}/config.yaml", config_yaml) # 5. Copy over Gym Env Python file shutil.copyfile(template_files[2], f"{dataset_path}/{args.template_body}.py") return env_id, dataset_path
from utils import read_yaml, make_paths, path2rootdir from pprint import pprint d = read_yaml('dsk_train_out.yaml') allp = make_paths(d) pprint(allp) from ROOT import TFile dirs = [] for path in allp: rfile, rdir = path2rootdir(path['path']) dirs.append(rdir) print dirs # for d in dirs: # d.ls() # t = path2rootdir('dsk_train_out.root')
def _read_yaml(params_file): """ Get params from yaml """ return utils.read_yaml(params_file)
import yaml from base64 import b64encode from datetime import datetime import logging as log import paho.mqtt.client as mqtt # import paho.mqtt.publish as publish import utils # def read_yaml(file_path): # with open(file_path, "r") as f: # return yaml.safe_load(f) creds = utils.read_yaml("credentials.yml") camera_config = utils.read_yaml("camera_config.yml") # MQTT server environment variables MQTT_HOST =creds['broker'] MQTT_PORT = creds['port'] MQTT_KEEPALIVE_INTERVAL = creds['keep_alive'] MQTT_USER = creds['user'] MQTT_PW = creds['password'] client_id = camera_config['camera_id'] QOS = creds['qos'] # quality of service def create_timestamp(): now = datetime.now() date_time = now.strftime("%Y%m%d_%H%M%S") #print(date_time)
def update_numbering(filename): '''Update global numbering from saved data.''' global NUMBERING saved = utils.read_yaml(filename) for entry in saved['entries']: NUMBERING[entry['slug']] = entry['kind']
######################################################################################################## if __name__ == '__main__': parser = argparse.ArgumentParser( description="Fine-tune a BERT model for a specific NLP task.") parser.add_argument( '--yaml_file', type=str, help='''Path to the yaml file containing additional information on how the dataset is structured.''' ) args = parser.parse_args() # Fetch parameters parameters = read_yaml(args.yaml_file) check_folder(parameters['output_dir']) save_yaml(parameters, os.path.join(parameters['output_dir'], 'config.yml')) logging.basicConfig(filename=os.path.join(parameters['output_dir'], parameters['log_file']), filemode='w+', level=logging.INFO) logging.info("Parameters fetched.") logging.info("Setting seed for reproductibility...") set_seed(parameters['seed']) logging.info("\tDone.") logging.info("Set and retrieve the device on which to run...") device = get_device() task = parameters['task'].lower()
def test(args): tf.reset_default_graph() model_path = args.CBSDeep_model_folder + '//' + args.denoise_model # print(list(os.walk(model_path, topdown=False))[-1]) # print(list(os.walk(model_path, topdown=False))[-1][-1][0]) # print(list(os.walk(model_path, topdown=False))[-1][-2]) model_list = list(os.walk(model_path, topdown=False))[-1][-2] yaml_name = list(os.walk(model_path, topdown=False))[-1][-1][0] print(yaml_name) read_yaml(args, model_path + '//' + yaml_name) print('hhhhh ----->', args) name_list, noise_img, coordinate_list = test_preprocess_lessMemory(args) num_h = (math.floor((noise_img.shape[1] - args.img_h) / args.gap_h) + 1) num_w = (math.floor((noise_img.shape[2] - args.img_w) / args.gap_w) + 1) num_s = (math.floor((noise_img.shape[0] - args.img_s) / args.gap_s) + 1) TIME = args.datasets_folder + '_' + args.denoise_model #+'_'+args.model_name #+'_'+datetime.datetime.now().strftime("%Y%m%d-%H%M") results_path = args.results_folder + '//' + 'unet3d_' + TIME + '//' if not os.path.exists(args.results_folder): os.mkdir(args.results_folder) if not os.path.exists(results_path): os.mkdir(results_path) model_name = args.model_name print('model_name -----> ', model_name) output_path = results_path + '//' + model_name if not os.path.exists(output_path): os.mkdir(output_path) output_graph_path = args.CBSDeep_model_folder + '//' + args.denoise_model + '//' + model_name + '//' print('output_graph_path -----> ', output_graph_path) start_time = time.time() sess = tf.Session() with sess.as_default(): # sess.run(tf.global_variables_initializer()) meta_graph_def = tf.saved_model.loader.load(sess, ['3D_N2N'], output_graph_path) signature = meta_graph_def.signature_def in_tensor_name = signature['my_signature'].inputs['input0'].name out_tensor_name = signature['my_signature'].outputs['output0'].name input = sess.graph.get_tensor_by_name(in_tensor_name) output = sess.graph.get_tensor_by_name(out_tensor_name) # sess.run(tf.global_variables_initializer()) ''' variable_names = [v.name for v in tf.trainable_variables()] values = sess.run(variable_names) for k,v in zip(variable_names, values): if len(v.shape)==5: print("Variable: ", k, "Shape: ", v.shape,"value: ",v[0][0][0][0][0]) if len(v.shape)==1: print("Variable: ", k, "Shape: ", v.shape,"value: ",v[0]) ''' denoise_img = np.zeros(noise_img.shape) input_img = np.zeros(noise_img.shape) for index in range(len(name_list)): input_name = name_list[index] single_coordinate = coordinate_list[name_list[index]] init_h = single_coordinate['init_h'] end_h = single_coordinate['end_h'] init_w = single_coordinate['init_w'] end_w = single_coordinate['end_w'] init_s = single_coordinate['init_s'] end_s = single_coordinate['end_s'] noise_patch1 = noise_img[init_s:end_s, init_h:end_h, init_w:end_w] train_input = np.expand_dims( np.expand_dims(noise_patch1.transpose(1, 2, 0), 3), 0) # print('train_input -----> ',train_input.shape) data_name = name_list[index] train_output = sess.run(output, feed_dict={input: train_input}) train_input = np.squeeze(train_input).transpose(2, 0, 1) train_output = np.squeeze(train_output).transpose(2, 0, 1) stack_start_w ,stack_end_w ,patch_start_w ,patch_end_w ,\ stack_start_h ,stack_end_h ,patch_start_h ,patch_end_h ,\ stack_start_s ,stack_end_s ,patch_start_s ,patch_end_s = name2index(args, input_name, num_h, num_w, num_s) aaaa = train_output[patch_start_s:patch_end_s, patch_start_h:patch_end_h, patch_start_w:patch_end_w] bbbb = train_input[patch_start_s:patch_end_s, patch_start_h:patch_end_h, patch_start_w:patch_end_w] denoise_img[stack_start_s:stack_end_s, stack_start_w:stack_end_w, stack_start_h:stack_end_h] \ = train_output[patch_start_s:patch_end_s, patch_start_w:patch_end_w, patch_start_h:patch_end_h]*(np.sum(bbbb)/np.sum(aaaa))**0.5 input_img[stack_start_s:stack_end_s, stack_start_w:stack_end_w, stack_start_h:stack_end_h] \ = train_input[patch_start_s:patch_end_s, patch_start_w:patch_end_w, patch_start_h:patch_end_h] # print('output_img shape -----> ',output_img.shape) ''' output_img = denoise_img.squeeze().astype(np.float32)*args.normalize_factor output_img = np.clip(output_img, 0, 65535).astype('uint16') result_name = results_path+str(i)+'_'+str(index)+'_'+data_name+'_output.tif' io.imsave(result_name, output_img.transpose(2,0,1)) ''' # if index % 100 == 0: # print('denoise_img ---> ',denoise_img.max(),'---> ',denoise_img.min()) # print('input_img ---> ',input_img.max(),'---> ',input_img.min()) output_img = denoise_img.squeeze().astype( np.float32) * args.normalize_factor output_img = output_img - output_img.min() output_img = output_img / output_img.max() * 65535 output_img = np.clip(output_img, 0, 65535).astype('uint16') output_img = output_img - output_img.min() input_img = input_img.squeeze().astype( np.float32) * args.normalize_factor input_img = np.clip(input_img, 0, 65535).astype('uint16') result_name = output_path + '//' + 'output_' + model_name + '.tif' input_name = output_path + '//' + 'input_' + model_name + '.tif' io.imsave(result_name, output_img) io.imsave(input_name, input_img)
if args.y: args.ymin, args.ymax, args.ynum = [ float(a) for a in args.y.split(':') ] assert args.ymin and args.ymax and args.ynum, \ 'You specified some arguments for the y axis, but not all' except: raise Exception( 'Improper format for x- or y-coordinates. Try something like -1:1:51' ) #-------------------------------------------------------------------------- # Load models and extract parameters #-------------------------------------------------------------------------- # net = model_loader.load(args.dataset, args.model, args.model_file) config = read_yaml(args.model_config) config['use_cuda'] = True config['network_type'] = args.model config['dataset']['artificial_and'] = config['artificial_and'] dataloader = ShapeWorldLoader(config=config['dataset']) config['vocab'] = dataloader.vocab trainer = Trainer(config) trainer.model.load_model(args.model_file) net = trainer.model net.cpu() w = net_plotter.get_weights(net) # initial parameters s = copy.deepcopy( net.state_dict()) # deepcopy since state_dict are references if args.ngpu > 1:
default=False, help='Batch mode') optparser.add_argument('-c', dest='clnameglob', metavar='classifier', default=None, help='Only plot matching classifiers (globs allowed)') optparser.add_argument('--rarity', action='store_true', default=False, help='Plot rarity distributions') optparser.add_argument('--probab', action='store_true', default=False, help='Plot classifier probability distributions') options = optparser.parse_args() locals().update(_import_args(options)) from pprint import pprint import sys from utils import read_yaml, get_rpaths conf = read_yaml(yamlfile) if isinstance(conf, list): for entry in conf: print entry['file'] if entry['file'] != 'TMVA.root': continue rfiles = get_rpaths(files, entry) else: if conf['file'] == 'TMVA.root': rfiles = get_rpaths(files, conf) if not rfiles: sys.exit('Config parsing error.') from rplot.rdir import Rdir fnames = [ rfile[0]['file'] for rfile in rfiles ] rpath_tool = Rdir(fnames)
def execute(cls, unmatched_products_df, product_df): config_file = utils.read_yaml('config.yaml') precision = config_file['matching_text']['precision'] text_match_products_df = cls.match_by_text(unmatched_products_df, product_df, precision) return text_match_products_df
#!/usr/bin/env python3 import argparse import os import shutil import sys import utils import yaml from importlib import import_module from multiprocessing import Queue from os.path import join as pjoin from time import sleep # module alias (case-insensitive): base filename cgui_modules = utils.read_yaml('modules.yml') def log_exception(case_info, step_num, exc_info): global LOGFILE templ = 'Job "{}" ({}) encountered an exception on step {}:\n{}\n' if not 'jobid' in case_info: case_info['jobid'] = '-1' jobid = case_info['jobid'] label = case_info['label'] with open(LOGFILE, 'a') as fh: label = case_info['label'] fh.write(templ.format(label, jobid, step_num, exc_info)) def log_failure(case_info, step, elapsed_time): global LOGFILE templ = 'Job "{}" ({}) failed on step {} after {:.2f} seconds\n'
def convert(args): """ Converts files into JSON format for BRITS models :param yaml_file: configuration file with the following information - data_folder: path to folder storing data files to be converted - file_extension: the file extension of the data files to be converted - only the file types in SUPPORTED_FILE_TYPES (see constants at top) are supported - file_regex: regex to specify file names to be converted - if file_regex is None, then method will convert all files of type file_extension - train_split: proportion of files to use as training set - if None, then JSON data will be saved into one file - otherwise, will be saved into two files, one for train and one for test - val_split: proportion of training sequences to use as validation set - if None, then no splitting will be done - otherwise, validation data will saved into separate file - output_folder: path to folder to save JSON data - output_file_name: file name to save JSON output file - if train_split is not None, then JSON data will be saved into two files - train set will be prefixed by "train_", test set prefixed by "test_" - max_len: maximum sequence length - if sequence is longer than max_len, gets split - can be None for no splitting - seed: random seed - current randomness comes from random train/test split and random removal of values for imputation :param validate_after: boolean determining whether to run validation at the end - validation can always be run from the command line :param info_after: boolean determining whether to print info about JSON data - info can always be run from the command line """ print_banner() print("RUNNING CONVERSION\n") yaml_data = read_yaml(args["--yaml-file"]) data_folder = yaml_data["data_folder"] file_extension = yaml_data["file_extension"] file_regex = yaml_data["file_regex"] output_file_name = yaml_data["output_file_name"] output_folder = yaml_data["output_folder"] train_split = yaml_data["train_split"] val_split = yaml_data["val_split"] max_len = yaml_data["max_len"] seed = yaml_data["seed"] if file_extension not in SUPPORTED_FILE_TYPES: print(".{} is not currently a supported file type") return if seed is not None: print("Seeding random generator with seed {}".format(seed)) np.random.seed(seed) else: print("WARNING: The random number generator has not been seeded.") print( "You are encouraged to run again with a random seed for reproducibility!" ) file_regex = file_regex if file_regex is not None else ".*" file_regex += r"\." + file_extension json_data = extract_json(data_folder, file_regex, file_extension, max_len) output_files = [] train_data = json_data test_data = None val_data = None if train_split is not None: train_data, test_data = random_split(train_data, train_split) save_path = save_json(test_data, output_folder, "test_" + output_file_name) output_files.append(save_path) if val_split is not None: train_data, val_data = random_split(train_data, 1 - val_split) save_path = save_json(val_data, output_folder, "val_" + output_file_name) output_files.append(save_path) print("Using {} sequences for training".format(len(train_data))) save_path = save_json(train_data, output_folder, "train_" + output_file_name) output_files.append(save_path) print_banner() if args["--validate"]: # run validation on newly saved JSON for output_file in output_files: args["--json-file"] = output_file validate(args) if args["--info"]: # print info about newly saved JSON for output_file in output_files: print_info(output_file)