def main(network, dataset, config_file, experiment_id, restore_path, output_dir): environment.init(experiment_id) config = config_util.load_from_experiment() if config_file: config = config_util.merge(config, config_util.load(config_file)) if network: network_class = module_loader.load_network_class(network) config.NETWORK_CLASS = network_class if dataset: dataset_class = module_loader.load_dataset_class(dataset) config.DATASET_CLASS = dataset_class executor.init_logging(config) config_util.display(config) evaluate(config, restore_path, output_dir)
def main(model): if model == "yolov2": weight_file = 'inputs/yolo-voc.weights' experiment_id = "convert_weight_from_darknet/yolo_v2" config_file = "configs/convert_weight_from_darknet/yolo_v2.py" if model == "darknet19": weight_file = 'inputs/darknet19_448.weights' experiment_id = "convert_weight_from_darknet/darknet19" config_file = "configs/convert_weight_from_darknet/darknet19.py" recreate = True environment.init(experiment_id) executor.prepare_dirs(recreate) config = config_util.load(config_file) config_util.display(config) config_util.copy_to_experiment_dir(config_file) convert(config, weight_file)
def run(input_dir, output_dir, experiment_id, config_file, restore_path, save_images): environment.init(experiment_id) config = config_util.load_from_experiment() if config_file: config = config_util.merge(config, config_util.load(config_file)) if not os.path.isdir(input_dir): raise Exception("Input directory {} does not exist.".format(input_dir)) if restore_path is None: restore_file = search_restore_filename(environment.CHECKPOINTS_DIR) restore_path = os.path.join(environment.CHECKPOINTS_DIR, restore_file) print("Restore from {}".format(restore_path)) if not os.path.exists("{}.index".format(restore_path)): raise Exception("restore file {} dont exists.".format(restore_path)) print("---- start predict ----") _run(input_dir, output_dir, config, restore_path, save_images) print("---- end predict ----")
def run(network, dataset, config_file, experiment_id, recreate): environment.init(experiment_id) config = config_util.load(config_file) if network: network_class = module_loader.load_network_class(network) config.NETWORK_CLASS = network_class if dataset: dataset_class = module_loader.load_dataset_class(dataset) config.DATASET_CLASS = dataset_class if horovod_util.is_enabled(): horovod_util.setup() if horovod_util.is_rank0(): config_util.display(config) executor.init_logging(config) executor.prepare_dirs(recreate) config_util.copy_to_experiment_dir(config_file) config_util.save_yaml(environment.EXPERIMENT_DIR, config) start_training(config)
def run(config_file, tunable_id, local_dir): register_trainable(tunable_id, TrainTunable) lm_config = config_util.load(config_file) def smartdict_to_dict(config): if isinstance(config, SmartDict): config = dict(config) for key, value in config.items(): if isinstance(value, SmartDict): value = dict(value) smartdict_to_dict(value) config[key] = value return config tune_space = smartdict_to_dict(lm_config['TUNE_SPACE']) tune_spec = smartdict_to_dict(lm_config['TUNE_SPEC']) tune_spec['run'] = tunable_id tune_spec['config'] = {'lm_config': os.path.join(os.getcwd(), config_file)} tune_spec['local_dir'] = local_dir tune_spec['trial_name_creator'] = ray.tune.function(trial_str_creator) # Expecting use of gpus to do parameter search ray.init(num_cpus=multiprocessing.cpu_count() // 2, num_gpus=max(get_num_gpu(), 1)) algo = HyperOptSearch(tune_space, max_concurrent=4, reward_attr="mean_accuracy") scheduler = AsyncHyperBandScheduler(time_attr="training_iteration", reward_attr="mean_accuracy", max_t=200) trials = run_experiments(experiments={'exp_tune': tune_spec}, search_alg=algo, scheduler=scheduler) print("The best result is", get_best_result(trials, metric="mean_accuracy", param='config'))
def _setup(self, config): self.lm_config = config_util.load(self.config['lm_config']) executor.init_logging(self.lm_config) model_class = self.lm_config.NETWORK_CLASS network_kwargs = { key.lower(): val for key, val in self.lm_config.NETWORK.items() } network_kwargs = update_parameters_for_each_trial( network_kwargs, self.config) # No distributed training was implemented, therefore rank set to 0 self.train_dataset = setup_dataset(self.lm_config, "train", 0) self.validation_dataset = setup_dataset(self.lm_config, "validation", 0) if model_class.__module__.startswith( "blueoil.networks.object_detection"): model = model_class( classes=self.train_dataset.classes, num_max_boxes=self.train_dataset.num_max_boxes, is_debug=self.lm_config.IS_DEBUG, **network_kwargs, ) elif model_class.__module__.startswith( "blueoil.networks.segmentation"): model = model_class( classes=self.train_dataset.classes, label_colors=self.train_dataset.label_colors, is_debug=self.lm_config.IS_DEBUG, **network_kwargs, ) else: model = model_class( classes=self.train_dataset.classes, is_debug=self.lm_config.IS_DEBUG, **network_kwargs, ) self.is_training_placeholder = tf.compat.v1.placeholder( tf.bool, name="is_training_placeholder") self.images_placeholder, self.labels_placeholder = model.placeholders() output = model.inference(self.images_placeholder, self.is_training_placeholder) if model_class.__module__.startswith( "blueoil.networks.object_detection"): loss = model.loss(output, self.labels_placeholder, self.is_training_placeholder) else: loss = model.loss(output, self.labels_placeholder) opt = model.optimizer() train_op = model.train(loss, opt) metrics_ops_dict, metrics_update_op = model.metrics( output, self.labels_placeholder) self.train_op = train_op self.metrics_ops_dict = metrics_ops_dict self.metrics_update_op = metrics_update_op init_op = tf.compat.v1.global_variables_initializer() self.reset_metrics_op = tf.compat.v1.local_variables_initializer() session_config = tf.compat.v1.ConfigProto( gpu_options=tf.compat.v1.GPUOptions(allow_growth=True)) self.sess = tf.compat.v1.Session(config=session_config) self.sess.run([init_op, self.reset_metrics_op]) self.iterations = 0 self.saver = tf.compat.v1.train.Saver()
def run_all_steps(dirs, config_file): """ Test of the following steps. - Train using given config. - Convert using training result. - Predict using training result. """ config_path = os.path.join(dirs["config_dir"], config_file) config = load(config_path) # Train experiment_id, checkpoint_name = train(config_path) train_output_dir = os.path.join(dirs["train_output_dir"], experiment_id) assert os.path.exists(os.path.join(train_output_dir, 'checkpoints')) # Convert convert(experiment_id) convert_output_dir = os.path.join(train_output_dir, 'export', checkpoint_name) lib_dir = os.path.join( convert_output_dir, "{}x{}".format(config.IMAGE_SIZE[0], config.IMAGE_SIZE[1]), 'output', 'models', 'lib', ) assert os.path.exists(os.path.join(lib_dir, 'lm_aarch64.elf')) assert os.path.exists(os.path.join(lib_dir, 'lm_aarch64_fpga.elf')) assert os.path.exists(os.path.join(lib_dir, 'lm_arm.elf')) assert os.path.exists(os.path.join(lib_dir, 'lm_arm_fpga.elf')) assert os.path.exists(os.path.join(lib_dir, 'lm_x86.elf')) assert os.path.exists(os.path.join(lib_dir, 'lm_x86_avx.elf')) assert os.path.exists(os.path.join(lib_dir, 'libdlk_aarch64.so')) assert os.path.exists(os.path.join(lib_dir, 'libdlk_aarch64_fpga.so')) assert os.path.exists(os.path.join(lib_dir, 'libdlk_arm.so')) assert os.path.exists(os.path.join(lib_dir, 'libdlk_arm_fpga.so')) assert os.path.exists(os.path.join(lib_dir, 'libdlk_x86.so')) assert os.path.exists(os.path.join(lib_dir, 'libdlk_x86_avx.so')) assert os.path.exists(os.path.join(lib_dir, 'libdlk_aarch64.a')) assert os.path.exists(os.path.join(lib_dir, 'libdlk_aarch64_fpga.a')) assert os.path.exists(os.path.join(lib_dir, 'libdlk_arm.a')) assert os.path.exists(os.path.join(lib_dir, 'libdlk_arm_fpga.a')) assert os.path.exists(os.path.join(lib_dir, 'libdlk_x86.a')) assert os.path.exists(os.path.join(lib_dir, 'libdlk_x86_avx.a')) # Predict predict_input_dir = os.path.join(dirs["blueoil_dir"], "tests/unit/fixtures/sample_images") predict_output_dir = dirs["predict_output_dir"] predict(predict_input_dir, predict_output_dir, experiment_id, checkpoint=checkpoint_name) assert os.path.exists(os.path.join(predict_output_dir, 'images')) assert os.path.exists(os.path.join(predict_output_dir, 'json', '0.json')) assert os.path.exists(os.path.join(predict_output_dir, 'npy', '0.npy'))
def _run(config_file, experiment_id, restore_path, image_size, step_size, cpu): if experiment_id: environment.init(experiment_id) config = config_util.load_from_experiment() if config_file: config = config_util.merge(config, config_util.load(config_file)) if restore_path is None: restore_file = executor.search_restore_filename( environment.CHECKPOINTS_DIR) restore_path = os.path.join(environment.CHECKPOINTS_DIR, restore_file) if not os.path.exists("{}.index".format(restore_path)): raise Exception( "restore file {} dont exists.".format(restore_path)) else: experiment_id = "measure_latency" environment.init(experiment_id) config = config_util.load(config_file) config.BATCH_SIZE = 1 config.NETWORK.BATCH_SIZE = 1 config.DATASET.BATCH_SIZE = 1 if list(image_size) != [None, None]: config.IMAGE_SIZE = list(image_size) config.NETWORK.IMAGE_SIZE = list(image_size) # override pre processes image size. if config.PRE_PROCESSOR: config.PRE_PROCESSOR.set_image_size(image_size) # override post processes image size. if config.POST_PROCESSOR: config.POST_PROCESSOR.set_image_size(image_size) print("Override IMAGE_SIZE", config.IMAGE_SIZE) executor.init_logging(config) config_util.display(config) overall_times, only_network_times = _measure_time(config, restore_path, step_size) overall_times = np.array(overall_times) only_network_times = np.array(only_network_times) # list of physical_device_desc devices = [ device.physical_device_desc for device in device_lib.list_local_devices() if device.physical_device_desc ] message = """ ---- measure latency result ---- total number of execution (number of samples): {} network: {} use gpu by network: {} image size: {} devices: {} * overall (include pre-post-process which execute on cpu) total time: {:.4f} msec latency mean (SD=standard deviation): {:.4f} (SD={:.4f}) msec, min: {:.4f} msec, max: {:.4f} msec FPS mean (SD=standard deviation): {:.4f} (SD={:.4f}), min: {:.4f}, max: {:.4f} * network only (exclude pre-post-process): total time: {:.4f} msec latency mean (SD=standard deviation): {:.4f} (SD={:.4f}) msec, min: {:.4f} msec, max: {:.4f} msec FPS mean (SD=standard deviation): {:.4f} (SD={:.4f}), min: {:.4f}, max: {:.4f} ---- measure latency result ---- """.format( step_size, config.NETWORK_CLASS.__name__, not cpu, config.IMAGE_SIZE, devices, # overall np.sum(overall_times) * 1000, # latency np.mean(overall_times) * 1000, np.std(overall_times) * 1000, np.min(overall_times) * 1000, np.max(overall_times) * 1000, # FPS np.mean(1 / overall_times), np.std(1 / overall_times), np.min(1 / overall_times), np.max(1 / overall_times), # network only np.sum(only_network_times) * 1000, # latency np.mean(only_network_times) * 1000, np.std(only_network_times) * 1000, np.min(only_network_times) * 1000, np.max(only_network_times) * 1000, # FPS np.mean(1 / only_network_times), np.std(1 / only_network_times), np.min(1 / only_network_times), np.max(1 / only_network_times), ) print(message)