def run(experiment_id, restore_path, config_file, bit, unquant_layers): if config_file is None and experiment_id is None: raise Exception("config_file or experiment_id are required") if experiment_id: environment.init(experiment_id) config = config_util.load_from_experiment() if config_file: config = config_util.merge(config, config_util.load(config_file)) if restore_path is None: restore_file = executor.search_restore_filename(environment.CHECKPOINTS_DIR) restore_path = os.path.join(environment.CHECKPOINTS_DIR, restore_file) if not os.path.exists("{}.index".format(restore_path)): raise Exception("restore file {} dont exists.".format(restore_path)) else: experiment_id = "profile" environment.init(experiment_id) config = config_util.load(config_file) config.BATCH_SIZE = 1 config.NETWORK.BATCH_SIZE = 1 config.DATASET.BATCH_SIZE = 1 executor.init_logging(config) config_util.display(config) _profile(config, restore_path, bit, unquant_layers)
def run(experiment_id, restore_path=None, image_size=(None, None), image=DEFAULT_INFERENCE_TEST_DATA_IMAGE, config_file=None): environment.init(experiment_id) config = config_util.load_from_experiment() if config_file: config = config_util.merge(config, config_util.load(config_file)) config.BATCH_SIZE = 1 config.NETWORK.BATCH_SIZE = 1 config.DATASET.BATCH_SIZE = 1 if list(image_size) != [None, None]: config.IMAGE_SIZE = list(image_size) config.NETWORK.IMAGE_SIZE = list(image_size) # override pre processes image size. if config.PRE_PROCESSOR: config.PRE_PROCESSOR.set_image_size(image_size) # override post processes image size. if config.POST_PROCESSOR: config.POST_PROCESSOR.set_image_size(image_size) print("Override IMAGE_SIZE", config.IMAGE_SIZE) executor.init_logging(config) config_util.display(config) return _export(config, restore_path, image)
def _setup(self, config): self.lm_config = config_util.load(self.config['lm_config']) executor.init_logging(self.lm_config) model_class = self.lm_config.NETWORK_CLASS network_kwargs = {key.lower(): val for key, val in self.lm_config.NETWORK.items()} network_kwargs = update_parameters_for_each_trial(network_kwargs, self.config) # No distributed training was implemented, therefore rank set to 0 self.train_dataset = setup_dataset(self.lm_config, "train", 0) self.validation_dataset = setup_dataset(self.lm_config, "validation", 0) if model_class.__module__.startswith("blueoil.networks.object_detection"): model = model_class( classes=self.train_dataset.classes, num_max_boxes=self.train_dataset.num_max_boxes, is_debug=self.lm_config.IS_DEBUG, **network_kwargs, ) elif model_class.__module__.startswith("blueoil.networks.segmentation"): model = model_class( classes=self.train_dataset.classes, label_colors=self.train_dataset.label_colors, is_debug=self.lm_config.IS_DEBUG, **network_kwargs, ) else: model = model_class( classes=self.train_dataset.classes, is_debug=self.lm_config.IS_DEBUG, **network_kwargs, ) self.is_training_placeholder = tf.compat.v1.placeholder(tf.bool, name="is_training_placeholder") self.images_placeholder, self.labels_placeholder = model.placeholders() output = model.inference(self.images_placeholder, self.is_training_placeholder) if model_class.__module__.startswith("blueoil.networks.object_detection"): loss = model.loss(output, self.labels_placeholder, self.is_training_placeholder) else: loss = model.loss(output, self.labels_placeholder) opt = model.optimizer() train_op = model.train(loss, opt) metrics_ops_dict, metrics_update_op = model.metrics(output, self.labels_placeholder) self.train_op = train_op self.metrics_ops_dict = metrics_ops_dict self.metrics_update_op = metrics_update_op init_op = tf.compat.v1.global_variables_initializer() self.reset_metrics_op = tf.compat.v1.local_variables_initializer() session_config = tf.compat.v1.ConfigProto( gpu_options=tf.compat.v1.GPUOptions(allow_growth=True)) self.sess = tf.compat.v1.Session(config=session_config) self.sess.run([init_op, self.reset_metrics_op]) self.iterations = 0 self.saver = tf.compat.v1.train.Saver()
def main(config_file, experiment_id, restore_path, output_dir): environment.init(experiment_id) config = config_util.load_from_experiment() if config_file: config = config_util.merge(config, config_util.load(config_file)) executor.init_logging(config) config_util.display(config) evaluate(config, restore_path, output_dir)
def run(config_file, experiment_id, recreate): environment.init(experiment_id) config = config_util.load(config_file) if horovod_util.is_enabled(): horovod_util.setup() if horovod_util.is_rank0(): config_util.display(config) executor.init_logging(config) executor.prepare_dirs(recreate) config_util.copy_to_experiment_dir(config_file) config_util.save_yaml(environment.EXPERIMENT_DIR, config) start_training(config)
def main(network, dataset, config_file, experiment_id, restore_path, output_dir): environment.init(experiment_id) config = config_util.load_from_experiment() if config_file: config = config_util.merge(config, config_util.load(config_file)) if network: network_class = module_loader.load_network_class(network) config.NETWORK_CLASS = network_class if dataset: dataset_class = module_loader.load_dataset_class(dataset) config.DATASET_CLASS = dataset_class executor.init_logging(config) config_util.display(config) evaluate(config, restore_path, output_dir)
def run(network, dataset, config_file, experiment_id, recreate): environment.init(experiment_id) config = config_util.load(config_file) if network: network_class = module_loader.load_network_class(network) config.NETWORK_CLASS = network_class if dataset: dataset_class = module_loader.load_dataset_class(dataset) config.DATASET_CLASS = dataset_class if horovod_util.is_enabled(): horovod_util.setup() if horovod_util.is_rank0(): config_util.display(config) executor.init_logging(config) executor.prepare_dirs(recreate) config_util.copy_to_experiment_dir(config_file) config_util.save_yaml(environment.EXPERIMENT_DIR, config) start_training(config)
def _run(config_file, experiment_id, restore_path, image_size, step_size, cpu): if experiment_id: environment.init(experiment_id) config = config_util.load_from_experiment() if config_file: config = config_util.merge(config, config_util.load(config_file)) if restore_path is None: restore_file = executor.search_restore_filename( environment.CHECKPOINTS_DIR) restore_path = os.path.join(environment.CHECKPOINTS_DIR, restore_file) if not os.path.exists("{}.index".format(restore_path)): raise Exception( "restore file {} dont exists.".format(restore_path)) else: experiment_id = "measure_latency" environment.init(experiment_id) config = config_util.load(config_file) config.BATCH_SIZE = 1 config.NETWORK.BATCH_SIZE = 1 config.DATASET.BATCH_SIZE = 1 if list(image_size) != [None, None]: config.IMAGE_SIZE = list(image_size) config.NETWORK.IMAGE_SIZE = list(image_size) # override pre processes image size. if config.PRE_PROCESSOR: config.PRE_PROCESSOR.set_image_size(image_size) # override post processes image size. if config.POST_PROCESSOR: config.POST_PROCESSOR.set_image_size(image_size) print("Override IMAGE_SIZE", config.IMAGE_SIZE) executor.init_logging(config) config_util.display(config) overall_times, only_network_times = _measure_time(config, restore_path, step_size) overall_times = np.array(overall_times) only_network_times = np.array(only_network_times) # list of physical_device_desc devices = [ device.physical_device_desc for device in device_lib.list_local_devices() if device.physical_device_desc ] message = """ ---- measure latency result ---- total number of execution (number of samples): {} network: {} use gpu by network: {} image size: {} devices: {} * overall (include pre-post-process which execute on cpu) total time: {:.4f} msec latency mean (SD=standard deviation): {:.4f} (SD={:.4f}) msec, min: {:.4f} msec, max: {:.4f} msec FPS mean (SD=standard deviation): {:.4f} (SD={:.4f}), min: {:.4f}, max: {:.4f} * network only (exclude pre-post-process): total time: {:.4f} msec latency mean (SD=standard deviation): {:.4f} (SD={:.4f}) msec, min: {:.4f} msec, max: {:.4f} msec FPS mean (SD=standard deviation): {:.4f} (SD={:.4f}), min: {:.4f}, max: {:.4f} ---- measure latency result ---- """.format( step_size, config.NETWORK_CLASS.__name__, not cpu, config.IMAGE_SIZE, devices, # overall np.sum(overall_times) * 1000, # latency np.mean(overall_times) * 1000, np.std(overall_times) * 1000, np.min(overall_times) * 1000, np.max(overall_times) * 1000, # FPS np.mean(1 / overall_times), np.std(1 / overall_times), np.min(1 / overall_times), np.max(1 / overall_times), # network only np.sum(only_network_times) * 1000, # latency np.mean(only_network_times) * 1000, np.std(only_network_times) * 1000, np.min(only_network_times) * 1000, np.max(only_network_times) * 1000, # FPS np.mean(1 / only_network_times), np.std(1 / only_network_times), np.min(1 / only_network_times), np.max(1 / only_network_times), ) print(message)