def main(network, dataset, config_file, experiment_id, restore_path,
output_dir):
environment.init(experiment_id)
config = config_util.load_from_experiment()
if config_file:
config = config_util.merge(config, config_util.load(config_file))
if network:
network_class = module_loader.load_network_class(network)
config.NETWORK_CLASS = network_class
if dataset:
dataset_class = module_loader.load_dataset_class(dataset)
config.DATASET_CLASS = dataset_class
executor.init_logging(config)
config_util.display(config)
evaluate(config, restore_path, output_dir)
def main(model):
if model == "yolov2":
weight_file = 'inputs/yolo-voc.weights'
experiment_id = "convert_weight_from_darknet/yolo_v2"
config_file = "configs/convert_weight_from_darknet/yolo_v2.py"
if model == "darknet19":
weight_file = 'inputs/darknet19_448.weights'
experiment_id = "convert_weight_from_darknet/darknet19"
config_file = "configs/convert_weight_from_darknet/darknet19.py"
recreate = True
environment.init(experiment_id)
executor.prepare_dirs(recreate)
config = config_util.load(config_file)
config_util.display(config)
config_util.copy_to_experiment_dir(config_file)
convert(config, weight_file)
def run(input_dir, output_dir, experiment_id, config_file, restore_path, save_images):
environment.init(experiment_id)
config = config_util.load_from_experiment()
if config_file:
config = config_util.merge(config, config_util.load(config_file))
if not os.path.isdir(input_dir):
raise Exception("Input directory {} does not exist.".format(input_dir))
if restore_path is None:
restore_file = search_restore_filename(environment.CHECKPOINTS_DIR)
restore_path = os.path.join(environment.CHECKPOINTS_DIR, restore_file)
print("Restore from {}".format(restore_path))
if not os.path.exists("{}.index".format(restore_path)):
raise Exception("restore file {} dont exists.".format(restore_path))
print("---- start predict ----")
_run(input_dir, output_dir, config, restore_path, save_images)
print("---- end predict ----")
def run(network, dataset, config_file, experiment_id, recreate):
environment.init(experiment_id)
config = config_util.load(config_file)
if network:
network_class = module_loader.load_network_class(network)
config.NETWORK_CLASS = network_class
if dataset:
dataset_class = module_loader.load_dataset_class(dataset)
config.DATASET_CLASS = dataset_class
if horovod_util.is_enabled():
horovod_util.setup()
if horovod_util.is_rank0():
config_util.display(config)
executor.init_logging(config)
executor.prepare_dirs(recreate)
config_util.copy_to_experiment_dir(config_file)
config_util.save_yaml(environment.EXPERIMENT_DIR, config)
start_training(config)
def run(config_file, tunable_id, local_dir):
register_trainable(tunable_id, TrainTunable)
lm_config = config_util.load(config_file)
def smartdict_to_dict(config):
if isinstance(config, SmartDict):
config = dict(config)
for key, value in config.items():
if isinstance(value, SmartDict):
value = dict(value)
smartdict_to_dict(value)
config[key] = value
return config
tune_space = smartdict_to_dict(lm_config['TUNE_SPACE'])
tune_spec = smartdict_to_dict(lm_config['TUNE_SPEC'])
tune_spec['run'] = tunable_id
tune_spec['config'] = {'lm_config': os.path.join(os.getcwd(), config_file)}
tune_spec['local_dir'] = local_dir
tune_spec['trial_name_creator'] = ray.tune.function(trial_str_creator)
# Expecting use of gpus to do parameter search
ray.init(num_cpus=multiprocessing.cpu_count() // 2,
num_gpus=max(get_num_gpu(), 1))
algo = HyperOptSearch(tune_space,
max_concurrent=4,
reward_attr="mean_accuracy")
scheduler = AsyncHyperBandScheduler(time_attr="training_iteration",
reward_attr="mean_accuracy",
max_t=200)
trials = run_experiments(experiments={'exp_tune': tune_spec},
search_alg=algo,
scheduler=scheduler)
print("The best result is",
get_best_result(trials, metric="mean_accuracy", param='config'))
def _setup(self, config):
self.lm_config = config_util.load(self.config['lm_config'])
executor.init_logging(self.lm_config)
model_class = self.lm_config.NETWORK_CLASS
network_kwargs = {
key.lower(): val
for key, val in self.lm_config.NETWORK.items()
}
network_kwargs = update_parameters_for_each_trial(
network_kwargs, self.config)
# No distributed training was implemented, therefore rank set to 0
self.train_dataset = setup_dataset(self.lm_config, "train", 0)
self.validation_dataset = setup_dataset(self.lm_config, "validation",
0)
if model_class.__module__.startswith(
"blueoil.networks.object_detection"):
model = model_class(
classes=self.train_dataset.classes,
num_max_boxes=self.train_dataset.num_max_boxes,
is_debug=self.lm_config.IS_DEBUG,
**network_kwargs,
)
elif model_class.__module__.startswith(
"blueoil.networks.segmentation"):
model = model_class(
classes=self.train_dataset.classes,
label_colors=self.train_dataset.label_colors,
is_debug=self.lm_config.IS_DEBUG,
**network_kwargs,
)
else:
model = model_class(
classes=self.train_dataset.classes,
is_debug=self.lm_config.IS_DEBUG,
**network_kwargs,
)
self.is_training_placeholder = tf.compat.v1.placeholder(
tf.bool, name="is_training_placeholder")
self.images_placeholder, self.labels_placeholder = model.placeholders()
output = model.inference(self.images_placeholder,
self.is_training_placeholder)
if model_class.__module__.startswith(
"blueoil.networks.object_detection"):
loss = model.loss(output, self.labels_placeholder,
self.is_training_placeholder)
else:
loss = model.loss(output, self.labels_placeholder)
opt = model.optimizer()
train_op = model.train(loss, opt)
metrics_ops_dict, metrics_update_op = model.metrics(
output, self.labels_placeholder)
self.train_op = train_op
self.metrics_ops_dict = metrics_ops_dict
self.metrics_update_op = metrics_update_op
init_op = tf.compat.v1.global_variables_initializer()
self.reset_metrics_op = tf.compat.v1.local_variables_initializer()
session_config = tf.compat.v1.ConfigProto(
gpu_options=tf.compat.v1.GPUOptions(allow_growth=True))
self.sess = tf.compat.v1.Session(config=session_config)
self.sess.run([init_op, self.reset_metrics_op])
self.iterations = 0
self.saver = tf.compat.v1.train.Saver()
def run_all_steps(dirs, config_file):
"""
Test of the following steps.
- Train using given config.
- Convert using training result.
- Predict using training result.
"""
config_path = os.path.join(dirs["config_dir"], config_file)
config = load(config_path)
# Train
experiment_id, checkpoint_name = train(config_path)
train_output_dir = os.path.join(dirs["train_output_dir"], experiment_id)
assert os.path.exists(os.path.join(train_output_dir, 'checkpoints'))
# Convert
convert(experiment_id)
convert_output_dir = os.path.join(train_output_dir, 'export',
checkpoint_name)
lib_dir = os.path.join(
convert_output_dir,
"{}x{}".format(config.IMAGE_SIZE[0], config.IMAGE_SIZE[1]),
'output',
'models',
'lib',
)
assert os.path.exists(os.path.join(lib_dir, 'lm_aarch64.elf'))
assert os.path.exists(os.path.join(lib_dir, 'lm_aarch64_fpga.elf'))
assert os.path.exists(os.path.join(lib_dir, 'lm_arm.elf'))
assert os.path.exists(os.path.join(lib_dir, 'lm_arm_fpga.elf'))
assert os.path.exists(os.path.join(lib_dir, 'lm_x86.elf'))
assert os.path.exists(os.path.join(lib_dir, 'lm_x86_avx.elf'))
assert os.path.exists(os.path.join(lib_dir, 'libdlk_aarch64.so'))
assert os.path.exists(os.path.join(lib_dir, 'libdlk_aarch64_fpga.so'))
assert os.path.exists(os.path.join(lib_dir, 'libdlk_arm.so'))
assert os.path.exists(os.path.join(lib_dir, 'libdlk_arm_fpga.so'))
assert os.path.exists(os.path.join(lib_dir, 'libdlk_x86.so'))
assert os.path.exists(os.path.join(lib_dir, 'libdlk_x86_avx.so'))
assert os.path.exists(os.path.join(lib_dir, 'libdlk_aarch64.a'))
assert os.path.exists(os.path.join(lib_dir, 'libdlk_aarch64_fpga.a'))
assert os.path.exists(os.path.join(lib_dir, 'libdlk_arm.a'))
assert os.path.exists(os.path.join(lib_dir, 'libdlk_arm_fpga.a'))
assert os.path.exists(os.path.join(lib_dir, 'libdlk_x86.a'))
assert os.path.exists(os.path.join(lib_dir, 'libdlk_x86_avx.a'))
# Predict
predict_input_dir = os.path.join(dirs["blueoil_dir"],
"tests/unit/fixtures/sample_images")
predict_output_dir = dirs["predict_output_dir"]
predict(predict_input_dir,
predict_output_dir,
experiment_id,
checkpoint=checkpoint_name)
assert os.path.exists(os.path.join(predict_output_dir, 'images'))
assert os.path.exists(os.path.join(predict_output_dir, 'json', '0.json'))
assert os.path.exists(os.path.join(predict_output_dir, 'npy', '0.npy'))
def _run(config_file, experiment_id, restore_path, image_size, step_size, cpu):
if experiment_id:
environment.init(experiment_id)
config = config_util.load_from_experiment()
if config_file:
config = config_util.merge(config, config_util.load(config_file))
if restore_path is None:
restore_file = executor.search_restore_filename(
environment.CHECKPOINTS_DIR)
restore_path = os.path.join(environment.CHECKPOINTS_DIR,
restore_file)
if not os.path.exists("{}.index".format(restore_path)):
raise Exception(
"restore file {} dont exists.".format(restore_path))
else:
experiment_id = "measure_latency"
environment.init(experiment_id)
config = config_util.load(config_file)
config.BATCH_SIZE = 1
config.NETWORK.BATCH_SIZE = 1
config.DATASET.BATCH_SIZE = 1
if list(image_size) != [None, None]:
config.IMAGE_SIZE = list(image_size)
config.NETWORK.IMAGE_SIZE = list(image_size)
# override pre processes image size.
if config.PRE_PROCESSOR:
config.PRE_PROCESSOR.set_image_size(image_size)
# override post processes image size.
if config.POST_PROCESSOR:
config.POST_PROCESSOR.set_image_size(image_size)
print("Override IMAGE_SIZE", config.IMAGE_SIZE)
executor.init_logging(config)
config_util.display(config)
overall_times, only_network_times = _measure_time(config, restore_path,
step_size)
overall_times = np.array(overall_times)
only_network_times = np.array(only_network_times)
# list of physical_device_desc
devices = [
device.physical_device_desc
for device in device_lib.list_local_devices()
if device.physical_device_desc
]
message = """
---- measure latency result ----
total number of execution (number of samples): {}
network: {}
use gpu by network: {}
image size: {}
devices: {}
* overall (include pre-post-process which execute on cpu)
total time: {:.4f} msec
latency
mean (SD=standard deviation): {:.4f} (SD={:.4f}) msec, min: {:.4f} msec, max: {:.4f} msec
FPS
mean (SD=standard deviation): {:.4f} (SD={:.4f}), min: {:.4f}, max: {:.4f}
* network only (exclude pre-post-process):
total time: {:.4f} msec
latency
mean (SD=standard deviation): {:.4f} (SD={:.4f}) msec, min: {:.4f} msec, max: {:.4f} msec
FPS
mean (SD=standard deviation): {:.4f} (SD={:.4f}), min: {:.4f}, max: {:.4f}
---- measure latency result ----
""".format(
step_size,
config.NETWORK_CLASS.__name__,
not cpu,
config.IMAGE_SIZE,
devices,
# overall
np.sum(overall_times) * 1000,
# latency
np.mean(overall_times) * 1000,
np.std(overall_times) * 1000,
np.min(overall_times) * 1000,
np.max(overall_times) * 1000,
# FPS
np.mean(1 / overall_times),
np.std(1 / overall_times),
np.min(1 / overall_times),
np.max(1 / overall_times),
# network only
np.sum(only_network_times) * 1000,
# latency
np.mean(only_network_times) * 1000,
np.std(only_network_times) * 1000,
np.min(only_network_times) * 1000,
np.max(only_network_times) * 1000,
# FPS
np.mean(1 / only_network_times),
np.std(1 / only_network_times),
np.min(1 / only_network_times),
np.max(1 / only_network_times),
)
print(message)