def run(experiment_id, restore_path, config_file, bit, unquant_layers):
if config_file is None and experiment_id is None:
raise Exception("config_file or experiment_id are required")
if experiment_id:
environment.init(experiment_id)
config = config_util.load_from_experiment()
if config_file:
config = config_util.merge(config, config_util.load(config_file))
if restore_path is None:
restore_file = executor.search_restore_filename(
environment.CHECKPOINTS_DIR)
restore_path = os.path.join(environment.CHECKPOINTS_DIR,
restore_file)
if not os.path.exists("{}.index".format(restore_path)):
raise Exception(
"restore file {} dont exists.".format(restore_path))
else:
experiment_id = "profile"
environment.init(experiment_id)
config = config_util.load(config_file)
executor.init_logging(config)
config_util.display(config)
_profile(config, restore_path, bit, unquant_layers)
def run(experiment_id,
restore_path=None,
image_size=(None, None),
image=DEFAULT_INFERENCE_TEST_DATA_IMAGE,
config_file=None):
environment.init(experiment_id)
config = config_util.load_from_experiment()
if config_file:
config = config_util.merge(config, config_util.load(config_file))
config.BATCH_SIZE = 1
config.NETWORK.BATCH_SIZE = 1
config.DATASET.BATCH_SIZE = 1
if list(image_size) != [None, None]:
config.IMAGE_SIZE = list(image_size)
config.NETWORK.IMAGE_SIZE = list(image_size)
# override pre processes image size.
if config.PRE_PROCESSOR:
config.PRE_PROCESSOR.set_image_size(image_size)
# override post processes image size.
if config.POST_PROCESSOR:
config.POST_PROCESSOR.set_image_size(image_size)
print("Override IMAGE_SIZE", config.IMAGE_SIZE)
executor.init_logging(config)
config_util.display(config)
return _export(config, restore_path, image)
def _setup(self, config):
self.lm_config = config_util.load(self.config['lm_config'])
executor.init_logging(self.lm_config)
model_class = self.lm_config.NETWORK_CLASS
network_kwargs = {key.lower(): val for key, val in self.lm_config.NETWORK.items()}
network_kwargs = update_parameters_for_each_trial(network_kwargs, self.config)
# No distributed training was implemented, therefore rank set to 0
self.train_dataset = setup_dataset(self.lm_config, "train", 0)
self.validation_dataset = setup_dataset(self.lm_config, "validation", 0)
if model_class.__module__.startswith("lmnet.networks.object_detection"):
model = model_class(
classes=self.train_dataset.classes,
num_max_boxes=self.train_dataset.num_max_boxes,
is_debug=self.lm_config.IS_DEBUG,
**network_kwargs,
)
elif model_class.__module__.startswith("lmnet.networks.segmentation"):
model = model_class(
classes=self.train_dataset.classes,
label_colors=self.train_dataset.label_colors,
is_debug=self.lm_config.IS_DEBUG,
**network_kwargs,
)
else:
model = model_class(
classes=self.train_dataset.classes,
is_debug=self.lm_config.IS_DEBUG,
**network_kwargs,
)
self.global_step = tf.Variable(0, name="global_step", trainable=False)
self.is_training_placeholder = tf.placeholder(tf.bool, name="is_training_placeholder")
self.images_placeholder, self.labels_placeholder = model.placeholders()
output = model.inference(self.images_placeholder, self.is_training_placeholder)
if model_class.__module__.startswith("lmnet.networks.object_detection"):
loss = model.loss(output, self.labels_placeholder, self.is_training_placeholder)
else:
loss = model.loss(output, self.labels_placeholder)
opt = model.optimizer(self.global_step)
train_op = model.train(loss, opt, self.global_step)
metrics_ops_dict, metrics_update_op = model.metrics(output, self.labels_placeholder)
self.train_op = train_op
self.metrics_ops_dict = metrics_ops_dict
self.metrics_update_op = metrics_update_op
init_op = tf.global_variables_initializer()
self.reset_metrics_op = tf.local_variables_initializer()
session_config = tf.ConfigProto(
gpu_options=tf.GPUOptions(allow_growth=True))
self.sess = tf.Session(config=session_config)
self.sess.run([init_op, self.reset_metrics_op])
self.iterations = 0
self.saver = tf.train.Saver()
def main(network, dataset, config_file, experiment_id, restore_path):
environment.init(experiment_id)
config = config_util.load_from_experiment()
if config_file:
config = config_util.merge(config, config_util.load(config_file))
if network:
network_class = module_loader.load_network_class(network)
config.NETWORK_CLASS = network_class
if dataset:
dataset_class = module_loader.load_dataset_class(dataset)
config.DATASET_CLASS = dataset_class
executor.init_logging(config)
config_util.display(config)
evaluate(config, restore_path)
def run(network, dataset, config_file, experiment_id, recreate):
environment.init(experiment_id)
config = config_util.load(config_file)
if network:
network_class = module_loader.load_network_class(network)
config.NETWORK_CLASS = network_class
if dataset:
dataset_class = module_loader.load_dataset_class(dataset)
config.DATASET_CLASS = dataset_class
config_util.display(config)
executor.init_logging(config)
executor.prepare_dirs(recreate)
config_util.copy_to_experiment_dir(config_file)
config_util.save_yaml(environment.EXPERIMENT_DIR, config)
start_training(config)
def _run(config_file, experiment_id, restore_path, image_size, step_size, cpu):
if experiment_id:
environment.init(experiment_id)
config = config_util.load_from_experiment()
if config_file:
config = config_util.merge(config, config_util.load(config_file))
if restore_path is None:
restore_file = executor.search_restore_filename(
environment.CHECKPOINTS_DIR)
restore_path = os.path.join(environment.CHECKPOINTS_DIR,
restore_file)
if not os.path.exists("{}.index".format(restore_path)):
raise Exception(
"restore file {} dont exists.".format(restore_path))
else:
experiment_id = "measure_latency"
environment.init(experiment_id)
config = config_util.load(config_file)
config.BATCH_SIZE = 1
config.NETWORK.BATCH_SIZE = 1
config.DATASET.BATCH_SIZE = 1
if list(image_size) != [None, None]:
config.IMAGE_SIZE = list(image_size)
config.NETWORK.IMAGE_SIZE = list(image_size)
# override pre processes image size.
if config.PRE_PROCESSOR:
config.PRE_PROCESSOR.set_image_size(image_size)
# override post processes image size.
if config.POST_PROCESSOR:
config.POST_PROCESSOR.set_image_size(image_size)
print("Override IMAGE_SIZE", config.IMAGE_SIZE)
executor.init_logging(config)
config_util.display(config)
overall_times, only_network_times = _measure_time(config, restore_path,
step_size)
overall_times = np.array(overall_times)
only_network_times = np.array(only_network_times)
# list of physical_device_desc
devices = [
device.physical_device_desc
for device in device_lib.list_local_devices()
if device.physical_device_desc
]
message = """
---- measure latency result ----
total number of execution (number of samples): {}
network: {}
use gpu by network: {}
image size: {}
devices: {}
* overall (include pre-post-process which execute on cpu)
total time: {:.4f} msec
latency
mean (SD=standard deviation): {:.4f} (SD={:.4f}) msec, min: {:.4f} msec, max: {:.4f} msec
FPS
mean (SD=standard deviation): {:.4f} (SD={:.4f}), min: {:.4f}, max: {:.4f}
* network only (exclude pre-post-process):
total time: {:.4f} msec
latency
mean (SD=standard deviation): {:.4f} (SD={:.4f}) msec, min: {:.4f} msec, max: {:.4f} msec
FPS
mean (SD=standard deviation): {:.4f} (SD={:.4f}), min: {:.4f}, max: {:.4f}
---- measure latency result ----
""".format(
step_size,
config.NETWORK_CLASS.__name__,
not cpu,
config.IMAGE_SIZE,
devices,
# overall
np.sum(overall_times) * 1000,
# latency
np.mean(overall_times) * 1000,
np.std(overall_times) * 1000,
np.min(overall_times) * 1000,
np.max(overall_times) * 1000,
# FPS
np.mean(1 / overall_times),
np.std(1 / overall_times),
np.min(1 / overall_times),
np.max(1 / overall_times),
# network only
np.sum(only_network_times) * 1000,
# latency
np.mean(only_network_times) * 1000,
np.std(only_network_times) * 1000,
np.min(only_network_times) * 1000,
np.max(only_network_times) * 1000,
# FPS
np.mean(1 / only_network_times),
np.std(1 / only_network_times),
np.min(1 / only_network_times),
np.max(1 / only_network_times),
)
print(message)
