def _construct_and_fill_model(self):
self.device_ids = sly.remap_gpu_devices(self.config['gpu_devices'])
src_train_cfg_path = join(self.helper.paths.model_dir, 'model.cfg')
with open(src_train_cfg_path) as f:
src_config = f.readlines()
def repl_batch(row):
if 'batch=' in row:
return 'batch=1\n'
if 'subdivisions=' in row:
return 'subdivisions=1\n'
return row
changed_config = [repl_batch(x) for x in src_config]
inf_cfg_path = join(self.helper.paths.model_dir, 'inf_model.cfg')
if not os.path.exists(inf_cfg_path):
with open(inf_cfg_path, 'w') as f:
f.writelines(changed_config)
self.net = load_net(
inf_cfg_path.encode('utf-8'),
join(self.helper.paths.model_dir, 'model.weights').encode('utf-8'),
0)
# self.meta = load_meta(join(self.helper.paths.model_dir, 'model.names').encode('utf-8'))
logger.info('Weights are loaded.')
def _construct_and_fill_model(self):
model_dir = sly.TaskPaths(determine_in_project=False).model_dir
self.device_ids = sly.remap_gpu_devices([self.source_gpu_device])
src_train_cfg_path = join(model_dir, 'model.cfg')
with open(src_train_cfg_path) as f:
src_config = f.readlines()
def repl_batch(row):
if 'batch=' in row:
return 'batch=1\n'
if 'subdivisions=' in row:
return 'subdivisions=1\n'
return row
changed_config = [repl_batch(x) for x in src_config]
inf_cfg_path = join(model_dir, 'inf_model.cfg')
if not os.path.exists(inf_cfg_path):
with open(inf_cfg_path, 'w') as f:
f.writelines(changed_config)
self.net = load_net(inf_cfg_path.encode('utf-8'),
join(model_dir, 'model.weights').encode('utf-8'),
0)
logger.info('Weights are loaded.')
def _construct_and_fill_model(self):
self.device_ids = sly.remap_gpu_devices(self.config['gpu_devices'])
self.model = create_model(n_cls=len(self.train_classes), device_ids=self.device_ids)
self.model = WeightsRW(self.helper.paths.model_dir).load_strictly(self.model)
self.model.eval()
logger.info('Weights are loaded.')
def _construct_and_fill_model(self):
model_dir = sly.TaskPaths(determine_in_project=False).model_dir
self.device_ids = sly.remap_gpu_devices([self.source_gpu_device])
self.model = create_model(n_cls=len(self.train_classes),
device_ids=self.device_ids)
self.model = WeightsRW(model_dir).load_strictly(self.model)
self.model.eval()
logger.info('Weights are loaded.')
def _construct_and_fill_model(self):
model_dir = sly.TaskPaths(determine_in_project=False).model_dir
self.device_ids = sly.remap_gpu_devices([self.source_gpu_device])
if 'model.pb' not in os.listdir(model_dir):
logger.info('Freezing training checkpoint!')
freeze_graph('image_tensor', model_dir + '/model.config',
model_dir + '/model_weights/model.ckpt', model_dir)
self.detection_graph = create_detection_graph(model_dir)
self.session = tf.Session(graph=self.detection_graph)
logger.info('Weights are loaded.')
def train(self):
device_ids = sly.remap_gpu_devices(self.config['gpu_devices'])
def dump_model(saver, sess, is_best, opt_data):
out_dir = self.helper.checkpoints_saver.get_dir_to_write()
TrainConfigRW(out_dir).save(self.out_config)
model_fpath = os.path.join(out_dir, 'model_weights', 'model.ckpt')
saver.save(sess, model_fpath)
self.helper.checkpoints_saver.saved(is_best, opt_data)
def init_model_fn(sess):
if self.helper.model_dir_is_empty():
logger.info('Weights will be inited randomly.')
else:
exclude_list = ['global_step']
wi_type = self.config['weights_init_type']
ewit = {'weights_init_type': wi_type}
logger.info('Weights will be inited from given model.',
extra=ewit)
if wi_type == 'transfer_learning':
last_layers = model.get_extra_layer_scopes(False)
exclude_list.extend(last_layers)
ignore_missing_vars = True
elif wi_type == 'continue_training':
self._check_prev_model_config()
ignore_missing_vars = False
else:
raise NotImplementedError(
'Only transfer_learning and continue_training modes are available.'
)
logger.info('Weights are loaded.', extra=ewit)
variables_to_restore = slim.get_variables_to_restore(
exclude=exclude_list)
init_fn = slim.assign_from_checkpoint_fn(
join(self.helper.paths.model_dir, 'model_weights',
'model.ckpt'),
variables_to_restore,
ignore_missing_vars=ignore_missing_vars)
init_fn(sess)
input_shape_hw = (self.input_size_wh[1], self.input_size_wh[0])
train(data_dicts=self.tf_data_dicts,
class_num=len(self.out_classes),
input_size=input_shape_hw,
lr=self.config['lr'],
n_epochs=self.config['epochs'],
num_clones=len(device_ids),
iters_cnt=self.iters_cnt,
val_every=self.config['val_every'],
model_init_fn=init_model_fn,
save_cback=dump_model)
def _construct_and_fill_model(self):
self.device_ids = sly.remap_gpu_devices(self.config['gpu_devices'])
if 'model.pb' not in os.listdir(self.helper.paths.model_dir):
logger.info('Freezing training checkpoint!')
freeze_graph(
'image_tensor', self.helper.paths.model_dir + '/model.config',
self.helper.paths.model_dir + '/model_weights/model.ckpt',
self.helper.paths.model_dir)
self.detection_graph = create_detection_graph(
self.helper.paths.model_dir)
self.session = tf.Session(graph=self.detection_graph)
logger.info('Weights are loaded.')
def __init__(self):
logger.info('Will init all required to inference.')
self.helper = sly.TaskHelperInference()
self._determine_settings()
self.device_ids = sly.remap_gpu_devices(self.config['gpu_devices'])
self._load_train_config()
self._construct_and_fill_model()
self._determine_input_data()
self.debug_copy_images = os.getenv('DEBUG_COPY_IMAGES') is not None
logger.info('Model is ready to inference.')
def __init__(self):
logger.info('Will init all required to train.')
self.helper = sly.TaskHelperTrain()
self._determine_settings()
self.device_ids = sly.remap_gpu_devices(self.config['gpu_devices'])
tf.set_random_seed(RANDOM_SEED) # @TODO: from config
self._determine_model_classes()
self._determine_out_config()
self._explore_input_project()
self._tf_common_init()
self._load_weights()
self.epoch_flt = 0 # real progress
logger.info('Model is ready to train.')
def _construct_and_fill_model(self):
model_dir = sly.TaskPaths(determine_in_project=False).model_dir
self.device_ids = sly.remap_gpu_devices([self.source_gpu_device])
logger.info('Will create model.')
with tf.get_default_graph().as_default():
img_np = tf.placeholder(tf.float32, shape=(None, None, 3))
img_shape = tf.shape(img_np)
w, h = self.input_size_wh
img_np_4d = tf.expand_dims(img_np, axis=0)
image_rs_4d = tf.image.resize_bilinear(img_np_4d, (h, w), align_corners=True)
image_rs = tf.squeeze(image_rs_4d, axis=0)
img = preprocess(image_rs, h, w)
if 'model' in self.train_config and self.train_config['model'] == 'pspnet101':
PSPNet = PSPNet101
allign_corners = True
else:
PSPNet = PSPNet50
allign_corners = False
net = PSPNet({'data': img}, is_training=False, num_classes=len(self.train_classes))
raw_output = net.layers['conv6'] # 4d
# Predictions.
raw_output_up = tf.image.resize_bilinear(raw_output,
size=[img_shape[0], img_shape[1]], align_corners=False)
# raw_output_up = tf.argmax(raw_output_up, dimension=3)
logger.info('Will load weights from trained model.')
# Init tf Session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
loader = tf.train.Saver(var_list=tf.global_variables())
# last_checkpoint = tf_saver.latest_checkpoint(output_train_dir)
last_checkpoint = osp.join(model_dir, 'model.ckpt')
loader.restore(sess, last_checkpoint)
self.input_images = img_np
self.predictions = raw_output_up
self.sess = sess
logger.info('Model has been created & weights are loaded.')
def _construct_and_fill_model(self):
self.device_ids = sly.remap_gpu_devices(self.config['gpu_devices'])
initialized_model = namedtuple('Model',
['input_images', 'predictions', 'sess'])
with tf.get_default_graph().as_default():
img = tf.placeholder(tf.float32, shape=(None, None, 3))
input_w, input_h = self.input_size_wh
img_re = tf.image.resize_images(img, (input_h, input_w))
original_image, image, label = input_preprocess.preprocess_image_and_label(
img_re,
None,
crop_height=input_h,
crop_width=input_w,
is_training=False,
model_variant="xception_65")
image = tf.expand_dims(image, 0)
model_options = ModelOptions(
outputs_to_num_classes={'semantic': len(self.train_classes)},
crop_size=(input_h, input_w),
atrous_rates=[6, 12, 18],
output_stride=16)
predictions = model.predict_logits(tf.shape(img)[0:2],
image,
model_options=model_options,
image_pyramid=None)
predictions = predictions['semantic']
saver = tf.train.Saver(slim.get_variables_to_restore())
sess = tf.train.MonitoredTrainingSession(master='')
saver.restore(
sess,
self.helper.paths.model_dir + '/model_weights/model.ckpt')
initialized_model.input_images = img
initialized_model.predictions = predictions
initialized_model.sess = sess
self.initialized_model = initialized_model
logger.info('Weights are loaded.')
def train(self):
device_ids = sly.remap_gpu_devices(self.config['gpu_devices'])
def dump_model(saver, sess, is_best, opt_data):
out_dir = self.helper.checkpoints_saver.get_dir_to_write()
TrainConfigRW(out_dir).save(self.out_config)
save_config(osp.join(out_dir, 'model.config'), self.tf_config)
model_fpath = os.path.join(out_dir, 'model_weights', 'model.ckpt')
saver.save(sess, model_fpath)
self.helper.checkpoints_saver.saved(is_best, opt_data)
train(self.tf_data_dicts,
self.config['epochs'],
self.config['val_every'],
self.iters_cnt,
self.config['validate_with_model_eval'],
pipeline_config=self.tf_config,
num_clones=len(device_ids),
save_cback=dump_model)
def _construct_and_fill_model(self):
self.device_ids = sly.remap_gpu_devices(self.config['gpu_devices'])
self.model = create_model(n_cls=len(self.out_classes),
device_ids=self.device_ids)
if self.helper.model_dir_is_empty():
logger.info('Weights will not be inited.')
# @TODO: add random init (m.weight.data.normal_(0, math.sqrt(2. / n))
else:
wi_type = self.config['weights_init_type']
ewit = {'weights_init_type': wi_type}
logger.info('Weights will be inited from given model.', extra=ewit)
weights_rw = WeightsRW(self.helper.paths.model_dir)
if wi_type == 'transfer_learning':
self.model = weights_rw.load_for_transfer_learning(self.model)
elif wi_type == 'continue_training':
self._check_prev_model_config()
self.model = weights_rw.load_strictly(self.model)
logger.info('Weights are loaded.', extra=ewit)
def _construct_and_fill_model(self):
model_dir = sly.TaskPaths(determine_in_project=False).model_dir
self.device_ids = sly.remap_gpu_devices([self.source_gpu_device])
self.detection_graph = create_detection_graph(model_dir)
self.session = tf.Session(graph=self.detection_graph)
logger.info('Weights are loaded.')
def _construct_and_fill_model(self):
self.device_ids = sly.remap_gpu_devices(self.config['gpu_devices'])
self.detection_graph = create_detection_graph(self.helper.paths.model_dir)
self.session = tf.Session(graph=self.detection_graph)
logger.info('Weights are loaded.')
