def _construct_and_fill_model(self): self.device_ids = sly.remap_gpu_devices(self.config['gpu_devices']) src_train_cfg_path = join(self.helper.paths.model_dir, 'model.cfg') with open(src_train_cfg_path) as f: src_config = f.readlines() def repl_batch(row): if 'batch=' in row: return 'batch=1\n' if 'subdivisions=' in row: return 'subdivisions=1\n' return row changed_config = [repl_batch(x) for x in src_config] inf_cfg_path = join(self.helper.paths.model_dir, 'inf_model.cfg') if not os.path.exists(inf_cfg_path): with open(inf_cfg_path, 'w') as f: f.writelines(changed_config) self.net = load_net( inf_cfg_path.encode('utf-8'), join(self.helper.paths.model_dir, 'model.weights').encode('utf-8'), 0) # self.meta = load_meta(join(self.helper.paths.model_dir, 'model.names').encode('utf-8')) logger.info('Weights are loaded.')
def _construct_and_fill_model(self): model_dir = sly.TaskPaths(determine_in_project=False).model_dir self.device_ids = sly.remap_gpu_devices([self.source_gpu_device]) src_train_cfg_path = join(model_dir, 'model.cfg') with open(src_train_cfg_path) as f: src_config = f.readlines() def repl_batch(row): if 'batch=' in row: return 'batch=1\n' if 'subdivisions=' in row: return 'subdivisions=1\n' return row changed_config = [repl_batch(x) for x in src_config] inf_cfg_path = join(model_dir, 'inf_model.cfg') if not os.path.exists(inf_cfg_path): with open(inf_cfg_path, 'w') as f: f.writelines(changed_config) self.net = load_net(inf_cfg_path.encode('utf-8'), join(model_dir, 'model.weights').encode('utf-8'), 0) logger.info('Weights are loaded.')
def _construct_and_fill_model(self): self.device_ids = sly.remap_gpu_devices(self.config['gpu_devices']) self.model = create_model(n_cls=len(self.train_classes), device_ids=self.device_ids) self.model = WeightsRW(self.helper.paths.model_dir).load_strictly(self.model) self.model.eval() logger.info('Weights are loaded.')
def _construct_and_fill_model(self): model_dir = sly.TaskPaths(determine_in_project=False).model_dir self.device_ids = sly.remap_gpu_devices([self.source_gpu_device]) self.model = create_model(n_cls=len(self.train_classes), device_ids=self.device_ids) self.model = WeightsRW(model_dir).load_strictly(self.model) self.model.eval() logger.info('Weights are loaded.')
def _construct_and_fill_model(self): model_dir = sly.TaskPaths(determine_in_project=False).model_dir self.device_ids = sly.remap_gpu_devices([self.source_gpu_device]) if 'model.pb' not in os.listdir(model_dir): logger.info('Freezing training checkpoint!') freeze_graph('image_tensor', model_dir + '/model.config', model_dir + '/model_weights/model.ckpt', model_dir) self.detection_graph = create_detection_graph(model_dir) self.session = tf.Session(graph=self.detection_graph) logger.info('Weights are loaded.')
def train(self): device_ids = sly.remap_gpu_devices(self.config['gpu_devices']) def dump_model(saver, sess, is_best, opt_data): out_dir = self.helper.checkpoints_saver.get_dir_to_write() TrainConfigRW(out_dir).save(self.out_config) model_fpath = os.path.join(out_dir, 'model_weights', 'model.ckpt') saver.save(sess, model_fpath) self.helper.checkpoints_saver.saved(is_best, opt_data) def init_model_fn(sess): if self.helper.model_dir_is_empty(): logger.info('Weights will be inited randomly.') else: exclude_list = ['global_step'] wi_type = self.config['weights_init_type'] ewit = {'weights_init_type': wi_type} logger.info('Weights will be inited from given model.', extra=ewit) if wi_type == 'transfer_learning': last_layers = model.get_extra_layer_scopes(False) exclude_list.extend(last_layers) ignore_missing_vars = True elif wi_type == 'continue_training': self._check_prev_model_config() ignore_missing_vars = False else: raise NotImplementedError( 'Only transfer_learning and continue_training modes are available.' ) logger.info('Weights are loaded.', extra=ewit) variables_to_restore = slim.get_variables_to_restore( exclude=exclude_list) init_fn = slim.assign_from_checkpoint_fn( join(self.helper.paths.model_dir, 'model_weights', 'model.ckpt'), variables_to_restore, ignore_missing_vars=ignore_missing_vars) init_fn(sess) input_shape_hw = (self.input_size_wh[1], self.input_size_wh[0]) train(data_dicts=self.tf_data_dicts, class_num=len(self.out_classes), input_size=input_shape_hw, lr=self.config['lr'], n_epochs=self.config['epochs'], num_clones=len(device_ids), iters_cnt=self.iters_cnt, val_every=self.config['val_every'], model_init_fn=init_model_fn, save_cback=dump_model)
def _construct_and_fill_model(self): self.device_ids = sly.remap_gpu_devices(self.config['gpu_devices']) if 'model.pb' not in os.listdir(self.helper.paths.model_dir): logger.info('Freezing training checkpoint!') freeze_graph( 'image_tensor', self.helper.paths.model_dir + '/model.config', self.helper.paths.model_dir + '/model_weights/model.ckpt', self.helper.paths.model_dir) self.detection_graph = create_detection_graph( self.helper.paths.model_dir) self.session = tf.Session(graph=self.detection_graph) logger.info('Weights are loaded.')
def __init__(self): logger.info('Will init all required to inference.') self.helper = sly.TaskHelperInference() self._determine_settings() self.device_ids = sly.remap_gpu_devices(self.config['gpu_devices']) self._load_train_config() self._construct_and_fill_model() self._determine_input_data() self.debug_copy_images = os.getenv('DEBUG_COPY_IMAGES') is not None logger.info('Model is ready to inference.')
def __init__(self): logger.info('Will init all required to train.') self.helper = sly.TaskHelperTrain() self._determine_settings() self.device_ids = sly.remap_gpu_devices(self.config['gpu_devices']) tf.set_random_seed(RANDOM_SEED) # @TODO: from config self._determine_model_classes() self._determine_out_config() self._explore_input_project() self._tf_common_init() self._load_weights() self.epoch_flt = 0 # real progress logger.info('Model is ready to train.')
def _construct_and_fill_model(self): model_dir = sly.TaskPaths(determine_in_project=False).model_dir self.device_ids = sly.remap_gpu_devices([self.source_gpu_device]) logger.info('Will create model.') with tf.get_default_graph().as_default(): img_np = tf.placeholder(tf.float32, shape=(None, None, 3)) img_shape = tf.shape(img_np) w, h = self.input_size_wh img_np_4d = tf.expand_dims(img_np, axis=0) image_rs_4d = tf.image.resize_bilinear(img_np_4d, (h, w), align_corners=True) image_rs = tf.squeeze(image_rs_4d, axis=0) img = preprocess(image_rs, h, w) if 'model' in self.train_config and self.train_config['model'] == 'pspnet101': PSPNet = PSPNet101 allign_corners = True else: PSPNet = PSPNet50 allign_corners = False net = PSPNet({'data': img}, is_training=False, num_classes=len(self.train_classes)) raw_output = net.layers['conv6'] # 4d # Predictions. raw_output_up = tf.image.resize_bilinear(raw_output, size=[img_shape[0], img_shape[1]], align_corners=False) # raw_output_up = tf.argmax(raw_output_up, dimension=3) logger.info('Will load weights from trained model.') # Init tf Session config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) init = tf.global_variables_initializer() sess.run(init) loader = tf.train.Saver(var_list=tf.global_variables()) # last_checkpoint = tf_saver.latest_checkpoint(output_train_dir) last_checkpoint = osp.join(model_dir, 'model.ckpt') loader.restore(sess, last_checkpoint) self.input_images = img_np self.predictions = raw_output_up self.sess = sess logger.info('Model has been created & weights are loaded.')
def _construct_and_fill_model(self): self.device_ids = sly.remap_gpu_devices(self.config['gpu_devices']) initialized_model = namedtuple('Model', ['input_images', 'predictions', 'sess']) with tf.get_default_graph().as_default(): img = tf.placeholder(tf.float32, shape=(None, None, 3)) input_w, input_h = self.input_size_wh img_re = tf.image.resize_images(img, (input_h, input_w)) original_image, image, label = input_preprocess.preprocess_image_and_label( img_re, None, crop_height=input_h, crop_width=input_w, is_training=False, model_variant="xception_65") image = tf.expand_dims(image, 0) model_options = ModelOptions( outputs_to_num_classes={'semantic': len(self.train_classes)}, crop_size=(input_h, input_w), atrous_rates=[6, 12, 18], output_stride=16) predictions = model.predict_logits(tf.shape(img)[0:2], image, model_options=model_options, image_pyramid=None) predictions = predictions['semantic'] saver = tf.train.Saver(slim.get_variables_to_restore()) sess = tf.train.MonitoredTrainingSession(master='') saver.restore( sess, self.helper.paths.model_dir + '/model_weights/model.ckpt') initialized_model.input_images = img initialized_model.predictions = predictions initialized_model.sess = sess self.initialized_model = initialized_model logger.info('Weights are loaded.')
def train(self): device_ids = sly.remap_gpu_devices(self.config['gpu_devices']) def dump_model(saver, sess, is_best, opt_data): out_dir = self.helper.checkpoints_saver.get_dir_to_write() TrainConfigRW(out_dir).save(self.out_config) save_config(osp.join(out_dir, 'model.config'), self.tf_config) model_fpath = os.path.join(out_dir, 'model_weights', 'model.ckpt') saver.save(sess, model_fpath) self.helper.checkpoints_saver.saved(is_best, opt_data) train(self.tf_data_dicts, self.config['epochs'], self.config['val_every'], self.iters_cnt, self.config['validate_with_model_eval'], pipeline_config=self.tf_config, num_clones=len(device_ids), save_cback=dump_model)
def _construct_and_fill_model(self): self.device_ids = sly.remap_gpu_devices(self.config['gpu_devices']) self.model = create_model(n_cls=len(self.out_classes), device_ids=self.device_ids) if self.helper.model_dir_is_empty(): logger.info('Weights will not be inited.') # @TODO: add random init (m.weight.data.normal_(0, math.sqrt(2. / n)) else: wi_type = self.config['weights_init_type'] ewit = {'weights_init_type': wi_type} logger.info('Weights will be inited from given model.', extra=ewit) weights_rw = WeightsRW(self.helper.paths.model_dir) if wi_type == 'transfer_learning': self.model = weights_rw.load_for_transfer_learning(self.model) elif wi_type == 'continue_training': self._check_prev_model_config() self.model = weights_rw.load_strictly(self.model) logger.info('Weights are loaded.', extra=ewit)
def _construct_and_fill_model(self): model_dir = sly.TaskPaths(determine_in_project=False).model_dir self.device_ids = sly.remap_gpu_devices([self.source_gpu_device]) self.detection_graph = create_detection_graph(model_dir) self.session = tf.Session(graph=self.detection_graph) logger.info('Weights are loaded.')
def _construct_and_fill_model(self): self.device_ids = sly.remap_gpu_devices(self.config['gpu_devices']) self.detection_graph = create_detection_graph(self.helper.paths.model_dir) self.session = tf.Session(graph=self.detection_graph) logger.info('Weights are loaded.')