def load_weight(self): with self._graph.as_default(): if cfg.MODEL.continue_train: #########################restore the params variables_restore = tf.get_collection( tf.GraphKeys.MODEL_VARIABLES) print(variables_restore) variables_restore_n = [ v for v in variables_restore if 'postprocessing' not in v.name ] # Conv2d_1c_1x1 Bottleneck saver2 = tf.train.Saver(variables_restore) saver2.restore(self.sess, cfg.MODEL.pretrained_model) elif cfg.MODEL.pretrained_model is not None: #########################restore the params variables_restore = tf.get_collection( tf.GraphKeys.MODEL_VARIABLES, scope=cfg.MODEL.net_structure) print(variables_restore) # print('......................................................') # # saver2 = tf.train.Saver(variables_restore) variables_restore_n = [ v for v in variables_restore if 'GN' not in v.name ] # Conv2d_1c_1x1 Bottleneck # print(variables_restore_n) saver2 = tf.train.Saver(variables_restore_n) saver2.restore(self.sess, cfg.MODEL.pretrained_model) else: logger.info('no pretrained model, train from sctrach')
def balance(anns): res_anns = copy.deepcopy(anns) for ann in anns: label = ann[-1] label = np.array([label.split(' ')], dtype=np.float).reshape((-1, 2)) bbox = np.array([ np.min(label[:, 0]), np.min(label[:, 1]), np.max(label[:, 0]), np.max(label[:, 1]) ]) bbox_width = bbox[2] - bbox[0] bbox_height = bbox[3] - bbox[1] if bbox_width < 40 or bbox_height < 40: res_anns.remove(ann) if np.sqrt(np.square(label[37,0]-label[41,0])+np.square(label[37,1]-label[41,1]))/bbox_height<0.02 \ or np.sqrt(np.square(label[38, 0] - label[40, 0]) + np.square(label[38, 1] - label[40, 1])) / bbox_height < 0.02 \ or np.sqrt(np.square(label[43,0]-label[47,0])+np.square(label[43,1]-label[47,1]))/bbox_height<0.02 \ or np.sqrt(np.square(label[44, 0] - label[46, 0]) + np.square(label[44, 1] - label[46, 1])) / bbox_height < 0.02 : for i in range(10): res_anns.append(ann) random.shuffle(res_anns) logger.info('befor balance the dataset contains %d images' % (len(anns))) logger.info('after balanced the datasets contains %d samples' % (len(res_anns))) return res_anns
def remove_enforcement(self, enforcement: Enforcement) -> None: application = V1alpha1Application(metadata=V1ObjectMeta( name=enforcement.name)) self._application_service.delete_mixin9(application.metadata.name, cascade=False) logger.info(f"Application {application.metadata.name} removed")
def get_train_data_list(im_root_path, ann_txt): """ train_im_path : image folder name train_ann_path : coco json file name """ logger.info("[x] Get data from {}".format(im_root_path)) # data = PoseInfo(im_path, ann_path, False) data = data_info(im_root_path, ann_txt) all_samples = data.get_all_sample() return all_samples
def read_txt(self): with open(self.txt_file) as _f: txt_lines = _f.readlines() txt_lines.sort() for line in txt_lines: line = line.rstrip() _img_path = line.rsplit('| ', 1)[0] _label = line.rsplit('| ', 1)[-1] current_img_path = os.path.join(self.root_path, _img_path) current_img_label = _label self.metas.append([current_img_path, current_img_label]) ###some change can be made here logger.info('the dataset contains %d images' % (len(txt_lines))) logger.info('the datasets contains %d samples' % (len(self.metas)))
def __analyzeUrl(self,url): """ 检查本url中的有用信息,包括new url,及本url信息 :param url: :return:分析出的new url,本页面有效信息写入db """ logger.debug("number:%s"%self.number) self.number +=1 logger.info("begin analysis url:%s"%url) result_urls=[] try: req = urllib.request.Request(url) req.add_header('User-Agent','Mozilla/5.0 (X11; Linux i686; rv:8.0) Gecko/20100101 Firefox/8.0') response = urllib.request.urlopen(req) pagedata = response.read() chardit = chardet.detect(pagedata)['encoding'] pagedata=pagedata.decode(chardit) #logger.debug("%s coding is %s"%(url,chardit)) soup=BeautifulSoup(pagedata,"html.parser") for link in soup.find_all('a',attrs={"href":re.compile(r'^http://[^\s]*.smzdm.com[^\s]*')}): #print(link.get('href')) if self.__filterUrl(link.get('href')): result_urls.append(link.get('href')) rc = re.compile(r'^http://[^\s]*.smzdm.com/p/\d+/') dataurl=rc.findall(url) #print(dataurl) if dataurl: #http://www.smzdm.com/p/6275447/ my=soup.find('h1',attrs={"class":"article_title"}) #post like http://post.smzdm.com/p/469942/ if my is None: my=soup.find('h1',attrs={"class":"item-name"}) print(my.text) print("dataurl:"+dataurl[0]) except Exception as err: logger.error("url data error %s"%url) traceback.print_exc() return result_urls
def train_loop(self): """Train faces data for a number of epoch.""" self.build() self.load_weight() with self._graph.as_default(): # Create a saver. self.saver = tf.train.Saver(tf.global_variables(), max_to_keep=None) # Build the summary operation from the last tower summaries. self.summary_op = tf.summary.merge(self.summaries) self.summary_writer = tf.summary.FileWriter( cfg.MODEL.model_path, self.sess.graph) min_loss_control = 1000. for epoch in range(cfg.TRAIN.epoch): self._train(self.train_ds, epoch) val_loss = self._val(self.val_ds, epoch) logger.info('**************' 'val_loss %f ' % (val_loss)) #tmp_model_name=cfg.MODEL.model_path + \ # 'epoch_' + str(epoch ) + \ # 'L2_' + str(cfg.TRAIN.weight_decay_factor) + \ # '.ckpt' #logger.info('save model as %s \n'%tmp_model_name) #self.saver.save(self.sess, save_path=tmp_model_name) if 1: min_loss_control = val_loss low_loss_model_name = cfg.MODEL.model_path + \ 'epoch_' + str(epoch) + \ 'L2_' + str(cfg.TRAIN.weight_decay_factor) + '.ckpt' logger.info('A new low loss model saved as %s \n' % low_loss_model_name) self.saver.save(self.sess, save_path=low_loss_model_name) self.sess.close()
def prepare_data(image, boxes, klass, is_crowd=0): boxes = np.copy(boxes) im = image assert im is not None im = im.astype('float32') # assume floatbox as input assert boxes.dtype == np.float32, "Loader has to return floating point boxes!" ret = {'image': im} # rpn anchor: try: if cfg.MODEL.MODE_FPN: multilevel_anchor_inputs = get_multilevel_rpn_anchor_input( im, boxes, is_crowd) for i, (anchor_labels, anchor_boxes) in enumerate(multilevel_anchor_inputs): ret['anchor_labels_lvl{}'.format(i + 2)] = anchor_labels ret['anchor_boxes_lvl{}'.format(i + 2)] = anchor_boxes else: # anchor_labels, anchor_boxes ret['anchor_labels'], ret['anchor_boxes'] = get_rpn_anchor_input( im, boxes, is_crowd) boxes = boxes[is_crowd == 0] # skip crowd boxes in training target klass = klass[is_crowd == 0] ret['gt_boxes'] = boxes ret['gt_labels'] = klass if not len(boxes): raise MalformedData("No valid gt_boxes!") except MalformedData as e: logger.info( "Input {} is filtered for training: {}".format('err', str(e)), 'warn') return None return ret
def test_humidity(self, host): data = CaseLoader().load()['test case'][function_name()] r = requests.request(data['request']['method'], url=host + data['request']['url'], headers=data['request']['headers']) logger.info(r.status_code) assert r.status_code == Config.CODE_SUCCESS res = self.get_humidity_of_day_after_tomorrow(r) logger.info(f'Day after tomorrow: {res["forecast_date"]}') # extract relative humidity logger.info(f'Relative humidity: {res["min_rh"]}% - {res["max_rh"]}%') # assert forecast_date is the day after tomorrow assert res['forecast_date'] == day_after_tomorrow()
def wrapper(*args, **kwargs): logger.info(f'Call [{func.__name__}], Arguments [{args}, {kwargs}]') return func(*args, **kwargs)
def _train(self,train_ds,_epoch): for step in range(cfg.TRAIN.iter_num_per_epoch): self.ite_num += 1 ########show_flag check the data if cfg.TRAIN.vis: example_image,example_all_boxes,example_all_labels = next(train_ds) print(example_all_boxes.shape) for i in range(example_all_boxes.shape[0]): img=example_image[i] box_encode=example_all_boxes[i] label=example_all_labels[i] print(np.sum(label[label>0])) # for j in range(label.shape[0]): # if label[j]>0: # print(box_encode[j]) cv2.namedWindow('img', 0) cv2.imshow('img', img) cv2.waitKey(0) else: start_time = time.time() feed_dict = {} for n in range(cfg.TRAIN.num_gpu): examples = next(train_ds) feed_dict[self.inputs[0][n]] = examples[0][n*cfg.TRAIN.batch_size:(n+1)*cfg.TRAIN.batch_size] feed_dict[self.inputs[1][n]] = examples[1][n*cfg.TRAIN.batch_size:(n+1)*cfg.TRAIN.batch_size] feed_dict[self.inputs[2][n]] = examples[2][n*cfg.TRAIN.batch_size:(n+1)*cfg.TRAIN.batch_size] feed_dict[self.inputs[3]] = cfg.TRAIN.dropout feed_dict[self.inputs[4]] = cfg.TRAIN.weight_decay_factor feed_dict[self.inputs[5]] = True fetch_duration = time.time() - start_time start_time2 = time.time() _, total_loss_value,reg_loss_value,cla_loss_value,l2_loss_value,lr_value = \ self.sess.run([*self.outputs], feed_dict=feed_dict) duration = time.time() - start_time2 run_duration = duration if self.ite_num % cfg.TRAIN.log_interval == 0: num_examples_per_step = cfg.TRAIN.batch_size * cfg.TRAIN.num_gpu examples_per_sec = num_examples_per_step / duration sec_per_batch = duration / cfg.TRAIN.num_gpu format_str = ('epoch %d: iter %d, ' 'total_loss=%.6f ' 'reg_loss=%.6f ' 'cla_loss=%.6f ' 'l2_loss=%.6f ' 'learning rate =%e ' '(%.1f examples/sec; %.3f sec/batch) ' 'fetch data time = %.6f' 'run time = %.6f') logger.info(format_str % (_epoch, self.ite_num, total_loss_value, reg_loss_value, cla_loss_value, l2_loss_value, lr_value, examples_per_sec, sec_per_batch, fetch_duration, run_duration)) if self.ite_num % 100 == 0: summary_str = self.sess.run(self.summary_op, feed_dict=feed_dict) self.summary_writer.add_summary(summary_str, self.ite_num)
def train_loop(self): """Train faces data for a number of epoch.""" with tf.Graph().as_default(), tf.device('/cpu:0'): # Create a variable to count the number of train() calls. This equals the # number of batches processed * FLAGS.num_gpus. global_step = tf.get_variable( 'global_step', [], initializer=tf.constant_initializer(0),dtype=tf.int32, trainable=False) # Decay the learning rate lr = tf.train.piecewise_constant(global_step, cfg.TRAIN.lr_decay_every_step, cfg.TRAIN.lr_value_every_step ) keep_prob = tf.placeholder(tf.float32, name="keep_prob") L2_reg = tf.placeholder(tf.float32, name="L2_reg") training = tf.placeholder(tf.bool, name="training_flag") total_loss_to_show=0. images_place_holder_list = [] labels_place_holder_list = [] boxes_place_holder_list = [] # Create an optimizer that performs gradient descent. #opt = tf.train.AdamOptimizer(lr) opt = tf.train.MomentumOptimizer(lr,momentum=0.9,use_nesterov=False) # Get images and labels weights_initializer = slim.xavier_initializer() biases_initializer = tf.constant_initializer(0.) biases_regularizer = tf.no_regularizer weights_regularizer = tf.contrib.layers.l2_regularizer(L2_reg) # Calculate the gradients for each model tower. tower_grads = [] with tf.variable_scope(tf.get_variable_scope()): for i in range(cfg.TRAIN.num_gpu): with tf.device('/gpu:%d' % i): with tf.name_scope('tower_%d' % (i)) as scope: with slim.arg_scope([slim.model_variable, slim.variable], device='/cpu:0'): images_ = tf.placeholder(tf.float32, [None,None,None, 3], name="images") boxes_ = tf.placeholder(tf.float32, [cfg.TRAIN.batch_size,None,4],name="input_boxes") labels_ = tf.placeholder(tf.int64, [cfg.TRAIN.batch_size,None], name="input_labels") ###total anchor images_place_holder_list.append(images_) labels_place_holder_list.append(labels_) boxes_place_holder_list.append(boxes_) with slim.arg_scope([slim.conv2d, slim.conv2d_in_plane, \ slim.conv2d_transpose, slim.separable_conv2d, slim.fully_connected], weights_regularizer=weights_regularizer, biases_regularizer=biases_regularizer, weights_initializer=weights_initializer, biases_initializer=biases_initializer): reg_loss,cla_loss,l2_loss = self.tower_loss( scope, images_, labels_,boxes_, L2_reg, training) ##use muti gpu ,large batch if i == cfg.TRAIN.num_gpu - 1: total_loss = tf.add_n([ reg_loss,cla_loss,l2_loss]) else: total_loss = tf.add_n([ reg_loss,cla_loss]) total_loss_to_show+=total_loss # Reuse variables for the next tower. tf.get_variable_scope().reuse_variables() ##when use batchnorm, updates operations only from the ## final tower. Ideally, we should grab the updates from all towers # but these stats accumulate extremely fast so we can ignore the # other stats from the other towers without significant detriment. bn_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, scope=scope) # Retain the summaries from the final tower. summaries = tf.get_collection(tf.GraphKeys.SUMMARIES, scope) # Calculate the gradients for the batch of data on this CIFAR tower. grads = opt.compute_gradients(total_loss) # Keep track of the gradients across all towers. tower_grads.append(grads) # We must calculate the mean of each gradient. Note that this is the # synchronization point across all towers. grads = self.average_gradients(tower_grads) # Add a summary to track the learning rate. summaries.append(tf.summary.scalar('learning_rate', lr)) summaries.append(tf.summary.scalar('total_loss', total_loss_to_show)) summaries.append(tf.summary.scalar('loc_loss', reg_loss)) summaries.append(tf.summary.scalar('cla_loss', cla_loss)) summaries.append(tf.summary.scalar('l2_loss', l2_loss)) # Add histograms for gradients. for grad, var in grads: if grad is not None: summaries.append(tf.summary.histogram(var.op.name + '/gradients', grad)) # Apply the gradients to adjust the shared variables. apply_gradient_op = opt.apply_gradients(grads, global_step=global_step) # Add histograms for trainable variables. for var in tf.trainable_variables(): summaries.append(tf.summary.histogram(var.op.name, var)) if False: # Track the moving averages of all trainable variables. variable_averages = tf.train.ExponentialMovingAverage( 0.9, global_step) variables_averages_op = variable_averages.apply(tf.trainable_variables()) # Group all updates to into a single train op. train_op = tf.group(apply_gradient_op, variables_averages_op, *bn_update_ops) else: train_op = tf.group(apply_gradient_op, *bn_update_ops) self.inputs=[images_place_holder_list,boxes_place_holder_list,labels_place_holder_list,keep_prob,L2_reg,training] self.outputs=[train_op,total_loss_to_show,reg_loss,cla_loss,l2_loss,lr ] self.val_outputs=[grads,total_loss_to_show,reg_loss,cla_loss,l2_loss,lr ] # Create a saver. self.saver = tf.train.Saver(tf.global_variables(), max_to_keep=None) # Build the summary operation from the last tower summaries. self.summary_op = tf.summary.merge(summaries) # Build an initialization operation to run below. init = tf.global_variables_initializer() # Start running operations on the Graph. allow_soft_placement must be set to # True to build towers on GPU, as some of the ops do not have GPU # implementations. tf_config = tf.ConfigProto( allow_soft_placement=True, log_device_placement=False) tf_config.gpu_options.allow_growth = True self.sess = tf.Session(config=tf_config) self.sess.run(init) if cfg.MODEL.continue_train: #########################restore the params variables_restore = tf.get_collection(tf.GraphKeys.MODEL_VARIABLES) print(variables_restore) variables_restore_n = [v for v in variables_restore if 'postprocessing' not in v.name] # Conv2d_1c_1x1 Bottleneck saver2 = tf.train.Saver(variables_restore) saver2.restore(self.sess, cfg.MODEL.pretrained_model) elif cfg.MODEL.pretrained_model is not None: #########################restore the params variables_restore = tf.get_collection(tf.GraphKeys.MODEL_VARIABLES,scope=cfg.MODEL.net_structure) print(variables_restore) # print('......................................................') # # saver2 = tf.train.Saver(variables_restore) variables_restore_n = [v for v in variables_restore if 'GN' not in v.name] # Conv2d_1c_1x1 Bottleneck # print(variables_restore_n) saver2 = tf.train.Saver(variables_restore_n) saver2.restore(self.sess, cfg.MODEL.pretrained_model) # self.coord = tf.train.Coordinator() # Start the queue runners. # self.threads = tf.train.start_queue_runners(sess=self.sess, coord=self.coord) self.summary_writer = tf.summary.FileWriter(cfg.MODEL.model_path, self.sess.graph) train_ds=self.make_data(self.train_data_set,is_training=True) val_ds=self.make_data(self.val_data_set,is_training=False) ###### min_loss_control=1000. for epoch in range(cfg.TRAIN.epoch): self._train(train_ds,epoch) val_loss=self._val(val_ds,epoch) logger.info('**************' 'val_loss %f '%(val_loss)) #tmp_model_name=cfg.MODEL.model_path + \ # 'epoch_' + str(epoch ) + \ # 'L2_' + str(cfg.TRAIN.weight_decay_factor) + \ # '.ckpt' #logger.info('save model as %s \n'%tmp_model_name) #self.saver.save(self.sess, save_path=tmp_model_name) if 1: min_loss_control=val_loss low_loss_model_name = cfg.MODEL.model_path + \ 'epoch_' + str(epoch) + \ 'L2_' + str(cfg.TRAIN.weight_decay_factor) + '.ckpt' logger.info('A new low loss model saved as %s \n' % low_loss_model_name) self.saver.save(self.sess, save_path=low_loss_model_name) self.sess.close()
def _train(self,train_ds,_epoch): for step in range(cfg.TRAIN.iter_num_per_epoch): self.ite_num += 1 ########show_flag check the data if cfg.TRAIN.vis: examples = next(train_ds) print(examples) example_image=examples['image'] # for i in range(cfg.TRAIN.batch_size): # example = examples[i] # images=example['image'] # # # print(example) # # print(example_image.shape) # # print(example_label.shape) # # print(example_matche.shape) cv2.namedWindow('img', 0) cv2.imshow('img', example_image) cv2.waitKey(0) else: start_time = time.time() feed_dict = {} for n in range(cfg.TRAIN.num_gpu): examples = next(train_ds) feed_dict[self.inputs[0][n]] = examples['image'] feed_dict[self.inputs[1][n]] = examples['gt_boxes'] feed_dict[self.inputs[2][n]] = examples['gt_labels'] for k in range(len(cfg.FPN.ANCHOR_STRIDES)): feed_dict[self.inputs[3][n][2*k]]=examples['anchor_labels_lvl{}'.format(k + 2)] feed_dict[self.inputs[3][n][2*k+1]] = examples['anchor_boxes_lvl{}'.format(k + 2)] #print(examples['anchor_labels_lvl{}'.format(k + 2)].shape) feed_dict[self.inputs[4]] = cfg.TRAIN.dropout feed_dict[self.inputs[5]] = cfg.TRAIN.weight_decay_factor feed_dict[self.inputs[6]] = True fetch_duration = time.time() - start_time start_time = time.time() _, total_loss_value,loss_value,l2_loss_value,lr_value = \ self.sess.run([*self.outputs], feed_dict=feed_dict) duration = time.time() - start_time run_duration = duration - fetch_duration if self.ite_num % cfg.TRAIN.log_interval == 0: num_examples_per_step = cfg.TRAIN.batch_size * cfg.TRAIN.num_gpu examples_per_sec = num_examples_per_step / duration sec_per_batch = duration / cfg.TRAIN.num_gpu logger.info(loss_value) format_str = ('epoch %d: iter %d, ' 'total_loss=%.6f ' 'l2_loss=%.6f ' 'learning rate =%e ' '(%.1f examples/sec; %.3f sec/batch) ' 'fetch data time = %.6f' 'run time = %.6f') logger.info(format_str % (_epoch, self.ite_num, total_loss_value, l2_loss_value, lr_value, examples_per_sec, sec_per_batch, fetch_duration, run_duration))