def load_weight(self):
with self._graph.as_default():
if cfg.MODEL.continue_train:
#########################restore the params
variables_restore = tf.get_collection(
tf.GraphKeys.MODEL_VARIABLES)
print(variables_restore)
variables_restore_n = [
v for v in variables_restore
if 'postprocessing' not in v.name
] # Conv2d_1c_1x1 Bottleneck
saver2 = tf.train.Saver(variables_restore)
saver2.restore(self.sess, cfg.MODEL.pretrained_model)
elif cfg.MODEL.pretrained_model is not None:
#########################restore the params
variables_restore = tf.get_collection(
tf.GraphKeys.MODEL_VARIABLES,
scope=cfg.MODEL.net_structure)
print(variables_restore)
# print('......................................................')
# # saver2 = tf.train.Saver(variables_restore)
variables_restore_n = [
v for v in variables_restore if 'GN' not in v.name
] # Conv2d_1c_1x1 Bottleneck
# print(variables_restore_n)
saver2 = tf.train.Saver(variables_restore_n)
saver2.restore(self.sess, cfg.MODEL.pretrained_model)
else:
logger.info('no pretrained model, train from sctrach')
def balance(anns):
res_anns = copy.deepcopy(anns)
for ann in anns:
label = ann[-1]
label = np.array([label.split(' ')], dtype=np.float).reshape((-1, 2))
bbox = np.array([
np.min(label[:, 0]),
np.min(label[:, 1]),
np.max(label[:, 0]),
np.max(label[:, 1])
])
bbox_width = bbox[2] - bbox[0]
bbox_height = bbox[3] - bbox[1]
if bbox_width < 40 or bbox_height < 40:
res_anns.remove(ann)
if np.sqrt(np.square(label[37,0]-label[41,0])+np.square(label[37,1]-label[41,1]))/bbox_height<0.02 \
or np.sqrt(np.square(label[38, 0] - label[40, 0]) + np.square(label[38, 1] - label[40, 1])) / bbox_height < 0.02 \
or np.sqrt(np.square(label[43,0]-label[47,0])+np.square(label[43,1]-label[47,1]))/bbox_height<0.02 \
or np.sqrt(np.square(label[44, 0] - label[46, 0]) + np.square(label[44, 1] - label[46, 1])) / bbox_height < 0.02 :
for i in range(10):
res_anns.append(ann)
random.shuffle(res_anns)
logger.info('befor balance the dataset contains %d images' % (len(anns)))
logger.info('after balanced the datasets contains %d samples' %
(len(res_anns)))
return res_anns
def remove_enforcement(self, enforcement: Enforcement) -> None:
application = V1alpha1Application(metadata=V1ObjectMeta(
name=enforcement.name))
self._application_service.delete_mixin9(application.metadata.name,
cascade=False)
logger.info(f"Application {application.metadata.name} removed")
def get_train_data_list(im_root_path, ann_txt):
"""
train_im_path : image folder name
train_ann_path : coco json file name
"""
logger.info("[x] Get data from {}".format(im_root_path))
# data = PoseInfo(im_path, ann_path, False)
data = data_info(im_root_path, ann_txt)
all_samples = data.get_all_sample()
return all_samples
def read_txt(self):
with open(self.txt_file) as _f:
txt_lines = _f.readlines()
txt_lines.sort()
for line in txt_lines:
line = line.rstrip()
_img_path = line.rsplit('| ', 1)[0]
_label = line.rsplit('| ', 1)[-1]
current_img_path = os.path.join(self.root_path, _img_path)
current_img_label = _label
self.metas.append([current_img_path, current_img_label])
###some change can be made here
logger.info('the dataset contains %d images' % (len(txt_lines)))
logger.info('the datasets contains %d samples' % (len(self.metas)))
def __analyzeUrl(self,url):
"""
检查本url中的有用信息，包括new url，及本url信息
:param url:
:return:分析出的new url，本页面有效信息写入db
"""
logger.debug("number:%s"%self.number)
self.number +=1
logger.info("begin analysis url:%s"%url)
result_urls=[]
try:
req = urllib.request.Request(url)
req.add_header('User-Agent','Mozilla/5.0 (X11; Linux i686; rv:8.0) Gecko/20100101 Firefox/8.0')
response = urllib.request.urlopen(req)
pagedata = response.read()
chardit = chardet.detect(pagedata)['encoding']
pagedata=pagedata.decode(chardit)
#logger.debug("%s coding is %s"%(url,chardit))
soup=BeautifulSoup(pagedata,"html.parser")
for link in soup.find_all('a',attrs={"href":re.compile(r'^http://[^\s]*.smzdm.com[^\s]*')}):
#print(link.get('href'))
if self.__filterUrl(link.get('href')):
result_urls.append(link.get('href'))
rc = re.compile(r'^http://[^\s]*.smzdm.com/p/\d+/')
dataurl=rc.findall(url)
#print(dataurl)
if dataurl:
#http://www.smzdm.com/p/6275447/
my=soup.find('h1',attrs={"class":"article_title"})
#post like http://post.smzdm.com/p/469942/
if my is None:
my=soup.find('h1',attrs={"class":"item-name"})
print(my.text)
print("dataurl:"+dataurl[0])
except Exception as err:
logger.error("url data error %s"%url)
traceback.print_exc()
return result_urls
def train_loop(self):
"""Train faces data for a number of epoch."""
self.build()
self.load_weight()
with self._graph.as_default():
# Create a saver.
self.saver = tf.train.Saver(tf.global_variables(),
max_to_keep=None)
# Build the summary operation from the last tower summaries.
self.summary_op = tf.summary.merge(self.summaries)
self.summary_writer = tf.summary.FileWriter(
cfg.MODEL.model_path, self.sess.graph)
min_loss_control = 1000.
for epoch in range(cfg.TRAIN.epoch):
self._train(self.train_ds, epoch)
val_loss = self._val(self.val_ds, epoch)
logger.info('**************' 'val_loss %f ' % (val_loss))
#tmp_model_name=cfg.MODEL.model_path + \
# 'epoch_' + str(epoch ) + \
# 'L2_' + str(cfg.TRAIN.weight_decay_factor) + \
# '.ckpt'
#logger.info('save model as %s \n'%tmp_model_name)
#self.saver.save(self.sess, save_path=tmp_model_name)
if 1:
min_loss_control = val_loss
low_loss_model_name = cfg.MODEL.model_path + \
'epoch_' + str(epoch) + \
'L2_' + str(cfg.TRAIN.weight_decay_factor) + '.ckpt'
logger.info('A new low loss model saved as %s \n' %
low_loss_model_name)
self.saver.save(self.sess, save_path=low_loss_model_name)
self.sess.close()
def prepare_data(image, boxes, klass, is_crowd=0):
boxes = np.copy(boxes)
im = image
assert im is not None
im = im.astype('float32')
# assume floatbox as input
assert boxes.dtype == np.float32, "Loader has to return floating point boxes!"
ret = {'image': im}
# rpn anchor:
try:
if cfg.MODEL.MODE_FPN:
multilevel_anchor_inputs = get_multilevel_rpn_anchor_input(
im, boxes, is_crowd)
for i, (anchor_labels,
anchor_boxes) in enumerate(multilevel_anchor_inputs):
ret['anchor_labels_lvl{}'.format(i + 2)] = anchor_labels
ret['anchor_boxes_lvl{}'.format(i + 2)] = anchor_boxes
else:
# anchor_labels, anchor_boxes
ret['anchor_labels'], ret['anchor_boxes'] = get_rpn_anchor_input(
im, boxes, is_crowd)
boxes = boxes[is_crowd == 0] # skip crowd boxes in training target
klass = klass[is_crowd == 0]
ret['gt_boxes'] = boxes
ret['gt_labels'] = klass
if not len(boxes):
raise MalformedData("No valid gt_boxes!")
except MalformedData as e:
logger.info(
"Input {} is filtered for training: {}".format('err', str(e)),
'warn')
return None
return ret
def test_humidity(self, host):
data = CaseLoader().load()['test case'][function_name()]
r = requests.request(data['request']['method'],
url=host + data['request']['url'],
headers=data['request']['headers'])
logger.info(r.status_code)
assert r.status_code == Config.CODE_SUCCESS
res = self.get_humidity_of_day_after_tomorrow(r)
logger.info(f'Day after tomorrow: {res["forecast_date"]}')
# extract relative humidity
logger.info(f'Relative humidity: {res["min_rh"]}% - {res["max_rh"]}%')
# assert forecast_date is the day after tomorrow
assert res['forecast_date'] == day_after_tomorrow()
def wrapper(*args, **kwargs):
logger.info(f'Call [{func.__name__}], Arguments [{args}, {kwargs}]')
return func(*args, **kwargs)
def _train(self,train_ds,_epoch):
for step in range(cfg.TRAIN.iter_num_per_epoch):
self.ite_num += 1
########show_flag check the data
if cfg.TRAIN.vis:
example_image,example_all_boxes,example_all_labels = next(train_ds)
print(example_all_boxes.shape)
for i in range(example_all_boxes.shape[0]):
img=example_image[i]
box_encode=example_all_boxes[i]
label=example_all_labels[i]
print(np.sum(label[label>0]))
# for j in range(label.shape[0]):
# if label[j]>0:
# print(box_encode[j])
cv2.namedWindow('img', 0)
cv2.imshow('img', img)
cv2.waitKey(0)
else:
start_time = time.time()
feed_dict = {}
for n in range(cfg.TRAIN.num_gpu):
examples = next(train_ds)
feed_dict[self.inputs[0][n]] = examples[0][n*cfg.TRAIN.batch_size:(n+1)*cfg.TRAIN.batch_size]
feed_dict[self.inputs[1][n]] = examples[1][n*cfg.TRAIN.batch_size:(n+1)*cfg.TRAIN.batch_size]
feed_dict[self.inputs[2][n]] = examples[2][n*cfg.TRAIN.batch_size:(n+1)*cfg.TRAIN.batch_size]
feed_dict[self.inputs[3]] = cfg.TRAIN.dropout
feed_dict[self.inputs[4]] = cfg.TRAIN.weight_decay_factor
feed_dict[self.inputs[5]] = True
fetch_duration = time.time() - start_time
start_time2 = time.time()
_, total_loss_value,reg_loss_value,cla_loss_value,l2_loss_value,lr_value = \
self.sess.run([*self.outputs],
feed_dict=feed_dict)
duration = time.time() - start_time2
run_duration = duration
if self.ite_num % cfg.TRAIN.log_interval == 0:
num_examples_per_step = cfg.TRAIN.batch_size * cfg.TRAIN.num_gpu
examples_per_sec = num_examples_per_step / duration
sec_per_batch = duration / cfg.TRAIN.num_gpu
format_str = ('epoch %d: iter %d, '
'total_loss=%.6f '
'reg_loss=%.6f '
'cla_loss=%.6f '
'l2_loss=%.6f '
'learning rate =%e '
'(%.1f examples/sec; %.3f sec/batch) '
'fetch data time = %.6f'
'run time = %.6f')
logger.info(format_str % (_epoch,
self.ite_num,
total_loss_value,
reg_loss_value,
cla_loss_value,
l2_loss_value,
lr_value,
examples_per_sec,
sec_per_batch,
fetch_duration,
run_duration))
if self.ite_num % 100 == 0:
summary_str = self.sess.run(self.summary_op, feed_dict=feed_dict)
self.summary_writer.add_summary(summary_str, self.ite_num)
def train_loop(self):
"""Train faces data for a number of epoch."""
with tf.Graph().as_default(), tf.device('/cpu:0'):
# Create a variable to count the number of train() calls. This equals the
# number of batches processed * FLAGS.num_gpus.
global_step = tf.get_variable(
'global_step', [],
initializer=tf.constant_initializer(0),dtype=tf.int32, trainable=False)
# Decay the learning rate
lr = tf.train.piecewise_constant(global_step,
cfg.TRAIN.lr_decay_every_step,
cfg.TRAIN.lr_value_every_step
)
keep_prob = tf.placeholder(tf.float32, name="keep_prob")
L2_reg = tf.placeholder(tf.float32, name="L2_reg")
training = tf.placeholder(tf.bool, name="training_flag")
total_loss_to_show=0.
images_place_holder_list = []
labels_place_holder_list = []
boxes_place_holder_list = []
# Create an optimizer that performs gradient descent.
#opt = tf.train.AdamOptimizer(lr)
opt = tf.train.MomentumOptimizer(lr,momentum=0.9,use_nesterov=False)
# Get images and labels
weights_initializer = slim.xavier_initializer()
biases_initializer = tf.constant_initializer(0.)
biases_regularizer = tf.no_regularizer
weights_regularizer = tf.contrib.layers.l2_regularizer(L2_reg)
# Calculate the gradients for each model tower.
tower_grads = []
with tf.variable_scope(tf.get_variable_scope()):
for i in range(cfg.TRAIN.num_gpu):
with tf.device('/gpu:%d' % i):
with tf.name_scope('tower_%d' % (i)) as scope:
with slim.arg_scope([slim.model_variable, slim.variable], device='/cpu:0'):
images_ = tf.placeholder(tf.float32, [None,None,None, 3], name="images")
boxes_ = tf.placeholder(tf.float32, [cfg.TRAIN.batch_size,None,4],name="input_boxes")
labels_ = tf.placeholder(tf.int64, [cfg.TRAIN.batch_size,None], name="input_labels")
###total anchor
images_place_holder_list.append(images_)
labels_place_holder_list.append(labels_)
boxes_place_holder_list.append(boxes_)
with slim.arg_scope([slim.conv2d, slim.conv2d_in_plane, \
slim.conv2d_transpose, slim.separable_conv2d,
slim.fully_connected],
weights_regularizer=weights_regularizer,
biases_regularizer=biases_regularizer,
weights_initializer=weights_initializer,
biases_initializer=biases_initializer):
reg_loss,cla_loss,l2_loss = self.tower_loss(
scope, images_, labels_,boxes_, L2_reg, training)
##use muti gpu ,large batch
if i == cfg.TRAIN.num_gpu - 1:
total_loss = tf.add_n([ reg_loss,cla_loss,l2_loss])
else:
total_loss = tf.add_n([ reg_loss,cla_loss])
total_loss_to_show+=total_loss
# Reuse variables for the next tower.
tf.get_variable_scope().reuse_variables()
##when use batchnorm, updates operations only from the
## final tower. Ideally, we should grab the updates from all towers
# but these stats accumulate extremely fast so we can ignore the
# other stats from the other towers without significant detriment.
bn_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, scope=scope)
# Retain the summaries from the final tower.
summaries = tf.get_collection(tf.GraphKeys.SUMMARIES, scope)
# Calculate the gradients for the batch of data on this CIFAR tower.
grads = opt.compute_gradients(total_loss)
# Keep track of the gradients across all towers.
tower_grads.append(grads)
# We must calculate the mean of each gradient. Note that this is the
# synchronization point across all towers.
grads = self.average_gradients(tower_grads)
# Add a summary to track the learning rate.
summaries.append(tf.summary.scalar('learning_rate', lr))
summaries.append(tf.summary.scalar('total_loss', total_loss_to_show))
summaries.append(tf.summary.scalar('loc_loss', reg_loss))
summaries.append(tf.summary.scalar('cla_loss', cla_loss))
summaries.append(tf.summary.scalar('l2_loss', l2_loss))
# Add histograms for gradients.
for grad, var in grads:
if grad is not None:
summaries.append(tf.summary.histogram(var.op.name + '/gradients', grad))
# Apply the gradients to adjust the shared variables.
apply_gradient_op = opt.apply_gradients(grads, global_step=global_step)
# Add histograms for trainable variables.
for var in tf.trainable_variables():
summaries.append(tf.summary.histogram(var.op.name, var))
if False:
# Track the moving averages of all trainable variables.
variable_averages = tf.train.ExponentialMovingAverage(
0.9, global_step)
variables_averages_op = variable_averages.apply(tf.trainable_variables())
# Group all updates to into a single train op.
train_op = tf.group(apply_gradient_op, variables_averages_op, *bn_update_ops)
else:
train_op = tf.group(apply_gradient_op, *bn_update_ops)
self.inputs=[images_place_holder_list,boxes_place_holder_list,labels_place_holder_list,keep_prob,L2_reg,training]
self.outputs=[train_op,total_loss_to_show,reg_loss,cla_loss,l2_loss,lr ]
self.val_outputs=[grads,total_loss_to_show,reg_loss,cla_loss,l2_loss,lr ]
# Create a saver.
self.saver = tf.train.Saver(tf.global_variables(), max_to_keep=None)
# Build the summary operation from the last tower summaries.
self.summary_op = tf.summary.merge(summaries)
# Build an initialization operation to run below.
init = tf.global_variables_initializer()
# Start running operations on the Graph. allow_soft_placement must be set to
# True to build towers on GPU, as some of the ops do not have GPU
# implementations.
tf_config = tf.ConfigProto(
allow_soft_placement=True,
log_device_placement=False)
tf_config.gpu_options.allow_growth = True
self.sess = tf.Session(config=tf_config)
self.sess.run(init)
if cfg.MODEL.continue_train:
#########################restore the params
variables_restore = tf.get_collection(tf.GraphKeys.MODEL_VARIABLES)
print(variables_restore)
variables_restore_n = [v for v in variables_restore if
'postprocessing' not in v.name] # Conv2d_1c_1x1 Bottleneck
saver2 = tf.train.Saver(variables_restore)
saver2.restore(self.sess, cfg.MODEL.pretrained_model)
elif cfg.MODEL.pretrained_model is not None:
#########################restore the params
variables_restore = tf.get_collection(tf.GraphKeys.MODEL_VARIABLES,scope=cfg.MODEL.net_structure)
print(variables_restore)
# print('......................................................')
# # saver2 = tf.train.Saver(variables_restore)
variables_restore_n = [v for v in variables_restore if
'GN' not in v.name] # Conv2d_1c_1x1 Bottleneck
# print(variables_restore_n)
saver2 = tf.train.Saver(variables_restore_n)
saver2.restore(self.sess, cfg.MODEL.pretrained_model)
# self.coord = tf.train.Coordinator()
# Start the queue runners.
# self.threads = tf.train.start_queue_runners(sess=self.sess, coord=self.coord)
self.summary_writer = tf.summary.FileWriter(cfg.MODEL.model_path, self.sess.graph)
train_ds=self.make_data(self.train_data_set,is_training=True)
val_ds=self.make_data(self.val_data_set,is_training=False)
######
min_loss_control=1000.
for epoch in range(cfg.TRAIN.epoch):
self._train(train_ds,epoch)
val_loss=self._val(val_ds,epoch)
logger.info('**************'
'val_loss %f '%(val_loss))
#tmp_model_name=cfg.MODEL.model_path + \
# 'epoch_' + str(epoch ) + \
# 'L2_' + str(cfg.TRAIN.weight_decay_factor) + \
# '.ckpt'
#logger.info('save model as %s \n'%tmp_model_name)
#self.saver.save(self.sess, save_path=tmp_model_name)
if 1:
min_loss_control=val_loss
low_loss_model_name = cfg.MODEL.model_path + \
'epoch_' + str(epoch) + \
'L2_' + str(cfg.TRAIN.weight_decay_factor) + '.ckpt'
logger.info('A new low loss model saved as %s \n' % low_loss_model_name)
self.saver.save(self.sess, save_path=low_loss_model_name)
self.sess.close()
def _train(self,train_ds,_epoch):
for step in range(cfg.TRAIN.iter_num_per_epoch):
self.ite_num += 1
########show_flag check the data
if cfg.TRAIN.vis:
examples = next(train_ds)
print(examples)
example_image=examples['image']
# for i in range(cfg.TRAIN.batch_size):
# example = examples[i]
# images=example['image']
#
#
# print(example)
# # print(example_image.shape)
# # print(example_label.shape)
# # print(example_matche.shape)
cv2.namedWindow('img', 0)
cv2.imshow('img', example_image)
cv2.waitKey(0)
else:
start_time = time.time()
feed_dict = {}
for n in range(cfg.TRAIN.num_gpu):
examples = next(train_ds)
feed_dict[self.inputs[0][n]] = examples['image']
feed_dict[self.inputs[1][n]] = examples['gt_boxes']
feed_dict[self.inputs[2][n]] = examples['gt_labels']
for k in range(len(cfg.FPN.ANCHOR_STRIDES)):
feed_dict[self.inputs[3][n][2*k]]=examples['anchor_labels_lvl{}'.format(k + 2)]
feed_dict[self.inputs[3][n][2*k+1]] = examples['anchor_boxes_lvl{}'.format(k + 2)]
#print(examples['anchor_labels_lvl{}'.format(k + 2)].shape)
feed_dict[self.inputs[4]] = cfg.TRAIN.dropout
feed_dict[self.inputs[5]] = cfg.TRAIN.weight_decay_factor
feed_dict[self.inputs[6]] = True
fetch_duration = time.time() - start_time
start_time = time.time()
_, total_loss_value,loss_value,l2_loss_value,lr_value = \
self.sess.run([*self.outputs],
feed_dict=feed_dict)
duration = time.time() - start_time
run_duration = duration - fetch_duration
if self.ite_num % cfg.TRAIN.log_interval == 0:
num_examples_per_step = cfg.TRAIN.batch_size * cfg.TRAIN.num_gpu
examples_per_sec = num_examples_per_step / duration
sec_per_batch = duration / cfg.TRAIN.num_gpu
logger.info(loss_value)
format_str = ('epoch %d: iter %d, '
'total_loss=%.6f '
'l2_loss=%.6f '
'learning rate =%e '
'(%.1f examples/sec; %.3f sec/batch) '
'fetch data time = %.6f'
'run time = %.6f')
logger.info(format_str % (_epoch,
self.ite_num,
total_loss_value,
l2_loss_value,
lr_value,
examples_per_sec,
sec_per_batch,
fetch_duration,
run_duration))