def __init__(self, remote_url, local_save_path, project_path, main_lnk_paths, sync_files = [], ignore_files = []):
'''
@summary: 更新代码初始化函数
---------
@param remote_url: 远程代码发布地址
@param local_save_path: 代码下载路径
@param project_path: 本地项目路径
@param main_lnk_paths: 本地项目执行文件快捷方式地址
@param sync_files: 同步的文件 .* 表示同步全部
@param ignore_files: 忽略的文件
---------
@result:
'''
self._remote_url = remote_url
self._local_save_path = local_save_path
self._project_path = project_path
self._main_lnk_paths = main_lnk_paths
self._sync_files = sync_files
self._ignore_files = ignore_files
self._remote_zip_url = ''
self._tag = ''
self._zip_path = ''
self._unpack_path = ''
self._project_name = tools.get_info(remote_url, '/([^/]*?)/releases', fetch_one = True)
self._tag_json = tools.get_json(tools.read_file(VERSION_FILE)) or {}
def __init_args(self):
Log.add_log("data agument strategy : " + str(self.agument))
# parameters of data augment strategy
self.multi_scale_img = self.agument[0] # multiscale zoom the image
self.keep_img_shape = self.agument[
1] # keep image's shape when we reshape the image
self.flip_img = self.agument[2] # flip image
self.gray_img = self.agument[3] # gray image
self.label_smooth = self.agument[4] # label smooth strategy
self.erase_img = self.agument[5] # random erase image
self.invert_img = self.agument[6] # invert image pixel
self.rotate_img = self.agument[7] # rotate image
Log.add_log("message: begin to initial images path")
# init imgs path
self.imgs_path = tools.read_file(self.data_file)
if not self.imgs_path:
Log.add_log("error:imgs_path not exist")
raise ValueError("no file find in :" + str(self.data_file))
Log.add_log("message:there are " + str(len(self.imgs_path)) +
" pictures in all")
# init labels path
for img in self.imgs_path:
label_path = img.replace("JPEGImages", "labels")
label_path = label_path.replace(img.split('.')[-1], "txt")
self.labels_path.append(label_path)
Log.add_log(
"message: initialize images path and corresponding labels complete"
)
return
def main():
# 用记事本打开文件后，会在conf文本头前面加上\ufeff，需要处理掉
content = tools.read_file('config.conf')
tools.write_file('config.conf', content.replace('\ufeff', ''))
# 读配置
cp = configparser.ConfigParser(allow_no_value = True)
with codecs.open('config.conf', 'r', encoding='utf-8') as f:
cp.read_file(f)
sections = cp.sections()
for section in sections:
remote_url = cp.get(section, 'remote_url')
local_save_path = cp.get(section, 'local_save_path')
project_path = cp.get(section, 'project_path')
main_lnk_paths = cp.get(section, 'main_lnk_paths').split(',')
sync_files = cp.get(section, 'sync_files').split(',')
ignore_files = cp.get(section, 'ignore_files').split(',')
# # 调用
update_code = UpdateCode(remote_url, local_save_path, project_path, main_lnk_paths, sync_files, ignore_files)
if update_code.check_remote_tag():
update_code.download_code()
update_code.copy_file()
update_code.close_process()
update_code.start_process()
def __init_args(self):
Log.add_log("data agument strategy : "+str(self.agument))
# 初始化数据增强策略的参数
self.multi_scale_img = self.agument[0] # 多尺度缩放图片
self.keep_img_shape = self.agument[1] # keep image's shape when we reshape the image
self.flip_img = self.agument[2] # flip image
self.gray_img = self.agument[3] # gray image
self.label_smooth = self.agument[4] # label smooth strategy
self.erase_img = self.agument[5] # random erase image
self.invert_img = self.agument[6] # invert image pixel
self.rotate_img = self.agument[7] # rotate image
Log.add_log("message:开始初始化路径")
# init imgs path
self.imgs_path = tools.read_file(self.data_file)
if not self.imgs_path:
Log.add_log("error:imgs_path文件不存在")
raise ValueError(str(self.data_file) + ":文件中不存在路径")
Log.add_log("message:一共有 "+str(len(self.imgs_path)) + " 张图片")
# init labels path
for img in self.imgs_path:
label_path = img.replace("JPEGImages", "labels")
label_path = label_path.replace(img.split('.')[-1], "txt")
self.labels_path.append(label_path)
Log.add_log("message:数据路径初始化完成")
return
def add_cookies_from_file(self, file):
cookies = tools.read_file(file, True)
for cookie in cookies:
cookie = cookie.strip()
print(cookie)
if cookie:
sql = "insert into sogou_cookies (cookie) values ('%s')" % (cookie)
self._sqlite3db.add(sql)
def close_process(self):
files = tools.walk_file(self._project_path)
for file_name in files:
if 'pid' in file_name and file_name.endswith('.txt'):
pid = tools.read_file(file_name)
command = 'taskkill /F /PID %s'%pid
log.info('%s : %s'%(file_name, command))
tools.exec_command(command)
def read_file_to_dataset(
self, img_path=None, label_path=None,
label_format="labelimg",
rescale=1/255,
preprocessing=None,
augmenter=None,
aug_times=1,
shuffle=True, seed=None,
encoding="big5",
thread_num=10):
"""Read the images and annotaions
created by labelimg or labelme as ndarray.
Args:
img_path: A string,
file path of images.
label_path: A string,
file path of annotations.
label_format: A string,
one of "labelimg" and "labelme".
rescale: A float or None,
specifying how the image value should be scaled.
If None, no scaled.
preprocessing: A function of data preprocessing,
(e.g. noralization, shape manipulation, etc.)
augmenter: A `imgaug.augmenters.meta.Sequential` instance.
aug_times: An integer,
the default is 1, which means no augmentation.
shuffle: Boolean, default: True.
seed: An integer, random seed, default: None.
encoding: A string,
encoding format of file,
default: "big5".
thread_num: An integer,
specifying the number of threads to read files.
Returns:
A tuple of 2 ndarrays, (img, label),
- shape of img: (batch_size, img_heights, img_widths, channel)
- shape of label: (batch_size, grid_heights, grid_widths, info)
"""
img_data, label_data, path_list = tools.read_file(
img_path=img_path,
label_path=label_path,
label_format=label_format,
size=self.input_shape[:2],
grid_shape=self.grid_shape,
class_names=self.class_names,
rescale=rescale,
preprocessing=preprocessing,
augmenter=augmenter,
aug_times=aug_times,
shuffle=shuffle, seed=seed,
encoding=encoding,
thread_num=thread_num)
self.file_names = path_list
return img_data, label_data
def __init__(self, table):
self._record_time = tools.get_json(
tools.read_file(SYNC_TIME_FILE)) or {}
self._compare_keywords = CompareKeywords()
self._summary = Summary()
self._emotion = Emotion()
self._word_cloud = WordCloud()
self._es = ES()
self._hot_sync = HotSync()
self._vip_checked = VipChecked()
self._table = table
self._per_record_time_key = '{table}_record_time'.format(
table=self._table)
def read_label(self, label_file):
'''
读取 label_file, 并生成 label_y1, label_y2, label_y3
return:label_y1, label_y2, label_y3
'''
contents = tools.read_file(label_file)
if not contents:
return None, None, None
label_y1 = np.zeros(
(self.height // 32, self.width // 32, 3, 5 + self.class_num),
np.float32)
label_y2 = np.zeros(
(self.height // 16, self.width // 16, 3, 5 + self.class_num),
np.float32)
label_y3 = np.zeros(
(self.height // 8, self.width // 8, 3, 5 + self.class_num),
np.float32)
y_true = [label_y3, label_y2, label_y1]
ratio = {0: 8, 1: 16, 2: 32}
for label in contents:
label = label.split()
if len(label) != 5:
Log.add_log("error:文件'" + str(label_file) + "'标签数量错误")
raise ValueError(str(label_file) + ":标签数量错误")
label_id = int(label[0])
box = np.asarray(label[1:5]).astype(
np.float32) # label中保存的就是 x,y,w,h
best_giou = 0
best_index = 0
for i in range(len(self.anchors)):
min_wh = np.minimum(box[2:4], self.anchors[i])
max_wh = np.maximum(box[2:4], self.anchors[i])
giou = (min_wh[0] * min_wh[1]) / (max_wh[0] * max_wh[1])
if giou > best_giou:
best_giou = giou
best_index = i
# 012->0, 345->1, 678->2
x = int(np.floor(box[0] * self.width / ratio[best_index // 3]))
y = int(np.floor(box[1] * self.height / ratio[best_index // 3]))
k = best_index % 3
y_true[best_index // 3][y, x, k, 0:4] = box
y_true[best_index // 3][y, x, k, 4:5] = 1.0
y_true[best_index // 3][y, x, k, 5 + label_id] = 1.0
return label_y1, label_y2, label_y3
def get_url(time_lenght = 60):
'''
@summary:
---------
@param time_lenght: 时间段 分钟
---------
@result:
'''
current_date = tools.get_current_date()
per_date = tools.read_file(STO_PER_SYNC_TIME) or tools.timestamp_to_date(tools.get_current_timestamp() - time_lenght * 60)
tools.write_file(STO_PER_SYNC_TIME, current_date)
root_url = 'http://192.168.60.38:8001/hotspot_al/interface/getCluesDataSearchInfo?pageNo=%d&pageSize=100&updateSTime={per_date}&updateETime={current_date}&sort=5&isDesc=0'.format(per_date = per_date, current_date = current_date)
return root_url
def __init_args(self):
Log.add_log("message:开始初始化路径")
# init imgs path
self.imgs_path = tools.read_file(self.data_file)
if not self.imgs_path:
Log.add_log("error:imgs_path文件不存在")
raise ValueError(str(self.data_file) + ":文件中不存在路径")
print("data:一共有 %d 张图片" %(len(self.imgs_path)))
# init labels path
for img in self.imgs_path:
label_path = img.replace("JPEGImages", "labels")
label_path = label_path.replace(img.split('.')[-1], "txt")
self.labels_path.append(label_path)
Log.add_log("message:数据路径初始化完成")
return
def __init__(self, table):
self._sync_time_file = SYNC_TIME_FILE + table + '.txt'
self._record_time = tools.get_json(
tools.read_file(self._sync_time_file)) or {}
self._compare_keywords = CompareKeywords()
self._summary = Summary()
self._emotion = Emotion()
self._word_cloud = WordCloud()
self._yqtj_es = ES(YQTJ)
self._data_pool_es = ES(DATA_POOL)
self._hot_sync = HotSync()
self._vip_checked = VipChecked()
self._province_filter = ProvinceFilter()
self._table = table
self._per_record_time_key = '{table}_record_time'.format(
table=self._table)
self._vip_checked.start()
self._compare_keywords.start()
def __init_args(self):
Log.add_log("message: begin to initial images path")
# init imgs path
self.imgs_path = tools.read_file(self.data_file)
if not self.imgs_path:
Log.add_log("error:imgs_path not exist")
raise ValueError("no file find in :" + str(self.data_file))
self.num_imgs = len(self.imgs_path)
self.steps_per_epoch = int(self.num_imgs / self.batch_size)
Log.add_log("message:there are " + str(self.num_imgs) +
" pictures in all")
# init labels path
for img in self.imgs_path:
label_path = img.replace("JPEGImages", "labels")
label_path = label_path.replace(img.split('.')[-1], "txt")
self.labels_path.append(label_path)
Log.add_log(
"message: initialize images path and corresponding labels complete"
)
return
def __init__(self):
self._es = ES()
self._current_csr_res_id = tools.read_file(STO_CURRENT_ID_FILE)
self._current_csr_res_id = self._current_csr_res_id and int(
self._current_csr_res_id) or 0
def __init__(self):
self._es = ES()
self._db = OracleDB()
self._max_id = tools.read_file(STO_MAX_ID_FILE)
self._max_id = self._max_id and eval(self._max_id) or {}
def __init__(self):
self._record_time = tools.read_file(SYNC_TIME_FILE) or {}
def read_label(self, label_file, anchors, new_w, new_h):
'''
读取 label_file, 并生成 label_y1, label_y2, label_y3
return:label_y1, label_y2, label_y3
'''
contents = tools.read_file(label_file)
if not contents:
return None, None, None
if self.keep_img_shape:
x_pad = (self.width - new_w) // 2
y_pad = (self.height - new_h) // 2
label_y1 = np.zeros((self.height // 32, self.width // 32, 3, 5 + self.class_num), np.float32)
label_y2 = np.zeros((self.height // 16, self.width // 16, 3, 5 + self.class_num), np.float32)
label_y3 = np.zeros((self.height // 8, self.width // 8, 3, 5 + self.class_num), np.float32)
y_true = [label_y3, label_y2, label_y1]
ratio = {0:8, 1:16, 2:32}
test_result = []
for label in contents:
label = label.split()
if len(label) != 5:
Log.add_log("error:文件'" + str(label_file) + "'标签数量错误")
raise ValueError(str(label_file) + ":标签数量错误")
label_id = int(label[0])
box = np.asarray(label[1: 5]).astype(np.float32) # label中保存的就是 x,y,w,h
# flip the label
if self.flip_img:
box[0] = 1.0 - box[0]
if self.keep_img_shape:
# 加入填充的黑边宽高，重新修正坐标
box[0:2] = (box[0:2] * [new_w, new_h ] + [x_pad, y_pad]) / [self.width, self.height]
box[2:4] = (box[2:4] * [new_w, new_h]) / [self.width, self.height]
test_result.append([box[0]-box[2]/2, box[1]-box[3]/2, box[0]+box[2]/2, box[1]+box[3]/2])
best_giou = 0
best_index = 0
for i in range(len(anchors)):
min_wh = np.minimum(box[2:4], anchors[i])
max_wh = np.maximum(box[2:4], anchors[i])
giou = (min_wh[0] * min_wh[1]) / (max_wh[0] * max_wh[1])
if giou > best_giou:
best_giou = giou
best_index = i
# 012->0, 345->1, 678->2
x = int(np.floor(box[0] * self.width / ratio[best_index // 3]))
y = int(np.floor(box[1] * self.height / ratio[best_index // 3]))
k = best_index % 3
y_true[best_index // 3][y, x, k, 0:4] = box
# label smooth
label_value = 1.0 if not self.label_smooth else ((1-self.smooth_delta) + self.smooth_delta * 1 / self.class_num)
y_true[best_index // 3][y, x, k, 4:5] = label_value
y_true[best_index // 3][y, x, k, 5 + label_id] = label_value
return label_y1, label_y2, label_y3, test_result
def read_label(self, label_file, anchors, new_w, new_h):
'''
parsement label_file, and generates label_y1, label_y2, label_y3
return:label_y1, label_y2, label_y3
'''
contents = tools.read_file(label_file)
if not contents:
return None, None, None
label_y1 = np.zeros(
(self.height // 32, self.width // 32, 3, 5 + self.class_num),
np.float32)
label_y2 = np.zeros(
(self.height // 16, self.width // 16, 3, 5 + self.class_num),
np.float32)
label_y3 = np.zeros(
(self.height // 8, self.width // 8, 3, 5 + self.class_num),
np.float32)
delta = self.smooth_delta
if delta:
label_y1[:, :, :, 4] = delta / self.class_num
label_y2[:, :, :, 4] = delta / self.class_num
label_y3[:, :, :, 4] = delta / self.class_num
y_true = [label_y3, label_y2, label_y1]
ratio = {0: 8, 1: 16, 2: 32}
test_result = []
for label in contents:
label = label.split()
if len(label) != 5:
Log.add_log("error: in file '" + str(label_file) +
"', the number of parameter does not match")
raise ValueError(
str(label_file) +
": the number of label parameter does not match")
label_id = int(label[0])
box = np.asarray(label[1:5]).astype(
np.float32) # what saved in label are x,y,w,h
# flip the label
if self.is_flip:
box[0] = 1.0 - box[0]
test_result.append([
box[0] - box[2] / 2, box[1] - box[3] / 2, box[0] + box[2] / 2,
box[1] + box[3] / 2
])
best_giou = 0
best_index = 0
for i in range(len(anchors)):
min_wh = np.minimum(box[2:4], anchors[i])
max_wh = np.maximum(box[2:4], anchors[i])
giou = (min_wh[0] * min_wh[1]) / (max_wh[0] * max_wh[1])
if giou > best_giou:
best_giou = giou
best_index = i
# 012->0, 345->1, 678->2
x = int(np.floor(box[0] * self.width / ratio[best_index // 3]))
y = int(np.floor(box[1] * self.height / ratio[best_index // 3]))
k = best_index % 3
y_true[best_index // 3][y, x, k, 0:4] = box
# label smooth
label_value = 1.0 if not delta else (1 - delta)
y_true[best_index // 3][y, x, k, 4:5] = label_value
y_true[best_index // 3][y, x, k, 5:] = delta / self.class_num
y_true[best_index // 3][y, x, k, 5 + label_id] = label_value
if self.is_flip:
self.is_flip = False
return label_y1, label_y2, label_y3, test_result
