def __init__(self, remote_url, local_save_path, project_path, main_lnk_paths, sync_files = [], ignore_files = []): ''' @summary: 更新代码初始化函数 --------- @param remote_url: 远程代码发布地址 @param local_save_path: 代码下载路径 @param project_path: 本地项目路径 @param main_lnk_paths: 本地项目执行文件快捷方式地址 @param sync_files: 同步的文件 .* 表示同步全部 @param ignore_files: 忽略的文件 --------- @result: ''' self._remote_url = remote_url self._local_save_path = local_save_path self._project_path = project_path self._main_lnk_paths = main_lnk_paths self._sync_files = sync_files self._ignore_files = ignore_files self._remote_zip_url = '' self._tag = '' self._zip_path = '' self._unpack_path = '' self._project_name = tools.get_info(remote_url, '/([^/]*?)/releases', fetch_one = True) self._tag_json = tools.get_json(tools.read_file(VERSION_FILE)) or {}
def __init_args(self): Log.add_log("data agument strategy : " + str(self.agument)) # parameters of data augment strategy self.multi_scale_img = self.agument[0] # multiscale zoom the image self.keep_img_shape = self.agument[ 1] # keep image's shape when we reshape the image self.flip_img = self.agument[2] # flip image self.gray_img = self.agument[3] # gray image self.label_smooth = self.agument[4] # label smooth strategy self.erase_img = self.agument[5] # random erase image self.invert_img = self.agument[6] # invert image pixel self.rotate_img = self.agument[7] # rotate image Log.add_log("message: begin to initial images path") # init imgs path self.imgs_path = tools.read_file(self.data_file) if not self.imgs_path: Log.add_log("error:imgs_path not exist") raise ValueError("no file find in :" + str(self.data_file)) Log.add_log("message:there are " + str(len(self.imgs_path)) + " pictures in all") # init labels path for img in self.imgs_path: label_path = img.replace("JPEGImages", "labels") label_path = label_path.replace(img.split('.')[-1], "txt") self.labels_path.append(label_path) Log.add_log( "message: initialize images path and corresponding labels complete" ) return
def main(): # 用记事本打开文件后,会在conf文本头前面加上\ufeff,需要处理掉 content = tools.read_file('config.conf') tools.write_file('config.conf', content.replace('\ufeff', '')) # 读配置 cp = configparser.ConfigParser(allow_no_value = True) with codecs.open('config.conf', 'r', encoding='utf-8') as f: cp.read_file(f) sections = cp.sections() for section in sections: remote_url = cp.get(section, 'remote_url') local_save_path = cp.get(section, 'local_save_path') project_path = cp.get(section, 'project_path') main_lnk_paths = cp.get(section, 'main_lnk_paths').split(',') sync_files = cp.get(section, 'sync_files').split(',') ignore_files = cp.get(section, 'ignore_files').split(',') # # 调用 update_code = UpdateCode(remote_url, local_save_path, project_path, main_lnk_paths, sync_files, ignore_files) if update_code.check_remote_tag(): update_code.download_code() update_code.copy_file() update_code.close_process() update_code.start_process()
def __init_args(self): Log.add_log("data agument strategy : "+str(self.agument)) # 初始化数据增强策略的参数 self.multi_scale_img = self.agument[0] # 多尺度缩放图片 self.keep_img_shape = self.agument[1] # keep image's shape when we reshape the image self.flip_img = self.agument[2] # flip image self.gray_img = self.agument[3] # gray image self.label_smooth = self.agument[4] # label smooth strategy self.erase_img = self.agument[5] # random erase image self.invert_img = self.agument[6] # invert image pixel self.rotate_img = self.agument[7] # rotate image Log.add_log("message:开始初始化路径") # init imgs path self.imgs_path = tools.read_file(self.data_file) if not self.imgs_path: Log.add_log("error:imgs_path文件不存在") raise ValueError(str(self.data_file) + ":文件中不存在路径") Log.add_log("message:一共有 "+str(len(self.imgs_path)) + " 张图片") # init labels path for img in self.imgs_path: label_path = img.replace("JPEGImages", "labels") label_path = label_path.replace(img.split('.')[-1], "txt") self.labels_path.append(label_path) Log.add_log("message:数据路径初始化完成") return
def add_cookies_from_file(self, file): cookies = tools.read_file(file, True) for cookie in cookies: cookie = cookie.strip() print(cookie) if cookie: sql = "insert into sogou_cookies (cookie) values ('%s')" % (cookie) self._sqlite3db.add(sql)
def close_process(self): files = tools.walk_file(self._project_path) for file_name in files: if 'pid' in file_name and file_name.endswith('.txt'): pid = tools.read_file(file_name) command = 'taskkill /F /PID %s'%pid log.info('%s : %s'%(file_name, command)) tools.exec_command(command)
def read_file_to_dataset( self, img_path=None, label_path=None, label_format="labelimg", rescale=1/255, preprocessing=None, augmenter=None, aug_times=1, shuffle=True, seed=None, encoding="big5", thread_num=10): """Read the images and annotaions created by labelimg or labelme as ndarray. Args: img_path: A string, file path of images. label_path: A string, file path of annotations. label_format: A string, one of "labelimg" and "labelme". rescale: A float or None, specifying how the image value should be scaled. If None, no scaled. preprocessing: A function of data preprocessing, (e.g. noralization, shape manipulation, etc.) augmenter: A `imgaug.augmenters.meta.Sequential` instance. aug_times: An integer, the default is 1, which means no augmentation. shuffle: Boolean, default: True. seed: An integer, random seed, default: None. encoding: A string, encoding format of file, default: "big5". thread_num: An integer, specifying the number of threads to read files. Returns: A tuple of 2 ndarrays, (img, label), - shape of img: (batch_size, img_heights, img_widths, channel) - shape of label: (batch_size, grid_heights, grid_widths, info) """ img_data, label_data, path_list = tools.read_file( img_path=img_path, label_path=label_path, label_format=label_format, size=self.input_shape[:2], grid_shape=self.grid_shape, class_names=self.class_names, rescale=rescale, preprocessing=preprocessing, augmenter=augmenter, aug_times=aug_times, shuffle=shuffle, seed=seed, encoding=encoding, thread_num=thread_num) self.file_names = path_list return img_data, label_data
def __init__(self, table): self._record_time = tools.get_json( tools.read_file(SYNC_TIME_FILE)) or {} self._compare_keywords = CompareKeywords() self._summary = Summary() self._emotion = Emotion() self._word_cloud = WordCloud() self._es = ES() self._hot_sync = HotSync() self._vip_checked = VipChecked() self._table = table self._per_record_time_key = '{table}_record_time'.format( table=self._table)
def read_label(self, label_file): ''' 读取 label_file, 并生成 label_y1, label_y2, label_y3 return:label_y1, label_y2, label_y3 ''' contents = tools.read_file(label_file) if not contents: return None, None, None label_y1 = np.zeros( (self.height // 32, self.width // 32, 3, 5 + self.class_num), np.float32) label_y2 = np.zeros( (self.height // 16, self.width // 16, 3, 5 + self.class_num), np.float32) label_y3 = np.zeros( (self.height // 8, self.width // 8, 3, 5 + self.class_num), np.float32) y_true = [label_y3, label_y2, label_y1] ratio = {0: 8, 1: 16, 2: 32} for label in contents: label = label.split() if len(label) != 5: Log.add_log("error:文件'" + str(label_file) + "'标签数量错误") raise ValueError(str(label_file) + ":标签数量错误") label_id = int(label[0]) box = np.asarray(label[1:5]).astype( np.float32) # label中保存的就是 x,y,w,h best_giou = 0 best_index = 0 for i in range(len(self.anchors)): min_wh = np.minimum(box[2:4], self.anchors[i]) max_wh = np.maximum(box[2:4], self.anchors[i]) giou = (min_wh[0] * min_wh[1]) / (max_wh[0] * max_wh[1]) if giou > best_giou: best_giou = giou best_index = i # 012->0, 345->1, 678->2 x = int(np.floor(box[0] * self.width / ratio[best_index // 3])) y = int(np.floor(box[1] * self.height / ratio[best_index // 3])) k = best_index % 3 y_true[best_index // 3][y, x, k, 0:4] = box y_true[best_index // 3][y, x, k, 4:5] = 1.0 y_true[best_index // 3][y, x, k, 5 + label_id] = 1.0 return label_y1, label_y2, label_y3
def get_url(time_lenght = 60): ''' @summary: --------- @param time_lenght: 时间段 分钟 --------- @result: ''' current_date = tools.get_current_date() per_date = tools.read_file(STO_PER_SYNC_TIME) or tools.timestamp_to_date(tools.get_current_timestamp() - time_lenght * 60) tools.write_file(STO_PER_SYNC_TIME, current_date) root_url = 'http://192.168.60.38:8001/hotspot_al/interface/getCluesDataSearchInfo?pageNo=%d&pageSize=100&updateSTime={per_date}&updateETime={current_date}&sort=5&isDesc=0'.format(per_date = per_date, current_date = current_date) return root_url
def __init_args(self): Log.add_log("message:开始初始化路径") # init imgs path self.imgs_path = tools.read_file(self.data_file) if not self.imgs_path: Log.add_log("error:imgs_path文件不存在") raise ValueError(str(self.data_file) + ":文件中不存在路径") print("data:一共有 %d 张图片" %(len(self.imgs_path))) # init labels path for img in self.imgs_path: label_path = img.replace("JPEGImages", "labels") label_path = label_path.replace(img.split('.')[-1], "txt") self.labels_path.append(label_path) Log.add_log("message:数据路径初始化完成") return
def __init__(self, table): self._sync_time_file = SYNC_TIME_FILE + table + '.txt' self._record_time = tools.get_json( tools.read_file(self._sync_time_file)) or {} self._compare_keywords = CompareKeywords() self._summary = Summary() self._emotion = Emotion() self._word_cloud = WordCloud() self._yqtj_es = ES(YQTJ) self._data_pool_es = ES(DATA_POOL) self._hot_sync = HotSync() self._vip_checked = VipChecked() self._province_filter = ProvinceFilter() self._table = table self._per_record_time_key = '{table}_record_time'.format( table=self._table) self._vip_checked.start() self._compare_keywords.start()
def __init_args(self): Log.add_log("message: begin to initial images path") # init imgs path self.imgs_path = tools.read_file(self.data_file) if not self.imgs_path: Log.add_log("error:imgs_path not exist") raise ValueError("no file find in :" + str(self.data_file)) self.num_imgs = len(self.imgs_path) self.steps_per_epoch = int(self.num_imgs / self.batch_size) Log.add_log("message:there are " + str(self.num_imgs) + " pictures in all") # init labels path for img in self.imgs_path: label_path = img.replace("JPEGImages", "labels") label_path = label_path.replace(img.split('.')[-1], "txt") self.labels_path.append(label_path) Log.add_log( "message: initialize images path and corresponding labels complete" ) return
def __init__(self): self._es = ES() self._current_csr_res_id = tools.read_file(STO_CURRENT_ID_FILE) self._current_csr_res_id = self._current_csr_res_id and int( self._current_csr_res_id) or 0
def __init__(self): self._es = ES() self._db = OracleDB() self._max_id = tools.read_file(STO_MAX_ID_FILE) self._max_id = self._max_id and eval(self._max_id) or {}
def __init__(self): self._record_time = tools.read_file(SYNC_TIME_FILE) or {}
def read_label(self, label_file, anchors, new_w, new_h): ''' 读取 label_file, 并生成 label_y1, label_y2, label_y3 return:label_y1, label_y2, label_y3 ''' contents = tools.read_file(label_file) if not contents: return None, None, None if self.keep_img_shape: x_pad = (self.width - new_w) // 2 y_pad = (self.height - new_h) // 2 label_y1 = np.zeros((self.height // 32, self.width // 32, 3, 5 + self.class_num), np.float32) label_y2 = np.zeros((self.height // 16, self.width // 16, 3, 5 + self.class_num), np.float32) label_y3 = np.zeros((self.height // 8, self.width // 8, 3, 5 + self.class_num), np.float32) y_true = [label_y3, label_y2, label_y1] ratio = {0:8, 1:16, 2:32} test_result = [] for label in contents: label = label.split() if len(label) != 5: Log.add_log("error:文件'" + str(label_file) + "'标签数量错误") raise ValueError(str(label_file) + ":标签数量错误") label_id = int(label[0]) box = np.asarray(label[1: 5]).astype(np.float32) # label中保存的就是 x,y,w,h # flip the label if self.flip_img: box[0] = 1.0 - box[0] if self.keep_img_shape: # 加入填充的黑边宽高,重新修正坐标 box[0:2] = (box[0:2] * [new_w, new_h ] + [x_pad, y_pad]) / [self.width, self.height] box[2:4] = (box[2:4] * [new_w, new_h]) / [self.width, self.height] test_result.append([box[0]-box[2]/2, box[1]-box[3]/2, box[0]+box[2]/2, box[1]+box[3]/2]) best_giou = 0 best_index = 0 for i in range(len(anchors)): min_wh = np.minimum(box[2:4], anchors[i]) max_wh = np.maximum(box[2:4], anchors[i]) giou = (min_wh[0] * min_wh[1]) / (max_wh[0] * max_wh[1]) if giou > best_giou: best_giou = giou best_index = i # 012->0, 345->1, 678->2 x = int(np.floor(box[0] * self.width / ratio[best_index // 3])) y = int(np.floor(box[1] * self.height / ratio[best_index // 3])) k = best_index % 3 y_true[best_index // 3][y, x, k, 0:4] = box # label smooth label_value = 1.0 if not self.label_smooth else ((1-self.smooth_delta) + self.smooth_delta * 1 / self.class_num) y_true[best_index // 3][y, x, k, 4:5] = label_value y_true[best_index // 3][y, x, k, 5 + label_id] = label_value return label_y1, label_y2, label_y3, test_result
def read_label(self, label_file, anchors, new_w, new_h): ''' parsement label_file, and generates label_y1, label_y2, label_y3 return:label_y1, label_y2, label_y3 ''' contents = tools.read_file(label_file) if not contents: return None, None, None label_y1 = np.zeros( (self.height // 32, self.width // 32, 3, 5 + self.class_num), np.float32) label_y2 = np.zeros( (self.height // 16, self.width // 16, 3, 5 + self.class_num), np.float32) label_y3 = np.zeros( (self.height // 8, self.width // 8, 3, 5 + self.class_num), np.float32) delta = self.smooth_delta if delta: label_y1[:, :, :, 4] = delta / self.class_num label_y2[:, :, :, 4] = delta / self.class_num label_y3[:, :, :, 4] = delta / self.class_num y_true = [label_y3, label_y2, label_y1] ratio = {0: 8, 1: 16, 2: 32} test_result = [] for label in contents: label = label.split() if len(label) != 5: Log.add_log("error: in file '" + str(label_file) + "', the number of parameter does not match") raise ValueError( str(label_file) + ": the number of label parameter does not match") label_id = int(label[0]) box = np.asarray(label[1:5]).astype( np.float32) # what saved in label are x,y,w,h # flip the label if self.is_flip: box[0] = 1.0 - box[0] test_result.append([ box[0] - box[2] / 2, box[1] - box[3] / 2, box[0] + box[2] / 2, box[1] + box[3] / 2 ]) best_giou = 0 best_index = 0 for i in range(len(anchors)): min_wh = np.minimum(box[2:4], anchors[i]) max_wh = np.maximum(box[2:4], anchors[i]) giou = (min_wh[0] * min_wh[1]) / (max_wh[0] * max_wh[1]) if giou > best_giou: best_giou = giou best_index = i # 012->0, 345->1, 678->2 x = int(np.floor(box[0] * self.width / ratio[best_index // 3])) y = int(np.floor(box[1] * self.height / ratio[best_index // 3])) k = best_index % 3 y_true[best_index // 3][y, x, k, 0:4] = box # label smooth label_value = 1.0 if not delta else (1 - delta) y_true[best_index // 3][y, x, k, 4:5] = label_value y_true[best_index // 3][y, x, k, 5:] = delta / self.class_num y_true[best_index // 3][y, x, k, 5 + label_id] = label_value if self.is_flip: self.is_flip = False return label_y1, label_y2, label_y3, test_result