def filter_third_mods(): res = re.compile('(' + '|'.join(API.KEY_WORDS) + ')$') ren = re.compile('[+-](([1-9][.0-9]*[%]{0,1}))') with open(paths.get_path(idir='json/FilterSecondMods.json'), 'r', encoding='utf-8') as fp: data = json.load(fp) mods = list() for mod in data: description = mod.pop('description', None) if description: prop = {} for item in description.split(','): key = res.search(item) if key: value = ren.search(item) if value: prop[API.en_to_zh(key.group())] = value.group() if len(prop) > 0: mod['property'] = prop mod['type'] = re.sub(r' Mod', '', mod.get('type')) mods.append(mod) with open(paths.get_path(idir='json/FilterThridMods.json'), 'w', encoding='utf-8') as fp: json.dump(mods, fp)
def filter_frist_mods(): fmods = {} with open(paths.get_path(idir='json/Mods.json'), 'r', encoding='utf-8') as fp: data = json.load(fp) for item in data: key = item.get('name', None) mod = {} mod['name'] = item.get('name') mod['polarity'] = item.get('polarity') mod['rarity'] = item.get('rarity') mod['baseDrain'] = item.get('baseDrain') mod['fusionLimit'] = item.get('fusionLimit') mod['description'] = item.get('description') mod['type'] = item.get('type') if key is not None: fmods[key] = mod fmods = list(fmods.values()) pprint(fmods) with open(paths.get_path(idir='json/FilterMods.json'), 'w', encoding='utf-8') as fp: json.dump(fmods, fp)
def get_data(): # 创建驱动 driver_path = paths.get_path('chromedriver.exe', paths.get_now_path(__file__)) driver = webdriver.Chrome(executable_path=driver_path) # 打开访问页面 driver.get(url=API.ZH_EN_DICT) # 等待数据显示并爬取 try: myd = {} # 创建浏览器隐式等待 wait = WebDriverWait(driver, 10) # 等待元素显示 table = wait.until( EC.presence_of_all_elements_located(locator=( By.XPATH, '//*[@id="mw-content-text"]/div[3]/table/tbody/tr[1]/td/div/table/tbody/tr' ))) print(len(table)) trs = list(map(sp_str, table)) with open(paths.get_path(idir='json/zh.json'), 'w', encoding='utf-8') as fp: json.dump(trs, fp) finally: # 无论是否爬取成功关闭浏览器 driver.quit()
def extract_exifs(ds_dir, img_ids): files = [get_path(ds_dir, img_id, 'CR2') for img_id in img_ids] with exiftool.ExifTool() as et: metadata = et.get_metadata_batch(files) for m, f, img_id in zip(metadata, files, img_ids): assert m['SourceFile'] == f json_save(get_path(ds_dir, img_id, 'json_exif'), m)
def join_meta_info(ds_dir, img_ids): rows = [] gt_keys = [ 'has_mean_gt', 'left_tr_illuminance', 'right_tr_illuminance', 'left_white_tr_illuminance', 'right_white_tr_illuminance', ] markup_keys = [ 'ds_version', 'left_white_overexposed', 'right_white_overexposed' ] markup_properties_keys = [ 'daytime', 'place', 'illumination', 'is_sharp', 'shadows', 'richness', 'has_known_objects', 'light_objects' ] # and 'estimation', moved for sorting purposes exif_keys = [ 'MakerNotes:InternalSerialNumber', 'EXIF:ISO', 'EXIF:ApertureValue', 'EXIF:ExposureTime', 'MakerNotes:PerChannelBlackLevel', 'MakerNotes:NormalWhiteLevel', 'EXIF:Model', 'MakerNotes:LensModel' ] for img_id in img_ids: row = { 'image': img_id, } json_gt = json_load(get_path(ds_dir, img_id, 'json_gt')) for k in gt_keys: row[k] = json_gt[k] json_markup = json_load(get_path(ds_dir, img_id, 'json_markup')) for k in markup_keys: value = json_markup[k] row[k] = str(value) for k in ['estimation'] + markup_properties_keys: value = json_markup['properties'][k] row[k] = str( value ) # str for 'light_objects', which value is not str but list json_exif = json_load(get_path(ds_dir, img_id, 'json_exif')) for k in exif_keys: row[k] = json_exif[k] rows.append(row) res = pd.DataFrame(rows, columns=['image', 'estimation'] + markup_keys + gt_keys + markup_properties_keys + exif_keys) path = get_common_path(ds_dir, 'properties') res.sort_values('image').to_csv(path, index=False)
class SimilarityImg(object): """图片相似度计算""" temp_path_list = [ paths.get_path('opencv_imgs/user1.png'), paths.get_path('opencv_imgs/user2.png'), paths.get_path('opencv_imgs/user3.png'), ] def __init__(self, game_path): self.game_path = game_path def calculate(self, image1, image2): # 灰度直方图算法 # 计算单通道的直方图的相似值 hist1 = cv2.calcHist([image1], [0], None, [256], [0.0, 255.0]) hist2 = cv2.calcHist([image2], [0], None, [256], [0.0, 255.0]) # 计算直方图的重合度 degree = 0 for i in range(len(hist1)): if hist1[i] != hist2[i]: degree = degree + \ (1 - abs(hist1[i] - hist2[i]) / max(hist1[i], hist2[i])) else: degree = degree + 1 degree = degree / len(hist1) return degree def run(self): similarity_list = [] for path in self.temp_path_list: # 打开图片 print(path) tem_img = cv_imread(path) game_img = cv_imread(self.game_path) # 计算相似度 similarity = self.calculate(tem_img, game_img) if type(similarity) is float: # 完全相似时返回的为float similarity_list.append(similarity) else: # 返回的时nparray similarity_list.append(similarity[0]) for s in similarity_list: if s < 0.6: return False return True
def audit_riven(request): if request.user.is_authenticated: if request.user.game_name is not None: try: wfuser = request.user riven_name = request.POST.get('riven_name') properties = json.loads(request.POST.get('properties')) price = request.POST.get('price') file = request.FILES.get('file') name = file.name name_re = re.compile('(.png|.jpg)$', flags=re.IGNORECASE) img_type = name_re.search(name).group() if img_type: img_name = shortuuid.uuid() if os.path.exists( paths.get_path(f'media/wfusers/{wfuser.uid}/rivens' )) is False: os.makedirs( paths.get_path( f'media/wfusers/{wfuser.uid}/rivens')) img_path = paths.get_path( f'media/wfusers/{wfuser.uid}/rivens/{img_name}{img_type}' ) with open(img_path, 'wb') as imgfp: for chunk in file.chunks(): imgfp.write(chunk) Riven.objects.create( riven_name=riven_name, properties=properties, price=price, is_sell=SellState.IS_AUDIT.value, thumbnail= f'/media/wfusers/{wfuser.uid}/rivens/{img_name}{img_type}', seller=wfuser) return restful.ok(message='提交审核成功') else: return restful.parameters_error( '请上传正确的格式的图片,后台只接受PNG与JPG图片。') except: return restful.server_error('上架紫卡失败,请稍后再试。') else: return restful.server_error('需要通过游戏ID审核才能发布紫卡。') else: return restful.authority_error('需要登录才能发布紫卡。')
def add_self_children_msg(request): wfuser = request.user if wfuser.game_name is not None: try: msg = request.POST.get('msg') parent_msg_id = request.POST.get('parent_msg_id') print(msg) gfw = DFAFilter() path = paths.get_path(f'sensitive_words.txt') gfw.parse(path) msg = gfw.filter(msg) parent_msg = RivenMsg.objects.filter(id=parent_msg_id).first() riven_id = parent_msg.riven_id RivenMsg.objects.create(view_state=True, content=msg, riven_id=riven_id, parent_id=parent_msg_id, writer=wfuser) return restful.ok('回复留言成功') except: return restful.ok('回复留言失败') else: return restful.authority_error('需要通过审核游戏ID后才能回复留言。')
def write_msg(request): msg = request.POST.get('msg') riven_id = request.POST.get('riven_id') gfw = DFAFilter() path = paths.get_path(f'sensitive_words.txt') gfw.parse(path) msg = gfw.filter(msg) wfuser = request.user if wfuser.is_authenticated: if wfuser.game_name is not None: try: RivenMsg.objects.create(content=msg, view_state=False, riven_id=riven_id, writer=wfuser) print('留言成功') return restful.ok('留言成功') except: return restful.server_error('发表留言失败,请稍后再试。') else: return restful.authority_error('需要通过审核游戏ID后才能留言。') else: return restful.authority_error('请登录后留言。')
def add_wfuser_gamename(wfuser, user_game): """图片识别用户游戏昵称""" try: img_path = paths.get_path(re.sub('^/', '', user_game.first_img)) s = SimilarityImg(img_path) result = s.run() if result: r = RecognitionImg(img_path) name_img = r.run() oct_img = cv2.cvtColor(name_img, cv2.COLOR_BGR2GRAY) text = tocr.image_to_string(oct_img, lang='eng') user_game.game_status = WfUserGame.HAVE_GAMENAME wfuser.game_name = text.strip() else: user_game.game_status = WfUserGame.NO_GAMENAME wfuser.game_name = '' except: user_game.game_status = WfUserGame.NO_GAMENAME wfuser.game_name = None finally: user_game.save() wfuser.save()
def calc_exif_statistics(ds_dir, img_ids): """Calculates some extra statistics about exifs, not used in the main ds part""" feature_vals = {} for img_id in img_ids: exif = json_load(get_path(ds_dir, img_id, 'json_exif')) for f, v in exif.items(): if f not in feature_vals: feature_vals[f] = [] feature_vals[f].append(v) rows = [] for f in sorted(feature_vals.keys()): vals = feature_vals[f] row = { 'feature': f, 'vals_count': len(vals), 'uniq_vals_count': len(set(vals)), } counter = Counter(vals) MAX_LENGTH = 3 for i, (v, _) in enumerate(counter.most_common(MAX_LENGTH)): row['top_{}_common_value'.format(i + 1)] = v rows.append(row) res = pd.DataFrame(rows, columns=[ 'feature', 'vals_count', 'uniq_vals_count', 'top_1_common_value', 'top_2_common_value', 'top_3_common_value' ]) path = get_common_path(ds_dir, 'exif_stat') res.sort_values('feature').to_csv(path, index=False)
def filter_second_mods(): prog = re.compile( r'(Warframe|Shotgun|Rifle|Primary|Aura|Secondary|Melee|Stance)') with open(paths.get_path(idir='json/FilterMods.json'), 'r', encoding='utf-8') as fp: data = json.load(fp) mods = list() for item in data: if prog.match(item.get('type')) is not None: mods.append(item) with open(paths.get_path(idir='json/FilterSecondMods.json'), 'w', encoding='utf-8') as fp: json.dump(mods, fp)
def update_imgs(): ws = Warframe.objects.all() imgs_name = os.listdir(paths.get_path('media/warframes')) for warframe in ws: name = warframe.name name = re.sub('\s', "", name) img_name = list(filter(lambda x: re.search(name, x), imgs_name))[0] warframe.img = f'/media/warframes/{img_name}' Warframe.objects.bulk_update(ws, ['img'])
def build_url(): urls: Dict[str, str] = {} with open(paths.get_path(idir='json/imgs.json'), 'r', encoding='utf-8') as fp: imgs = json.load(fp) for name in imgs.keys(): urls[name] = API.WIKI.format(url=name) return urls
def migrate_dict(self): with open(paths.get_path('json/translation/en_to_zh.json'), 'r', encoding='utf-8') as f: data = json.load(f) thesaurus_list: list = [] for en, zh in data.items(): new_en = en.lower().strip() thesaurus = Thesaurus(en_key=new_en, zh_value=zh) thesaurus_list.append(thesaurus) thesaurus_list.reverse() Thesaurus.objects.bulk_create(thesaurus_list)
def updata_name(): mods = Mod.objects.all() with open(paths.get_path(idir='json/zh.json'), 'r', encoding='utf-8') as fp: en_to_zh = json.load(fp) for mod in mods: name = mod.name name = en_to_zh.get(name, None) if name: mod.name = name pprint(list(mods)) Mod.objects.bulk_update(mods, ['name'])
def bulid_img(name, url, referer): http = urllib3.PoolManager() header = { 'Referer': f'{referer}', 'Sec-Fetch-Mode': 'no-cors', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36' } r = http.request('GET', url=url, headers=header) with open(paths.get_path(idir=f'media/warframes/{name}'), 'wb') as fp: fp.write(r.data)
def get_img(): """获取warframe图片地址""" # 创建驱动 driver_path = paths.get_path('chromedriver.exe', paths.get_now_path(__file__)) driver = webdriver.Chrome(executable_path=driver_path) # 打开访问页面 driver.get(url=API.WARFRAME_IMGS) # 等待数据显示并爬取 try: # 创建浏览器隐式等待 wait = WebDriverWait(driver, 10) # 等待元素显示 imgs = wait.until( EC.presence_of_all_elements_located( locator=(By.XPATH, '//*[@id="mw-content-text"]/div[3]/div[2]/div/a'))) print(len(imgs)) w_imgs = {} for a in imgs: name = a.get_attribute('title') img_tag = a.find_element_by_tag_name('IMG') img_path = img_tag.get_attribute('src') w_imgs[name.strip()] = img_path with open(paths.get_path(idir='json/imgs.json'), 'w', encoding='utf-8') as fp: json.dump(w_imgs, fp) finally: # 无论是否爬取成功关闭浏览器 driver.quit()
def download_img(): """下载warframe图片""" with open(paths.get_path(idir='json/Warframes.json'), 'r', encoding='utf-8') as fp: if fp: data = json.load(fp) names = set() for warframe in data: name = warframe.get('name', None) name = re.sub(r'(Prime|prime)', r'', name).strip() names.add(name) print(names) print(len(names))
def create_warframes(): ws = list() with open(paths.get_path(idir='json/Warframes.json'), 'r', encoding='utf-8') as fp: data = json.load(fp) for item in data: w = Warframe() w.name = item.get('name') w.health = item.get('health') w.shield = item.get('shield') w.armor = item.get('armor') w.power = item.get('power') w.sprint_speed = item.get('sprintSpeed') ws.append(w) pprint(ws)
def get_weapons_urls(self): response = requests.get(url=self.en_wiki.format(url='/wiki/Weapons')) soup = BeautifulSoup(response.text, 'lxml') div = soup.find('div', class_='tabbertab', title='Primary') spans = div.find_all('span', class_='weapon-tooltip') weapons_set: set = set() urls: dict = {} for span in spans: href = span.next.get('href') name = span.get('data-param') if name not in weapons_set: url_dict = {} weapons_set.add(name) url_dict[name] = href self.urls_queue.put(url_dict) urls[name] = href with open(paths.get_path('json/weapons/Weapons_urls.json'), 'w', encoding='utf-8') as fp: json.dump(urls, fp)
def create_mods(): mods = list() with open(paths.get_path(idir='json/FilterThridMods.json'), 'r', encoding='utf-8') as fp: data = json.load(fp) for mod in data: m = Mod() m.name = mod.get('name') m.polarity = mod.get('polarity') m.rarity = mod.get('rarity') m.baseDrain = mod.get('baseDrain') m.fusionLimit = mod.get('fusionLimit') m.modType = mod.get('type') m.property = mod.get('property') mods.append(m) pprint(mods) Mod.objects.bulk_create(mods)
def join_gt_info(ds_dir, img_ids): rows = [] keys = ['mean', 'left', 'right', 'left_white', 'right_white'] for img_id in img_ids: json_gt = json_load(get_path(ds_dir, img_id, 'json_gt')) row = {'image': img_id} for key in keys: for i, c in enumerate('rgb'): row['{}_{}'.format(key, c)] = json_gt[key][i] rows.append(row) res = pd.DataFrame(rows, columns=[ 'image', 'mean_r', 'mean_g', 'mean_b', 'left_r', 'left_g', 'left_b', 'right_r', 'right_g', 'right_b', 'left_white_r', 'left_white_g', 'left_white_b', 'right_white_r', 'right_white_g', 'right_white_b' ]) path = get_common_path(ds_dir, 'gt') res.sort_values('image').to_csv(path, index=False)
def join_camera_estimation_info(ds_dir, img_ids): rows = [] exif_keys = [ 'Composite:BlueBalance', 'Composite:RedBalance', 'Composite:LightValue', 'MakerNotes:ColorTempMeasured', 'MakerNotes:ColorTempAsShot' ] for img_id in img_ids: row = { 'image': img_id, } json_exif = json_load(get_path(ds_dir, img_id, 'json_exif')) for k in exif_keys: row[k] = json_exif[k] rows.append(row) res = pd.DataFrame(rows, columns=['image'] + exif_keys) path = get_common_path(ds_dir, 'cam_estimation') res.sort_values('image').to_csv(path, index=False)
class RecognitionImg(object): """游戏昵称识别""" temp_path = paths.get_path('opencv_imgs/user_biaozhi.png') keypoint_num = 10 def __init__(self, game_path): self.game_path = game_path def run(self): """识别游戏截图""" tem_img = cv_imread(self.temp_path) game_img = cv_imread(self.game_path) # 特征提取 (kps_t, des_t) = self.detect_and_describe(tem_img) (kps_g, des_g) = self.detect_and_describe(game_img) # 获取相似特征 (matches, matches_keypoints) = self.match_keypoints(des_t, des_g) # 最佳匹配特征点 (good_kps_t, good_kps_g) = self.find_good_keypoints( matches_keypoints[:self.keypoint_num], kps_t, kps_g) # 最佳匹配区域 rectangle_t = self.get_rectangle(good_kps_t) rectangle_g = self.get_rectangle(good_kps_g) rectangle = self.get_rectangle_tem(rectangle_t, rectangle_g, tem_img) end_img = self.incision_img(game_img, rectangle) return end_img def detect_and_describe(self, img): """提取特征与特征点""" # 图片灰度化 gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) # sift实例化 sift = cv2.xfeatures2d.SIFT_create() # 特征提取与计算 (kps, des) = sift.detectAndCompute(gray, None) return (kps, des) def match_keypoints(self, des_t, des_g): """""" # 初始化bf bf = cv2.BFMatcher() # 特征匹配 matches = bf.match(des_t, des_g) matches = sorted(matches, key=lambda x: x.distance) matches_keypoints = [] for m in matches: matches_keypoints.append((m.queryIdx, m.trainIdx)) return (matches, matches_keypoints) def find_good_keypoints(self, matches, kps_t, kps_g): kpt = np.array([kps_t[i] for (i, _) in matches]) kpg = np.array([kps_g[i] for (_, i) in matches]) return (kpt, kpg) def get_rectangle(self, kps): rectangle = {'left': 10000, 'right': 0, 'top': 10000, 'bottom': 0} for kp in kps: x, y = kp.pt x, y = round(x), round(y) if rectangle['left'] > x: rectangle['left'] = x if rectangle['right'] < x: rectangle['right'] = x if rectangle['top'] > y: rectangle['top'] = y if rectangle['bottom'] < y: rectangle['bottom'] = y return rectangle def get_rectangle_tem(self, rt, rg, imgt): """""" height, width = imgt.shape[:2] height = height - rt['bottom'] width = width - rt['right'] return { 'left': rg['left'] - rt['left'], 'right': rg['right'] + width, 'top': rg['top'] - rt['top'], 'bottom': rg['bottom'] + height } def trans_img(self, img, prop=1): """等比例缩小图片""" height, width = img.shape[:2] height = round(height * prop) width = round(width * prop) size = (width, height) t_img = cv2.resize(img, size) return t_img def incision_img(self, img, rectangle): """切取图片名称部分""" y_start = rectangle['top'] - 50 y_end = rectangle['top'] x_start = rectangle['left'] x_end = rectangle['right'] cut = img[y_start:y_end, x_start:x_end] return cut
def path(data_type): return get_path(self.ds_dir, img_id, data_type)
def upload_file(request): try: file = request.FILES.get('file') name = file.name name_re = re.compile('(.png|.jpg)$', flags=re.IGNORECASE) if name_re.search(name): wfuser = request.user user_game, judge = WfUserGame.objects.get_or_create(wfuser=wfuser) if judge: img_name = f'{wfuser.username}+{wfuser.uid}{name_re.search(name).group()}' if os.path.exists(paths.get_path( f'media/wfusers/{wfuser.uid}')) is False: os.makedirs(paths.get_path(f'media/wfusers/{wfuser.uid}')) with open( paths.get_path( f'media/wfusers/{wfuser.uid}/{img_name}'), 'wb') as imgfp: for chunk in file.chunks(): imgfp.write(chunk) user_game.first_img = f'/media/wfusers/{wfuser.uid}/{img_name}' user_game.game_status = WfUserGame.AUDIT_STATUS user_game.save() add_img_thread = threading.Thread(target=add_wfuser_gamename, args=(wfuser, user_game)) add_img_thread.start() return restful.ok('后台以开始审核,请注意查看状态。') else: img_dict = user_game.other_imgs if img_dict: num = len(img_dict) + 1 img_name = f'{wfuser.username}+{wfuser.uid}_{num}{name_re.search(name).group()}' if os.path.exists( paths.get_path( f'media/wfusers/{wfuser.uid}')) is False: os.makedirs( paths.get_path(f'media/wfusers/{wfuser.uid}')) with open( paths.get_path( f'media/wfusers/{wfuser.uid}/{img_name}'), 'wb') as imgfp: for chunk in file.chunks(): imgfp.write(chunk) img_dict[ f'{num}'] = f'/media/wfusers/{wfuser.uid}/{img_name}' user_game.game_status = WfUserGame.MANUAL_STATUS user_game.save() wfuser.game_name = None wfuser.save() return restful.ok('后台已开始审核,请注意查看状态。') else: img_name = f'{wfuser.username}+{wfuser.uid}_1{name_re.search(name).group()}' if os.path.exists( paths.get_path( f'media/wfusers/{wfuser.uid}')) is False: os.makedirs( paths.get_path(f'media/wfusers/{wfuser.uid}')) with open( paths.get_path( f'media/wfusers/{wfuser.uid}/{img_name}'), 'wb') as imgfp: for chunk in file.chunks(): imgfp.write(chunk) img_dict['1'] = f'/media/wfusers/{wfuser.uid}/{img_name}' user_game.game_status = WfUserGame.MANUAL_STATUS user_game.save() wfuser.game_name = None wfuser.save() return restful.ok('后台已开始审核,请注意查看状态。') else: return restful.parameters_error('请上传正确的格式的图片,后台只接受PNG与JPG图片。') except: return restful.server_error('服务器处理请求失败,请稍后再试。')
def get_weapons_data(self): with open(paths.get_path('json/weapons/Weapons.json'), 'r', encoding='utf-8') as f: weapons = json.load(f) for weapon in weapons: self.weapons_queue.put(weapon)
def save_weapons(self): """保存""" with open(paths.get_path('json/weapons/Weapons.json'), 'w', encoding='utf-8') as f: json.dump(self.weapons, f)
def get_data(): """获取战甲信息""" http = urllib3.PoolManager() r = http.request('GET', API.WARFRAME_WARFRAMES) with open(paths.get_path(idir='json/Warframes.json'), 'wb') as fp: fp.write(r.data)