def filter_third_mods():
res = re.compile('(' + '|'.join(API.KEY_WORDS) + ')$')
ren = re.compile('[+-](([1-9][.0-9]*[%]{0,1}))')
with open(paths.get_path(idir='json/FilterSecondMods.json'),
'r',
encoding='utf-8') as fp:
data = json.load(fp)
mods = list()
for mod in data:
description = mod.pop('description', None)
if description:
prop = {}
for item in description.split(','):
key = res.search(item)
if key:
value = ren.search(item)
if value:
prop[API.en_to_zh(key.group())] = value.group()
if len(prop) > 0:
mod['property'] = prop
mod['type'] = re.sub(r' Mod', '', mod.get('type'))
mods.append(mod)
with open(paths.get_path(idir='json/FilterThridMods.json'),
'w',
encoding='utf-8') as fp:
json.dump(mods, fp)
def filter_frist_mods():
fmods = {}
with open(paths.get_path(idir='json/Mods.json'), 'r',
encoding='utf-8') as fp:
data = json.load(fp)
for item in data:
key = item.get('name', None)
mod = {}
mod['name'] = item.get('name')
mod['polarity'] = item.get('polarity')
mod['rarity'] = item.get('rarity')
mod['baseDrain'] = item.get('baseDrain')
mod['fusionLimit'] = item.get('fusionLimit')
mod['description'] = item.get('description')
mod['type'] = item.get('type')
if key is not None:
fmods[key] = mod
fmods = list(fmods.values())
pprint(fmods)
with open(paths.get_path(idir='json/FilterMods.json'),
'w',
encoding='utf-8') as fp:
json.dump(fmods, fp)
def get_data():
# 创建驱动
driver_path = paths.get_path('chromedriver.exe',
paths.get_now_path(__file__))
driver = webdriver.Chrome(executable_path=driver_path)
# 打开访问页面
driver.get(url=API.ZH_EN_DICT)
# 等待数据显示并爬取
try:
myd = {}
# 创建浏览器隐式等待
wait = WebDriverWait(driver, 10)
# 等待元素显示
table = wait.until(
EC.presence_of_all_elements_located(locator=(
By.XPATH,
'//*[@id="mw-content-text"]/div[3]/table/tbody/tr[1]/td/div/table/tbody/tr'
)))
print(len(table))
trs = list(map(sp_str, table))
with open(paths.get_path(idir='json/zh.json'), 'w',
encoding='utf-8') as fp:
json.dump(trs, fp)
finally:
# 无论是否爬取成功关闭浏览器
driver.quit()
def extract_exifs(ds_dir, img_ids):
files = [get_path(ds_dir, img_id, 'CR2') for img_id in img_ids]
with exiftool.ExifTool() as et:
metadata = et.get_metadata_batch(files)
for m, f, img_id in zip(metadata, files, img_ids):
assert m['SourceFile'] == f
json_save(get_path(ds_dir, img_id, 'json_exif'), m)
def join_meta_info(ds_dir, img_ids):
rows = []
gt_keys = [
'has_mean_gt',
'left_tr_illuminance',
'right_tr_illuminance',
'left_white_tr_illuminance',
'right_white_tr_illuminance',
]
markup_keys = [
'ds_version', 'left_white_overexposed', 'right_white_overexposed'
]
markup_properties_keys = [
'daytime', 'place', 'illumination', 'is_sharp', 'shadows', 'richness',
'has_known_objects', 'light_objects'
] # and 'estimation', moved for sorting purposes
exif_keys = [
'MakerNotes:InternalSerialNumber', 'EXIF:ISO', 'EXIF:ApertureValue',
'EXIF:ExposureTime', 'MakerNotes:PerChannelBlackLevel',
'MakerNotes:NormalWhiteLevel', 'EXIF:Model', 'MakerNotes:LensModel'
]
for img_id in img_ids:
row = {
'image': img_id,
}
json_gt = json_load(get_path(ds_dir, img_id, 'json_gt'))
for k in gt_keys:
row[k] = json_gt[k]
json_markup = json_load(get_path(ds_dir, img_id, 'json_markup'))
for k in markup_keys:
value = json_markup[k]
row[k] = str(value)
for k in ['estimation'] + markup_properties_keys:
value = json_markup['properties'][k]
row[k] = str(
value
) # str for 'light_objects', which value is not str but list
json_exif = json_load(get_path(ds_dir, img_id, 'json_exif'))
for k in exif_keys:
row[k] = json_exif[k]
rows.append(row)
res = pd.DataFrame(rows,
columns=['image', 'estimation'] + markup_keys +
gt_keys + markup_properties_keys + exif_keys)
path = get_common_path(ds_dir, 'properties')
res.sort_values('image').to_csv(path, index=False)
class SimilarityImg(object):
"""图片相似度计算"""
temp_path_list = [
paths.get_path('opencv_imgs/user1.png'),
paths.get_path('opencv_imgs/user2.png'),
paths.get_path('opencv_imgs/user3.png'),
]
def __init__(self, game_path):
self.game_path = game_path
def calculate(self, image1, image2):
# 灰度直方图算法
# 计算单通道的直方图的相似值
hist1 = cv2.calcHist([image1], [0], None, [256], [0.0, 255.0])
hist2 = cv2.calcHist([image2], [0], None, [256], [0.0, 255.0])
# 计算直方图的重合度
degree = 0
for i in range(len(hist1)):
if hist1[i] != hist2[i]:
degree = degree + \
(1 - abs(hist1[i] - hist2[i]) / max(hist1[i], hist2[i]))
else:
degree = degree + 1
degree = degree / len(hist1)
return degree
def run(self):
similarity_list = []
for path in self.temp_path_list:
# 打开图片
print(path)
tem_img = cv_imread(path)
game_img = cv_imread(self.game_path)
# 计算相似度
similarity = self.calculate(tem_img, game_img)
if type(similarity) is float:
# 完全相似时返回的为float
similarity_list.append(similarity)
else:
# 返回的时nparray
similarity_list.append(similarity[0])
for s in similarity_list:
if s < 0.6:
return False
return True
def audit_riven(request):
if request.user.is_authenticated:
if request.user.game_name is not None:
try:
wfuser = request.user
riven_name = request.POST.get('riven_name')
properties = json.loads(request.POST.get('properties'))
price = request.POST.get('price')
file = request.FILES.get('file')
name = file.name
name_re = re.compile('(.png|.jpg)$', flags=re.IGNORECASE)
img_type = name_re.search(name).group()
if img_type:
img_name = shortuuid.uuid()
if os.path.exists(
paths.get_path(f'media/wfusers/{wfuser.uid}/rivens'
)) is False:
os.makedirs(
paths.get_path(
f'media/wfusers/{wfuser.uid}/rivens'))
img_path = paths.get_path(
f'media/wfusers/{wfuser.uid}/rivens/{img_name}{img_type}'
)
with open(img_path, 'wb') as imgfp:
for chunk in file.chunks():
imgfp.write(chunk)
Riven.objects.create(
riven_name=riven_name,
properties=properties,
price=price,
is_sell=SellState.IS_AUDIT.value,
thumbnail=
f'/media/wfusers/{wfuser.uid}/rivens/{img_name}{img_type}',
seller=wfuser)
return restful.ok(message='提交审核成功')
else:
return restful.parameters_error(
'请上传正确的格式的图片，后台只接受PNG与JPG图片。')
except:
return restful.server_error('上架紫卡失败，请稍后再试。')
else:
return restful.server_error('需要通过游戏ID审核才能发布紫卡。')
else:
return restful.authority_error('需要登录才能发布紫卡。')
def add_self_children_msg(request):
wfuser = request.user
if wfuser.game_name is not None:
try:
msg = request.POST.get('msg')
parent_msg_id = request.POST.get('parent_msg_id')
print(msg)
gfw = DFAFilter()
path = paths.get_path(f'sensitive_words.txt')
gfw.parse(path)
msg = gfw.filter(msg)
parent_msg = RivenMsg.objects.filter(id=parent_msg_id).first()
riven_id = parent_msg.riven_id
RivenMsg.objects.create(view_state=True,
content=msg,
riven_id=riven_id,
parent_id=parent_msg_id,
writer=wfuser)
return restful.ok('回复留言成功')
except:
return restful.ok('回复留言失败')
else:
return restful.authority_error('需要通过审核游戏ID后才能回复留言。')
def write_msg(request):
msg = request.POST.get('msg')
riven_id = request.POST.get('riven_id')
gfw = DFAFilter()
path = paths.get_path(f'sensitive_words.txt')
gfw.parse(path)
msg = gfw.filter(msg)
wfuser = request.user
if wfuser.is_authenticated:
if wfuser.game_name is not None:
try:
RivenMsg.objects.create(content=msg,
view_state=False,
riven_id=riven_id,
writer=wfuser)
print('留言成功')
return restful.ok('留言成功')
except:
return restful.server_error('发表留言失败，请稍后再试。')
else:
return restful.authority_error('需要通过审核游戏ID后才能留言。')
else:
return restful.authority_error('请登录后留言。')
def add_wfuser_gamename(wfuser, user_game):
"""图片识别用户游戏昵称"""
try:
img_path = paths.get_path(re.sub('^/', '', user_game.first_img))
s = SimilarityImg(img_path)
result = s.run()
if result:
r = RecognitionImg(img_path)
name_img = r.run()
oct_img = cv2.cvtColor(name_img, cv2.COLOR_BGR2GRAY)
text = tocr.image_to_string(oct_img, lang='eng')
user_game.game_status = WfUserGame.HAVE_GAMENAME
wfuser.game_name = text.strip()
else:
user_game.game_status = WfUserGame.NO_GAMENAME
wfuser.game_name = ''
except:
user_game.game_status = WfUserGame.NO_GAMENAME
wfuser.game_name = None
finally:
user_game.save()
wfuser.save()
def calc_exif_statistics(ds_dir, img_ids):
"""Calculates some extra statistics about exifs, not used in the main ds part"""
feature_vals = {}
for img_id in img_ids:
exif = json_load(get_path(ds_dir, img_id, 'json_exif'))
for f, v in exif.items():
if f not in feature_vals:
feature_vals[f] = []
feature_vals[f].append(v)
rows = []
for f in sorted(feature_vals.keys()):
vals = feature_vals[f]
row = {
'feature': f,
'vals_count': len(vals),
'uniq_vals_count': len(set(vals)),
}
counter = Counter(vals)
MAX_LENGTH = 3
for i, (v, _) in enumerate(counter.most_common(MAX_LENGTH)):
row['top_{}_common_value'.format(i + 1)] = v
rows.append(row)
res = pd.DataFrame(rows,
columns=[
'feature', 'vals_count', 'uniq_vals_count',
'top_1_common_value', 'top_2_common_value',
'top_3_common_value'
])
path = get_common_path(ds_dir, 'exif_stat')
res.sort_values('feature').to_csv(path, index=False)
def filter_second_mods():
prog = re.compile(
r'(Warframe|Shotgun|Rifle|Primary|Aura|Secondary|Melee|Stance)')
with open(paths.get_path(idir='json/FilterMods.json'),
'r',
encoding='utf-8') as fp:
data = json.load(fp)
mods = list()
for item in data:
if prog.match(item.get('type')) is not None:
mods.append(item)
with open(paths.get_path(idir='json/FilterSecondMods.json'),
'w',
encoding='utf-8') as fp:
json.dump(mods, fp)
def update_imgs():
ws = Warframe.objects.all()
imgs_name = os.listdir(paths.get_path('media/warframes'))
for warframe in ws:
name = warframe.name
name = re.sub('\s', "", name)
img_name = list(filter(lambda x: re.search(name, x), imgs_name))[0]
warframe.img = f'/media/warframes/{img_name}'
Warframe.objects.bulk_update(ws, ['img'])
def build_url():
urls: Dict[str, str] = {}
with open(paths.get_path(idir='json/imgs.json'), 'r',
encoding='utf-8') as fp:
imgs = json.load(fp)
for name in imgs.keys():
urls[name] = API.WIKI.format(url=name)
return urls
def migrate_dict(self):
with open(paths.get_path('json/translation/en_to_zh.json'), 'r', encoding='utf-8') as f:
data = json.load(f)
thesaurus_list: list = []
for en, zh in data.items():
new_en = en.lower().strip()
thesaurus = Thesaurus(en_key=new_en, zh_value=zh)
thesaurus_list.append(thesaurus)
thesaurus_list.reverse()
Thesaurus.objects.bulk_create(thesaurus_list)
def updata_name():
mods = Mod.objects.all()
with open(paths.get_path(idir='json/zh.json'), 'r',
encoding='utf-8') as fp:
en_to_zh = json.load(fp)
for mod in mods:
name = mod.name
name = en_to_zh.get(name, None)
if name:
mod.name = name
pprint(list(mods))
Mod.objects.bulk_update(mods, ['name'])
def bulid_img(name, url, referer):
http = urllib3.PoolManager()
header = {
'Referer':
f'{referer}',
'Sec-Fetch-Mode':
'no-cors',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36'
}
r = http.request('GET', url=url, headers=header)
with open(paths.get_path(idir=f'media/warframes/{name}'), 'wb') as fp:
fp.write(r.data)
def get_img():
"""获取warframe图片地址"""
# 创建驱动
driver_path = paths.get_path('chromedriver.exe',
paths.get_now_path(__file__))
driver = webdriver.Chrome(executable_path=driver_path)
# 打开访问页面
driver.get(url=API.WARFRAME_IMGS)
# 等待数据显示并爬取
try:
# 创建浏览器隐式等待
wait = WebDriverWait(driver, 10)
# 等待元素显示
imgs = wait.until(
EC.presence_of_all_elements_located(
locator=(By.XPATH,
'//*[@id="mw-content-text"]/div[3]/div[2]/div/a')))
print(len(imgs))
w_imgs = {}
for a in imgs:
name = a.get_attribute('title')
img_tag = a.find_element_by_tag_name('IMG')
img_path = img_tag.get_attribute('src')
w_imgs[name.strip()] = img_path
with open(paths.get_path(idir='json/imgs.json'), 'w',
encoding='utf-8') as fp:
json.dump(w_imgs, fp)
finally:
# 无论是否爬取成功关闭浏览器
driver.quit()
def download_img():
"""下载warframe图片"""
with open(paths.get_path(idir='json/Warframes.json'),
'r',
encoding='utf-8') as fp:
if fp:
data = json.load(fp)
names = set()
for warframe in data:
name = warframe.get('name', None)
name = re.sub(r'(Prime|prime)', r'', name).strip()
names.add(name)
print(names)
print(len(names))
def create_warframes():
ws = list()
with open(paths.get_path(idir='json/Warframes.json'),
'r',
encoding='utf-8') as fp:
data = json.load(fp)
for item in data:
w = Warframe()
w.name = item.get('name')
w.health = item.get('health')
w.shield = item.get('shield')
w.armor = item.get('armor')
w.power = item.get('power')
w.sprint_speed = item.get('sprintSpeed')
ws.append(w)
pprint(ws)
def get_weapons_urls(self):
response = requests.get(url=self.en_wiki.format(url='/wiki/Weapons'))
soup = BeautifulSoup(response.text, 'lxml')
div = soup.find('div', class_='tabbertab', title='Primary')
spans = div.find_all('span', class_='weapon-tooltip')
weapons_set: set = set()
urls: dict = {}
for span in spans:
href = span.next.get('href')
name = span.get('data-param')
if name not in weapons_set:
url_dict = {}
weapons_set.add(name)
url_dict[name] = href
self.urls_queue.put(url_dict)
urls[name] = href
with open(paths.get_path('json/weapons/Weapons_urls.json'), 'w', encoding='utf-8') as fp:
json.dump(urls, fp)
def create_mods():
mods = list()
with open(paths.get_path(idir='json/FilterThridMods.json'),
'r',
encoding='utf-8') as fp:
data = json.load(fp)
for mod in data:
m = Mod()
m.name = mod.get('name')
m.polarity = mod.get('polarity')
m.rarity = mod.get('rarity')
m.baseDrain = mod.get('baseDrain')
m.fusionLimit = mod.get('fusionLimit')
m.modType = mod.get('type')
m.property = mod.get('property')
mods.append(m)
pprint(mods)
Mod.objects.bulk_create(mods)
def join_gt_info(ds_dir, img_ids):
rows = []
keys = ['mean', 'left', 'right', 'left_white', 'right_white']
for img_id in img_ids:
json_gt = json_load(get_path(ds_dir, img_id, 'json_gt'))
row = {'image': img_id}
for key in keys:
for i, c in enumerate('rgb'):
row['{}_{}'.format(key, c)] = json_gt[key][i]
rows.append(row)
res = pd.DataFrame(rows,
columns=[
'image', 'mean_r', 'mean_g', 'mean_b', 'left_r',
'left_g', 'left_b', 'right_r', 'right_g', 'right_b',
'left_white_r', 'left_white_g', 'left_white_b',
'right_white_r', 'right_white_g', 'right_white_b'
])
path = get_common_path(ds_dir, 'gt')
res.sort_values('image').to_csv(path, index=False)
def join_camera_estimation_info(ds_dir, img_ids):
rows = []
exif_keys = [
'Composite:BlueBalance', 'Composite:RedBalance',
'Composite:LightValue', 'MakerNotes:ColorTempMeasured',
'MakerNotes:ColorTempAsShot'
]
for img_id in img_ids:
row = {
'image': img_id,
}
json_exif = json_load(get_path(ds_dir, img_id, 'json_exif'))
for k in exif_keys:
row[k] = json_exif[k]
rows.append(row)
res = pd.DataFrame(rows, columns=['image'] + exif_keys)
path = get_common_path(ds_dir, 'cam_estimation')
res.sort_values('image').to_csv(path, index=False)
class RecognitionImg(object):
"""游戏昵称识别"""
temp_path = paths.get_path('opencv_imgs/user_biaozhi.png')
keypoint_num = 10
def __init__(self, game_path):
self.game_path = game_path
def run(self):
"""识别游戏截图"""
tem_img = cv_imread(self.temp_path)
game_img = cv_imread(self.game_path)
# 特征提取
(kps_t, des_t) = self.detect_and_describe(tem_img)
(kps_g, des_g) = self.detect_and_describe(game_img)
# 获取相似特征
(matches, matches_keypoints) = self.match_keypoints(des_t, des_g)
# 最佳匹配特征点
(good_kps_t, good_kps_g) = self.find_good_keypoints(
matches_keypoints[:self.keypoint_num], kps_t, kps_g)
# 最佳匹配区域
rectangle_t = self.get_rectangle(good_kps_t)
rectangle_g = self.get_rectangle(good_kps_g)
rectangle = self.get_rectangle_tem(rectangle_t, rectangle_g, tem_img)
end_img = self.incision_img(game_img, rectangle)
return end_img
def detect_and_describe(self, img):
"""提取特征与特征点"""
# 图片灰度化
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# sift实例化
sift = cv2.xfeatures2d.SIFT_create()
# 特征提取与计算
(kps, des) = sift.detectAndCompute(gray, None)
return (kps, des)
def match_keypoints(self, des_t, des_g):
""""""
# 初始化bf
bf = cv2.BFMatcher()
# 特征匹配
matches = bf.match(des_t, des_g)
matches = sorted(matches, key=lambda x: x.distance)
matches_keypoints = []
for m in matches:
matches_keypoints.append((m.queryIdx, m.trainIdx))
return (matches, matches_keypoints)
def find_good_keypoints(self, matches, kps_t, kps_g):
kpt = np.array([kps_t[i] for (i, _) in matches])
kpg = np.array([kps_g[i] for (_, i) in matches])
return (kpt, kpg)
def get_rectangle(self, kps):
rectangle = {'left': 10000, 'right': 0, 'top': 10000, 'bottom': 0}
for kp in kps:
x, y = kp.pt
x, y = round(x), round(y)
if rectangle['left'] > x:
rectangle['left'] = x
if rectangle['right'] < x:
rectangle['right'] = x
if rectangle['top'] > y:
rectangle['top'] = y
if rectangle['bottom'] < y:
rectangle['bottom'] = y
return rectangle
def get_rectangle_tem(self, rt, rg, imgt):
""""""
height, width = imgt.shape[:2]
height = height - rt['bottom']
width = width - rt['right']
return {
'left': rg['left'] - rt['left'],
'right': rg['right'] + width,
'top': rg['top'] - rt['top'],
'bottom': rg['bottom'] + height
}
def trans_img(self, img, prop=1):
"""等比例缩小图片"""
height, width = img.shape[:2]
height = round(height * prop)
width = round(width * prop)
size = (width, height)
t_img = cv2.resize(img, size)
return t_img
def incision_img(self, img, rectangle):
"""切取图片名称部分"""
y_start = rectangle['top'] - 50
y_end = rectangle['top']
x_start = rectangle['left']
x_end = rectangle['right']
cut = img[y_start:y_end, x_start:x_end]
return cut
def path(data_type):
return get_path(self.ds_dir, img_id, data_type)
def upload_file(request):
try:
file = request.FILES.get('file')
name = file.name
name_re = re.compile('(.png|.jpg)$', flags=re.IGNORECASE)
if name_re.search(name):
wfuser = request.user
user_game, judge = WfUserGame.objects.get_or_create(wfuser=wfuser)
if judge:
img_name = f'{wfuser.username}+{wfuser.uid}{name_re.search(name).group()}'
if os.path.exists(paths.get_path(
f'media/wfusers/{wfuser.uid}')) is False:
os.makedirs(paths.get_path(f'media/wfusers/{wfuser.uid}'))
with open(
paths.get_path(
f'media/wfusers/{wfuser.uid}/{img_name}'),
'wb') as imgfp:
for chunk in file.chunks():
imgfp.write(chunk)
user_game.first_img = f'/media/wfusers/{wfuser.uid}/{img_name}'
user_game.game_status = WfUserGame.AUDIT_STATUS
user_game.save()
add_img_thread = threading.Thread(target=add_wfuser_gamename,
args=(wfuser, user_game))
add_img_thread.start()
return restful.ok('后台以开始审核，请注意查看状态。')
else:
img_dict = user_game.other_imgs
if img_dict:
num = len(img_dict) + 1
img_name = f'{wfuser.username}+{wfuser.uid}_{num}{name_re.search(name).group()}'
if os.path.exists(
paths.get_path(
f'media/wfusers/{wfuser.uid}')) is False:
os.makedirs(
paths.get_path(f'media/wfusers/{wfuser.uid}'))
with open(
paths.get_path(
f'media/wfusers/{wfuser.uid}/{img_name}'),
'wb') as imgfp:
for chunk in file.chunks():
imgfp.write(chunk)
img_dict[
f'{num}'] = f'/media/wfusers/{wfuser.uid}/{img_name}'
user_game.game_status = WfUserGame.MANUAL_STATUS
user_game.save()
wfuser.game_name = None
wfuser.save()
return restful.ok('后台已开始审核，请注意查看状态。')
else:
img_name = f'{wfuser.username}+{wfuser.uid}_1{name_re.search(name).group()}'
if os.path.exists(
paths.get_path(
f'media/wfusers/{wfuser.uid}')) is False:
os.makedirs(
paths.get_path(f'media/wfusers/{wfuser.uid}'))
with open(
paths.get_path(
f'media/wfusers/{wfuser.uid}/{img_name}'),
'wb') as imgfp:
for chunk in file.chunks():
imgfp.write(chunk)
img_dict['1'] = f'/media/wfusers/{wfuser.uid}/{img_name}'
user_game.game_status = WfUserGame.MANUAL_STATUS
user_game.save()
wfuser.game_name = None
wfuser.save()
return restful.ok('后台已开始审核，请注意查看状态。')
else:
return restful.parameters_error('请上传正确的格式的图片，后台只接受PNG与JPG图片。')
except:
return restful.server_error('服务器处理请求失败，请稍后再试。')
def get_weapons_data(self):
with open(paths.get_path('json/weapons/Weapons.json'), 'r', encoding='utf-8') as f:
weapons = json.load(f)
for weapon in weapons:
self.weapons_queue.put(weapon)
def save_weapons(self):
"""保存"""
with open(paths.get_path('json/weapons/Weapons.json'), 'w', encoding='utf-8') as f:
json.dump(self.weapons, f)
def get_data():
"""获取战甲信息"""
http = urllib3.PoolManager()
r = http.request('GET', API.WARFRAME_WARFRAMES)
with open(paths.get_path(idir='json/Warframes.json'), 'wb') as fp:
fp.write(r.data)
