def test_table_calc_results(self):
self.table.rows = [
Row([IsNotNone(), IsGreatThan(0)],
[Result(2), Result(0)]),
Row([IsNone(), IsEqual(0)], [Result(10), Result(1)]),
]
self.assertEqual(self.table.calc([3, 2]), [2, 0])
self.assertEqual(self.table.calc([None, 0]), [10, 1])
def _run_on_asset(self, asset):
# Override Executor._run_on_asset(self, asset), which runs a
# FeatureAssembler, collect a feature vector, run
# TrainTestModel.predict() on it, and return a Result object
# (in this case, both Executor._run_on_asset(self, asset) and
# QualityRunner._read_result(self, asset) get bypassed.
vmaf_fassembler = self._get_vmaf_feature_assembler_instance(asset)
vmaf_fassembler.run()
feature_result = vmaf_fassembler.results[0]
model = self._load_model(asset)
xs = model.get_per_unit_xs_from_a_result(feature_result)
if self.optional_dict is not None and 'disable_clip_score' in self.optional_dict:
disable_clip_score = self.optional_dict['disable_clip_score']
else:
disable_clip_score = False
if self.optional_dict is not None and 'enable_transform_score' in self.optional_dict:
enable_transform_score = self.optional_dict[
'enable_transform_score']
else:
enable_transform_score = False
ys_pred = self.predict_with_model(
model,
xs,
disable_clip_score=disable_clip_score,
enable_transform_score=enable_transform_score)
result_dict = {}
result_dict.update(feature_result.result_dict) # add feature result
result_dict[self.get_scores_key()] = ys_pred # add quality score
return Result(asset, self.executor_id, result_dict)
def _run_on_asset(self, asset):
# Override Executor._run_on_asset(self, asset), which runs a
# FeatureAssembler, collect a feature vector, run
# TrainTestModel.predict() on it, and return a Result object
# (in this case, both Executor._run_on_asset(self, asset) and
# QualityRunner._read_result(self, asset) get bypassed.
vmaf_fassembler = self._get_vmaf_feature_assembler_instance(asset)
vmaf_fassembler.run()
feature_result = vmaf_fassembler.results[0]
xs = TrainTestModel.get_perframe_xs_from_result(feature_result)
model = self._load_model()
ys_pred = model.predict(xs)
# 'score_clip'
ys_pred = self.clip_score(model, ys_pred)
result_dict = {}
# add all feature result
result_dict.update(feature_result.result_dict)
# add quality score
result_dict[self.get_scores_key()] = ys_pred
return Result(asset, self.executor_id, result_dict)
def _run_on_asset(self, asset):
# Override Executor._run_on_asset(self, asset)
vmaf_fassembler = self._get_vmaf_feature_assembler_instance(asset)
vmaf_fassembler.run()
feature_result = vmaf_fassembler.results[0]
result_dict = {
self.get_scores_key(): feature_result[VmafFeatureExtractor.get_scores_key(self.FEATURE_NAME)]
}
return Result(asset, self.executor_id, result_dict)
def _run_on_asset(self, asset):
# Override Executor._run_on_asset(self, asset)
vmaf_fassembler = self._get_feature_assembler_instance(asset)
vmaf_fassembler.run()
feature_result = vmaf_fassembler.results[0]
result_dict = {}
result_dict.update(feature_result.result_dict.copy()) # add feature result
result_dict[self.get_scores_key()] = feature_result.result_dict[
MsSsimFeatureExtractor.get_scores_key('ms_ssim')] # add ssim score
del result_dict[MsSsimFeatureExtractor.get_scores_key('ms_ssim')] # delete redundant
return Result(asset, self.executor_id, result_dict)
def start_threads(self, url):
node = Match(url)
node_ratio = MatchRatio(url, self.ratio_percents)
node_result = Result(url)
node.start()
node.join()
# Здесь сделать request кодов
xhash, id_sport, id_match, id_version = node.return_keys()
node_ratio.add_keys(xhash, id_sport, id_match, id_version)
node_result.add_keys(xhash, id_sport, id_match)
node_ratio.start()
node_result.start()
node.add_data(node_ratio, node_result)
return node
def _run_on_asset(self, asset):
# Override Executor._run_on_asset(self, asset), which runs a
# FeatureAssembler, collect a feature vector, run
# TrainTestModel.predict() on it, and return a Result object
# (in this case, both Executor._run_on_asset(self, asset) and
# QualityRunner._read_result(self, asset) get bypassed.
vmaf_fassembler = self._get_vmaf_feature_assembler_instance(asset)
vmaf_fassembler.run()
feature_result = vmaf_fassembler.results[0]
# =====================================================================
# SVR predict
model = self.svmutil.svm_load_model(self.SVM_MODEL_FILE)
ordered_scaled_scores_list = []
for scores_key in self.SVM_MODEL_ORDERED_SCORES_KEYS:
scaled_scores = self._rescale(
feature_result[scores_key],
self.FEATURE_RESCALE_DICT[scores_key])
ordered_scaled_scores_list.append(scaled_scores)
scores = []
for score_vector in zip(*ordered_scaled_scores_list):
vif, adm, ansnr, motion = score_vector
xs = [[vif, adm, ansnr, motion]]
score = self.svmutil.svm_predict([0], xs, model)[0][0]
score = self._post_correction(motion, score)
scores.append(score)
result_dict = {}
# add all feature result
result_dict.update(feature_result.result_dict)
# add quality score
result_dict[self.get_scores_key()] = scores
return Result(asset, self.executor_id, result_dict)
def extract(self,
response: requests.Response,
img_num,
soup=None) -> Optional[Result]:
data = response.text
start_end = re.search(pattern="var msg_title = .*? var msg_link = ",
string=data,
flags=re.S)
if not start_end:
return Empty
title = ""
description = ""
img = []
for i in start_end.group().split("\n"):
text = i.strip()
if text.startswith("var msg_title"):
title = text.split(" = ", maxsplit=1)[-1][1:-2]
elif text.startswith("var msg_desc"):
description = text.split(" = ", maxsplit=1)[-1][1:-2]
elif len(img) < img_num and text.startswith("var msg_cdn_url"):
img.append(Img(url=text.split(" = ", maxsplit=1)[-1][1:-2]))
elif len(img) < img_num and text.startswith("var cdn_url_1_1"):
img.append(Img(url=text.split(" = ", maxsplit=1)[-1][1:-2]))
elif len(img) < img_num and text.startswith("var cdn_url_235_1"):
img.append(Img(url=text.split(" = ", maxsplit=1)[-1][1:-2]))
else:
pass
if img_num > 0 and len(img) < 1:
img.append(
Img(url=
"https://res.wx.qq.com/a/wx_fed/assets/res/OTE0YTAw.png",
width=180,
height=180))
return Result(title=title,
keywords=None,
description=description,
img=img,
limit_img_num=img_num)
def _run_on_asset(self, asset):
# Override VmafQualityRunner._run_on_asset(self, asset), by adding
# additional local explanation info.
vmaf_fassembler = self._get_vmaf_feature_assembler_instance(asset)
vmaf_fassembler.run()
feature_result = vmaf_fassembler.results[0]
model = self._load_model(asset)
xs = model.get_per_unit_xs_from_a_result(feature_result)
ys_pred = self.predict_with_model(model, xs)
if self.optional_dict2 is not None and \
'explainer' in self.optional_dict2:
explainer = self.optional_dict2['explainer']
else:
explainer = LocalExplainer()
exps = explainer.explain(model, xs)
result_dict = {}
result_dict.update(feature_result.result_dict) # add feature result
result_dict[self.get_scores_key()] = ys_pred # add quality score
result_dict[
self.get_explanations_key()] = exps # add local explanations
return Result(asset, self.executor_id, result_dict)
def _read_result(self, asset):
result = {}
result.update(self._get_feature_scores(asset))
executor_id = self.executor_id
return Result(asset, executor_id, result)
def _read_result(self, asset):
result = {}
result.update(self._get_quality_scores(asset))
return Result(asset, self.executor_id, result)
try:
to = 10 if self.engine.getOption('http-proxy') is None else 20
response = urlopen(req, timeout=to)
except HTTPError, e:
self._addError(e.code, target.getAbsoluteUrl())
return
except URLError, e:
self._addError(e.reason, target.getAbsoluteUrl())
return
except:
self._addError('Unknown', target.getAbsoluteUrl())
return
else:
result = self.processResponse(response.read().lower(), pl)
for r in result:
self.results.append(Result(target, k, pl, r))
def _checkStoredInjections(self):
for r in self.results:
# At this state injections in Result obj are not
# compacted yet so it will only be 1st injected param
url, data = r.target.getPayloadedUrl(r.first_param, "")
# In case of proxy
if self.engine.getOption('http-proxy') is not None:
proxy = ProxyHandler({'http': self.engine.getOption('http-proxy')})
opener = build_opener(proxy)
install_opener(opener)
# Some headers
if self.engine.getOption('ua') is not None:
