def run(work_path):
# 系统初始化，参数要与创建技能时填写的检验值保持一致
hilens.init("driving")
# 初始化自带摄像头与HDMI显示器,
# hilens studio中VideoCapture如果不填写参数，则默认读取test/camera0.mp4文件，
# 在hilens kit中不填写参数则读取本地摄像头
camera = hilens.VideoCapture()
display = hilens.Display(hilens.HDMI)
# 初始化模型
model_path = os.path.join(work_path, 'model/yolo3.om')
driving_model = hilens.Model(model_path)
frame_index = 0
json_bbox_list = []
json_data = {'info': 'det_result'}
while True:
frame_index += 1
try:
time_start = time.time()
# 1. 设备接入 #####
input_yuv = camera.read() # 读取一帧图片(YUV NV21格式)
# 2. 数据预处理 #####
img_bgr = cv2.cvtColor(input_yuv,
cv2.COLOR_YUV2BGR_NV21) # 转为BGR格式
img_preprocess, img_w, img_h = preprocess(img_bgr) # 缩放为模型输入尺寸
# 3. 模型推理 #####
output = driving_model.infer([img_preprocess.flatten()])
# 4. 获取检测结果 #####
bboxes = get_result(output, img_w, img_h)
# 5-1. [比赛提交作品用] 将结果输出到json文件中 #####
if len(bboxes) > 0:
json_bbox = convert_to_json(bboxes, frame_index)
json_bbox_list.append(json_bbox)
# 5-2. [调试用] 将结果输出到模拟器中 #####
img_bgr = draw_boxes(img_bgr, bboxes) # 在图像上画框
output_yuv = hilens.cvt_color(img_bgr, hilens.BGR2YUV_NV21)
display.show(output_yuv) # 显示到屏幕上
time_frame = 1000 * (time.time() - time_start)
hilens.info('----- time_frame = %.2fms -----' % time_frame)
except RuntimeError:
print('last frame')
break
# 保存检测结果
hilens.info('write json result to file')
result_filename = './result.json'
json_data['result'] = json_bbox_list
save_json_to_file(json_data, result_filename)
hilens.terminate()
def find_pkgset():
"""
The function which returns a list of binary packages for the given source
package names.
Input GET params:
name * - source package name or list of packages
task ** - number of task
Output structure:
branch
sourcepkgname
date
packages
version
release
disttag
packager_email
buildtime
archs
"""
server.url_logging()
check_params = server.check_input_params()
if check_params is not True:
return check_params
values = server.get_dict_values([('name', 's', 'pkg_name'), ('task', 'i')])
if list(values.values()).count(None) > 1:
return utils.json_str_error("One parameter only ('name'/'task').")
if values['name']:
pkg_ls = values['name'].split(',')
else:
g.connection.request_line = (QM.find_pkgset_get_package_names, {
'task_id': values['task']
})
status, response = g.connection.send_request()
if status is False:
return response
pkg_ls = utils.join_tuples(response)
g.connection.request_line = (QM.find_pkgset_get_branch_with_pkgs, {
'pkgs': tuple(pkg_ls)
})
status, response = g.connection.send_request()
if status is False:
return response
param_ls = [
'branch', 'sourcepkgname', 'pkgset_datetime', 'packages', 'version',
'release', 'disttag', 'packager_email', 'buildtime', 'archs'
]
return utils.convert_to_json(param_ls, response)
def unpackaged_dirs():
"""
The function of searching unpacked directories by maintainer name.
Input GET params:
pkgr * - maintainer name
pkgset * - repository name
arch - architecture of packages
Output structure:
package
directory
version
release
epoch
packager
email
arch
"""
server.url_logging()
check_params = server.check_input_params()
if check_params is not True:
return check_params
values = server.get_dict_values([('pkgr', 's'),
('pkgset', 's', 'repo_name'),
('arch', 's')])
if not values['pkgr'] or not values['pkgset']:
return get_helper(server.helper(request.path))
parch = server.default_archs
if values['arch']:
parch = values['arch'].split(',')
if 'noarch' not in parch:
parch.append('noarch')
g.connection.request_line = (QM.unpackaged_dirs_get_pkg_dirs, {
'branch': values['pkgset'],
'email': '{}@%'.format(values['pkgr']),
'archs': tuple(parch)
})
status, response = g.connection.send_request()
if status is False:
return response
js_keys = [
'package', 'directory', 'version', 'release', 'epoch', 'packager',
'email', 'arch'
]
return utils.convert_to_json(js_keys, response)
def test_convert_to_json(self):
values = [('elem1', 'elem2', 'elem3'),
('elem1.1', 'elem2.1', 'elem3.1'),
('elem1.2', 'elem2.2', 'elem3.2'), ]
js = json.loads(
utils.convert_to_json(['key1', 'key2', 'key3'], values)
)
assert 'elem1' == js['0']['key1']
assert 'elem2.1' == js['1']['key2']
assert 'elem3.2' == js['2']['key3']
def read(self, cell_hash, frame_name, data_format=None, nrow=None):
"""
Tell the selected backend to read the file, and filter if required.
"""
data = self.store.read(cell_hash, frame_name)
if data_format == "application/json":
data = convert_to_json(data)
elif data_format == "application/octet-stream":
data = convert_to_arrow(data)
if nrow:
data = filter_data(data, nrow)
return data
def dependent_packages():
"""
The function of searching source packages whose binary packages depend on
the given package.
Input GET params:
name * - name of package
branch * - repository name
Output structure:
name
version
release
epoch
serial
sourcerpm
branch
archs
"""
server.url_logging()
check_params = server.check_input_params()
if check_params is not True:
return check_params
pname = server.get_one_value('name', 's', is_='pkg_name')
if not pname:
return get_helper(server.helper(request.path))
pbranch = server.get_one_value('branch', 's', is_='repo_name')
if not pbranch:
message = 'Branch is required parameter.'
logger.debug(message)
return utils.json_str_error(message)
g.connection.request_line = (QM.dependent_packages_get_dependent_pkgs, {
'name': pname,
'branch': pbranch
})
status, response = g.connection.send_request()
if status is False:
return response
js_keys = [
'name', 'version', 'release', 'epoch', 'serial', 'sourcerpm', 'branch',
'archs'
]
return utils.convert_to_json(js_keys, response)
def index():
tree_data = convert_to_json(args.data)
return render_template("index.html", tree_data=tree_data, title=args.title)
#!/bin/env python
import pickle
import utils
file = open('../database.pickle', 'rb')
database = pickle.load(file, encoding='latin1')
file.close()
utils.convert_to_json('../database.pickle', '../database.json')
#Put it in a nice table for easy parsing. Use semicolons to separate fields, making sure each individual field doesn't contain any semicolons since this would break parsing.
outtext = ["#Hydration free energy datbase v0.52, 6/11/17.\n"]
outtext += [
"#Semicolon-delimited text file with fields in the following format:\n"
]
outtext += [
"# compound id (and file prefix); SMILES; iupac name (or alternative if IUPAC is unavailable or not parseable by OEChem); experimental value (kcal/mol); experimental uncertainty (kcal/mol); Mobley group calculated value (GAFF) (kcal/mol); calculated uncertainty (kcal/mol); experimental reference (original or paper this value was taken from); calculated reference; text notes.\n"
]
cids = list(database.keys())
cids.sort()
for cid in cids:
notes = ''
for line in database[cid]['notes']:
if not '\n' in line: line += ' '
notes += line.replace(
'\n', ' ') #Remove line breaks as they will break format here
if ';' in notes: #Make sure no semicolon in notes
#Fix issue where I used a semicolon
def what_depends_build():
"""
The function of searching build dependencies.
The function search build dependencies of package, list packages or
packages from task. Also function can also use such parameters like as
leaf, searching depth.
Input GET params:
name * - package or list of packages
task ** - task id
branch (* - for pkg_ls only) - name of repository
arch - package architectures
leaf - assembly dependency chain
deep - sorting depth
dptype - type of package (source, binary, both)
reqfilter - filter result by dependency by binary pkg
reqfilterbysrc - filter result by dependency by source pkg
finitepkg - topological tree leaves
Output structure:
name
version
release
epoch
serial_
sourcerpm
branch
archs
buildtime
cycle
requires
acl
"""
server.url_logging()
check_params = server.check_input_params(source=1)
if check_params is not True:
return check_params
pname = server.get_one_value('name', 's', is_='pkg_name')
task_id = server.get_one_value('task', 'i')
# dptype option
depends_type_to_sql = {'source': (1, ), 'binary': (0, ), 'both': (1, 0)}
depends_type = server.get_one_value('dptype', 's')
if depends_type not in depends_type_to_sql:
depends_type = 'both'
sourcef = depends_type_to_sql[depends_type]
message = None
if pname and task_id:
message = "Only one parameter package 'name' or build 'task'."
elif not pname and not task_id:
message = "Source package 'name' or build 'task' " \
"is require parameters."
if message:
logger.debug(message)
return utils.json_str_error(message)
pbranch = server.get_one_value('branch', 's', is_='repo_name')
if pname and not pbranch:
return get_helper(server.helper(request.path))
arch = server.get_one_value('arch', 's')
if arch:
arch = [arch]
if 'noarch' not in arch:
arch.append('noarch')
else:
arch = ['x86_64', 'noarch']
# tree leaf - show only build path between 'name' and 'leaf'
leaf = server.get_one_value('leaf', 's', 'pkg_name')
if leaf and task_id:
return utils.json_str_error("'leaf' may be using with 'name' only.")
# process this query for task
if task_id:
# get the branch name from task
g.connection.request_line = (
"SELECT DISTINCT branch FROM Tasks WHERE task_id = %(id)s", {
'id': task_id
})
status, response = g.connection.send_request()
if status is False:
return response
if not response:
return utils.json_str_error('Unknown task id.')
# branch from task
pbranch = response[0][0]
if pbranch.lower() == 'sisyphus':
pbranch = 'Sisyphus'
# get the packages hashes from Task
g.connection.request_line = (
"SELECT pkgs FROM Tasks WHERE task_id = %(id)s", {
'id': task_id
})
status, response = g.connection.send_request()
if status is False:
return response
# join list of tuples of tuples
pkgs_hsh = ()
for tp_package in response:
for package in tp_package[0]:
pkgs_hsh += (package, )
# src packages from task
g.connection.request_line = (QM.wds_get_src_from_task, {'id': task_id})
status, response = g.connection.send_request()
if status is False:
return response
input_pkgs = utils.join_tuples(response)
# without task - get the packages name from URL
else:
input_pkgs = (pname, )
# deep level for recursive requires search
deep_level = server.get_one_value('deep', 'i')
if not deep_level:
deep_level = 1
tmp_table_name = 'tmp_pkg_ls'
# create tmp table with list of packages
g.connection.request_line = \
"""CREATE TEMPORARY TABLE {tmp_table} (pkgname String) \
""".format(tmp_table=tmp_table_name)
status, response = g.connection.send_request()
if status is False:
return response
# FIXME use package_deps module
# base query - first iteration, build requires depth 1
g.connection.request_line = (QM.wds_insert_build_req_deep_1.format(
tmp_table=tmp_table_name), {
'sfilter': sourcef,
'pkgs': input_pkgs,
'branch': pbranch,
'union': list(input_pkgs)
})
status, response = g.connection.send_request()
if status is False:
return response
max_allowed_depth = 10
if deep_level > 1:
if deep_level > max_allowed_depth:
return utils.json_str_error(
"Requires Depth cannot exceed {}".format(max_allowed_depth))
# sql wrapper for increase depth
deep_wrapper = QM.wds_increase_depth_wrap.format(
tmp_table=tmp_table_name)
# process depth for every level and add results to pkg_ls
for i in range(deep_level - 1):
g.connection.request_line = (
QM.wds_insert_result_for_depth_level.format(
wrapper=deep_wrapper, tmp_table=tmp_table_name), {
'sfilter': sourcef,
'branch': pbranch
})
status, response = g.connection.send_request()
if status is False:
return response
g.connection.request_line = (QM.wds_get_acl.format(
tmp_table=tmp_table_name), {
'branch': pbranch.lower()
})
status, response = g.connection.send_request()
if status is False:
return response
# get package acl
pkg_acl_dict = {}
for pkg in response:
pkg_acl_dict[pkg[0]] = pkg[1][0]
tmp_table_pkg_dep = 'package_dependency'
# create tmp table package - dependency
g.connection.request_line = \
"""CREATE TEMPORARY TABLE {}
(
pkgname String,
reqname String
)""".format(tmp_table_pkg_dep)
status, response = g.connection.send_request()
if status is False:
return response
# get source dependencies
if depends_type in ['source', 'both']:
# populate the temporary table with package names and their source
# dependencies
g.connection.request_line = (QM.wds_insert_src_deps.format(
tmp_deps=tmp_table_pkg_dep, tmp_table=tmp_table_name), {
'branch': pbranch,
'pkgs': list(input_pkgs)
})
status, response = g.connection.send_request()
if status is False:
return response
# get binary dependencies
if depends_type in ['binary', 'both']:
# populate the temporary table with package names and their binary
# dependencies
g.connection.request_line = (QM.wds_insert_binary_deps.format(
tmp_table=tmp_table_name, tmp_req=tmp_table_pkg_dep), {
'branch': pbranch,
'archs': tuple(arch)
})
status, response = g.connection.send_request()
if status is False:
return response
# select all filtered package with dependencies
g.connection.request_line = QM.wds_get_all_filtred_pkgs_with_deps
status, response = g.connection.send_request()
if status is False:
return response
pkgs_to_sort_dict = utils.tuplelist_to_dict(response, 1)
if not pkgs_to_sort_dict:
return json.dumps({})
finitepkg = server.get_one_value('finitepkg', 'b', is_='pkg_name')
if finitepkg:
all_dependencies = []
for pkg, deps in pkgs_to_sort_dict.items():
for dep in deps:
if dep not in all_dependencies:
all_dependencies.append(dep)
g.connection.request_line = \
("SELECT pkgname FROM {} WHERE pkgname NOT IN %(pkgs)s"
"".format(tmp_table_name), {'pkgs': tuple(all_dependencies)})
status, response = g.connection.send_request()
if status is False:
return response
filter_by_tops = utils.join_tuples(response)
# check leaf, if true, get dependencies of leaf package
if leaf:
if leaf not in pkgs_to_sort_dict.keys():
return utils.json_str_error(
"Package '{}' not in dependencies list.".format(leaf))
pkg_ls_with_empty_reqs = []
for pkg, reqs in pkgs_to_sort_dict.items():
if not reqs:
pkg_ls_with_empty_reqs.append(pkg)
# sort list of dependencies by their dependencies
sort = SortList(pkgs_to_sort_dict, pname)
circle_deps, sorted_list = sort.sort_list()
# create output dict with circle dependency
result_dict = {}
for name in sorted_list:
result_dict[name] = []
if name in circle_deps:
result_dict[name] += list(circle_deps[name].keys())
# if leaf, then select packages from the result list and their cyclic
# dependencies on which the leaf package and create a dictionary
if leaf:
def recursive_search(pkgname, structure):
for pkg in structure[pkgname]:
if pkg not in leaf_filter and pkg != pkgname:
leaf_filter.append(pkg)
recursive_search(pkg, structure)
leaf_filter = []
recursive_search(leaf, pkgs_to_sort_dict)
if leaf not in leaf_filter:
leaf_filter.append(leaf)
# filter result dict by leaf packages
result_dict = {
key: value
for (key, value) in result_dict.items() if key in leaf_filter
}
# list of result package names
sorted_pkgs = tuple(result_dict.keys())
# get output data for sorted package list
g.connection.request_line = (QM.wds_get_output_data.format(
tmp_table=tmp_table_name), {
'branch': pbranch
})
status, response = g.connection.send_request()
if status is False:
return response
# form list of packages with it information
pkg_info_list = []
for info in response:
for pkg, c_deps in result_dict.items():
if info[0] == pkg:
# add empty list if not acl
if pkg not in pkg_acl_dict:
pkg_acl_dict[pkg] = []
pkg_info_list.append(info + (c_deps, ) +
(pkgs_to_sort_dict[pkg], ) +
(pkg_acl_dict[pkg], ))
# filter result packages list by dependencies
reqfilter = server.get_dict_values([('reqfilter', 's', 'pkg_name'),
('reqfilterbysrc', 's', 'pkg_name')])
if None not in reqfilter.values():
message = "Parameters 'reqfilter' and 'reqfilterbysrc' cannot be " \
"used together."
return utils.json_str_error(message)
filter_pkgs = None
if reqfilter['reqfilter'] or reqfilter['reqfilterbysrc']:
if reqfilter['reqfilter']:
reqfilter_binpkgs = tuple(reqfilter['reqfilter'].split(','))
else:
g.connection.request_line = (QM.wds_req_filter_by_src, {
'srcpkg': reqfilter['reqfilterbysrc'],
'branch': pbranch
})
status, response = g.connection.send_request()
if status is False:
return response
reqfilter_binpkgs = utils.join_tuples(response)
base_query = QM.wds_req_filter_by_binary.format(
pkg="{pkg}", tmp_table=tmp_table_name)
if len(reqfilter_binpkgs) == 1:
base_query = base_query.format(pkg=reqfilter_binpkgs[0])
else:
last_query = None
for pkg in reqfilter_binpkgs:
if not last_query:
last_query = base_query.format(pkg=pkg)
last_query = "{} AND pkgname IN ({})" \
"".format(last_query, base_query.format(pkg=pkg))
base_query = last_query
g.connection.request_line = (QM.wds_get_filter_pkgs.format(
base_query=base_query), {
'branch': pbranch,
'archs': tuple(arch)
})
status, response = g.connection.send_request()
if status is False:
return response
filter_pkgs = utils.join_tuples(response)
# sort pkg info list
sorted_dict = {}
for pkg in pkg_info_list:
if (filter_pkgs and pkg[0] in filter_pkgs) or not filter_pkgs:
if task_id:
if pkg[0] not in input_pkgs:
sorted_dict[sorted_pkgs.index(pkg[0])] = pkg
else:
sorted_dict[sorted_pkgs.index(pkg[0])] = pkg
sorted_dict = list(dict(sorted(sorted_dict.items())).values())
if finitepkg:
sorted_dict = [pkg for pkg in sorted_dict if pkg[0] in filter_by_tops]
js_keys = [
'name', 'version', 'release', 'epoch', 'serial_', 'sourcerpm',
'branch', 'archs', 'buildtime', 'cycle', 'requires', 'acl'
]
return utils.convert_to_json(js_keys, sorted_dict)
def package_by_file():
"""
The function of searching binary packages that contain the specified file.
Input GET params:
file * - file name or pattern
md5 ** - file md5
branch * - name of repository
arch - package architecture
Output structure:
pkgcs
name
version
release
disttag
arch
branch
files
"""
server.url_logging()
check_params = server.check_input_params()
if check_params is not True:
return check_params
file = server.get_one_value('file', 'r', is_='pkg_name')
md5 = server.get_one_value('md5', 's')
if len([param for param in [file, md5] if param]) != 1:
return get_helper(server.helper(request.path))
pbranch = server.get_one_value('branch', 's', 'repo_name')
if not pbranch:
return utils.json_str_error('Branch require parameter!')
arch = server.get_one_value('arch', 's')
if arch:
arch = (arch, 'noarch')
else:
arch = server.known_archs
base_query = QM.package_by_file_get_hshs_by_files.format(
in_='{}', pkghash=QM.package_by_file_get_pkg_hashs, param='{}')
if file:
elem, query = file, "filename LIKE %(elem)s"
else:
elem, query = md5, "filemd5 = %(elem)s"
g.connection.request_line = (base_query.format(', filename', query), {
'branch': pbranch,
'arch': tuple(arch),
'elem': elem
})
status, response = g.connection.send_request()
if status is False:
return response
if not response:
return json.dumps({})
ids_filename_dict = utils.tuplelist_to_dict(response, 1)
pkghashs = tuple([key for key in ids_filename_dict.keys()])
g.connection.request_line = (QM.package_by_file_get_meta_by_hshs, {
'hashs': pkghashs,
'branch': pbranch
})
status, response = g.connection.send_request()
if status is False:
return response
output_values = []
for package in response:
package += (ids_filename_dict[package[0]], )
output_values.append(package[1:])
output_params = [
'pkgcs', 'name', 'version', 'release', 'disttag', 'arch', 'branch',
'files'
]
return utils.convert_to_json(output_params, tuple(output_values))
def package_info():
"""
The function of showing information about given package by user parameters.
Input GET params:
sha1 - package sha1
name - package name
version - package version
release - package release
arch - package arch
disttag - package disttag
buildtime - package buildtime
source - show source packages only (true/false)
packager - maintainer of package
packager_email - maintainer's email
branch - name of repository
full - show full package information
Output structure:
`without 'full' option`
pkgcs
packager
packager_email
name
arch
version
release
epoch
buildtime
sourcepackage
sourcerpm
filename
"""
server.url_logging()
check_params = server.check_input_params()
if check_params is not True:
return check_params
buildtime_action = None
buildtime_value = server.get_one_value('buildtime', 'i')
if buildtime_value and buildtime_value not in ['>', '<', '=']:
buildtime_action = "{} = {}"
pbranch = server.get_one_value('branch', 's', is_='repo_name')
input_params = {
'sha1': {
'rname': 'pkgcs',
'type': 's',
'action': None,
'notenpty': False,
},
'name': {
'rname': 'name',
'type': 's',
'action': None,
'notempty': False,
'is_': 'pkg_name',
},
'version': {
'rname': 'version',
'type': 's',
'action': None,
'notempty': False,
},
'release': {
'rname': 'release',
'type': 's',
'action': None,
'notempty': False,
},
'arch': {
'rname': 'arch',
'type': 's',
'action': None,
'notempty': False,
},
'disttag': {
'rname': 'disttag',
'type': 's',
'action': None,
'notempty': False,
},
'buildtime': {
'rname': 'buildtime',
'type': 'i',
'action': buildtime_action,
'notempty': False,
},
'source': {
'rname': 'sourcepackage',
'type': 'b',
'action': None,
'notempty': False,
},
'packager': {
'rname': 'name',
'type': 's',
'action': None,
'notempty': False,
},
'packager_email': {
'rname': 'packager_email',
'type': 's',
'action': None,
'notempty': False,
},
}
params_values = server.get_values_by_params(input_params)
if params_values is False:
return get_helper(server.helper(request.path))
full = bool(server.get_one_value('full', 'b'))
output_params = [
'pkgcs',
'packager',
'packager_email',
'name',
'arch',
'version',
'release',
'epoch',
'buildtime',
'sourcepackage',
'sourcerpm',
'filename',
]
if full:
output_params = server.package_params
g.connection.request_line = \
"SELECT pkg.pkghash, {p_params} FROM last_packages WHERE " \
"{p_values} {branch}".format(
p_params=", ".join(output_params),
p_values=" ".join(params_values),
branch='{}'
)
if pbranch:
g.connection.request_line = g.connection.request_line.format(
"AND assigment_name = %(branch)s")
else:
g.connection.request_line = g.connection.request_line.format('')
g.connection.request_line = (g.connection.request_line, {
'branch': pbranch
})
status, response = g.connection.send_request()
if status is False:
return response
json_retval = json.loads(
utils.convert_to_json(['pkghash'] + output_params, response))
if full and len(response) > 0:
pkghashs = utils.join_tuples(response)
# files
g.connection.request_line = (
"SELECT pkghash, groupUniqArray(filename) FROM File WHERE pkghash "
"IN %(pkghshs)s GROUP BY pkghash", {
'pkghshs': pkghashs
})
status, response = g.connection.send_request()
if status is False:
return response
files_dict = utils.tuplelist_to_dict(response, 1)
# add empty list if package has no files
for hsh in pkghashs:
if hsh not in files_dict:
files_dict[hsh] = []
# depends
g.connection.request_line = (
"SELECT pkghash, dptype, dpname FROM last_depends WHERE pkghash "
"IN %(pkghshs)s", {
'pkghshs': pkghashs
})
status, response = g.connection.send_request()
if status is False:
return response
depends_dict = utils.tuplelist_to_dict(response, 2)
depends_struct = {}
for pkg in depends_dict:
depend_ls = depends_dict[pkg]
depends_struct[pkg] = {}
for i in range(0, len(depend_ls), 2):
if depend_ls[i] not in depends_struct[pkg]:
depends_struct[pkg][depend_ls[i]] = []
depends_struct[pkg][depend_ls[i]].append(depend_ls[i + 1])
for elem in json_retval:
pkghash = json_retval[elem]['pkghash']
# add files to result structure
json_retval[elem]['files'] = files_dict[pkghash]
# add depends to result structure
for dep in depends_struct[pkghash]:
json_retval[elem][dep] = depends_struct[pkghash][dep]
# remove pkghash from result
for value in json_retval.values():
value.pop('pkghash', None)
return json.dumps(json_retval, sort_keys=False)
def misconflict_packages():
"""
The function of searching for conflicting files in packages that do not have
a conflict.
Input GET params:
pkg_ls * - package or list of packages
task ** - task id
branch (* - for pkg_ls only) - name of repository
arch - package architectures
Output structure:
input package
conflict package
version
release
epoch
architectures
files with conflict
"""
server.url_logging()
check_params = server.check_input_params(source=0)
if check_params is not True:
return check_params
values = server.get_dict_values([('pkg_ls', 's', 'pkg_name'), ('task',
'i'),
('branch', 's', 'repo_name'),
('arch', 's')])
if values['pkg_ls'] and values['task']:
return utils.json_str_error("One parameter only. ('name'/'task')")
if not values['pkg_ls'] and not values['task']:
return utils.json_str_error(
"'pkg_ls' or 'task' is require parameters.")
if values['pkg_ls'] and not values['branch']:
return get_helper(server.helper(request.path))
if values['arch']:
allowed_archs = values['arch'].split(',')
if 'noarch' not in allowed_archs:
allowed_archs.append('noarch')
else:
allowed_archs = server.default_archs
allowed_archs = tuple(allowed_archs)
# prepare packages list from Task
if values['task']:
# get branch of task
g.connection.request_line = (
"SELECT DISTINCT branch FROM Tasks WHERE task_id = %(task)d", {
'task': values['task']
})
status, response = g.connection.send_request()
if status is False:
return response
if not response:
return utils.json_str_error(
"Task {task} not found!".format(task=values['task']))
pbranch = response[0][0]
# get packages of task for last build iteration (hashes)
g.connection.request_line = (QM.misconflict_pkgs_get_pkgs_of_task, {
'task': values['task']
})
status, response = g.connection.send_request()
if status is False:
return response
if not response:
return utils.json_str_error(
"Error: Packages in task {task} not found!"
"".format(task=values['task']))
# joining tuples from response list
input_pkg_hshs = [hsh[0] for hsh in response]
# package list without task
else:
pkg_ls = tuple(values['pkg_ls'].split(','))
pbranch = values['branch']
# get hash for package names
g.connection.request_line = (QM.misconflict_pkgs_get_hshs_by_pkgs, {
'pkgs': tuple(pkg_ls),
'branch': pbranch,
'arch': allowed_archs
})
status, response = g.connection.send_request()
if status is False:
return response
if not response:
return utils.json_str_error(
"Error: Packages {pkgs} not found in pkgset {branch}!".format(
pkgs=pkg_ls, branch=pbranch))
# check the existence of a package by comparing the number of input
# and selected from database
if len(set([pkg[1] for pkg in response])) != len(pkg_ls):
return utils.json_str_error("Error of input data.")
# form a list of package hashes
input_pkg_hshs = [pkg[0] for pkg in response]
if not input_pkg_hshs:
return json.dumps({})
# get list of (input package | conflict package | conflict files)
g.connection.request_line = (QM.misconflict_pkgs_get_pkg_with_conflict, {
'hshs': tuple(input_pkg_hshs),
'branch': pbranch,
'arch': allowed_archs
})
status, response = g.connection.send_request()
if status is False:
return response
if not response:
return json.dumps({})
hshs_files = response
# list of conflicting package pairs
in_confl_hshs = [(hsh[0], hsh[1]) for hsh in hshs_files]
# filter conflicts by provides/conflicts
c_filter = ConflictFilter(pbranch, allowed_archs)
# check for the presence of the specified conflict each pair
# if the conflict between the packages in the pair is specified,
# then add the pair to the list
filter_ls = c_filter.detect_conflict(in_confl_hshs)
# create dict with package names by hashes
hsh_name_dict = defaultdict(dict)
for hsh_1, hsh_2, _, name_2, name_1, _ in response:
hsh_name_dict[hsh_1], hsh_name_dict[hsh_2] = name_1, name_2
# convert the hashes into names, put in the first place in the pair
# the name of the input package, if it is not
filter_ls_names = []
for hsh in filter_ls:
inp_pkg = hsh[0] if hsh[0] in input_pkg_hshs else hsh[1]
out_pkg = hsh[0] if hsh[0] != inp_pkg else hsh[1]
result_pair = (hsh_name_dict[inp_pkg], hsh_name_dict[out_pkg])
if result_pair not in filter_ls:
filter_ls_names.append(result_pair)
# form the list of tuples (input package | conflict package | conflict files)
result_list, output_pkgs = [], set()
for pkg in hshs_files:
[output_pkgs.add(i) for i in pkg[:2]]
pkg = (hsh_name_dict[pkg[0]], hsh_name_dict[pkg[1]], pkg[2])
if pkg not in result_list:
result_list.append(pkg)
# get architectures of found packages
g.connection.request_line = QM.misconflict_pkgs_get_pkg_archs.format(
hshs=tuple(output_pkgs))
status, response = g.connection.send_request()
if status is False:
return response
pkg_archs_dict = utils.tuplelist_to_dict(response, 1)
# look for duplicate pairs of packages in the list with different files
# and join them
result_dict_cleanup = defaultdict(list)
for pkg in result_list:
result_dict_cleanup[(pkg[0], pkg[1])] += pkg[2]
confl_pkgs = utils.remove_duplicate(
[pkg[1] for pkg in result_dict_cleanup.keys()])
# get main information of packages by package hashes
g.connection.request_line = (QM.misconflict_pkgs_get_meta_by_hshs, {
'pkgs': tuple(confl_pkgs),
'branch': pbranch,
'arch': allowed_archs
})
status, response = g.connection.send_request()
if status is False:
return response
# form dict name - package info
name_info_dict = {}
for pkg in response:
name_info_dict[pkg[0]] = pkg[1:]
# form list of tuples (input pkg | conflict pkg | pkg info | conflict files)
# and filter it
result_list_info = []
for pkg, files in result_dict_cleanup.items():
inp_pkg_archs = set(pkg_archs_dict[pkg[0]])
found_pkg_archs = set(pkg_archs_dict[pkg[1]])
intersect_pkg_archs = inp_pkg_archs.intersection(found_pkg_archs)
if (pkg[0], pkg[1]) not in filter_ls_names and intersect_pkg_archs:
pkg = (pkg[0], pkg[1]) + \
name_info_dict[pkg[1]][:-1] + \
(list(intersect_pkg_archs),) + (files,)
result_list_info.append(pkg)
return utils.convert_to_json([
'input_package', 'conflict_package', 'version', 'release', 'epoch',
'archs', 'files_with_conflict'
], result_list_info)
def task_info():
server.url_logging()
check_params = server.check_input_params()
if check_params is not True:
return check_params
task_id = server.get_one_value('task', type_='i')
if not task_id:
return get_helper(server.helper(request.path))
try_iteration = server.get_one_value('rebuild', type_='s')
if try_iteration:
try_iteration = tuple([int(i) for i in try_iteration.split('.')])
g.connection.request_line = """
SELECT DISTINCT concat(toString(try), '.', toString(iteration)),
branch,
userid
FROM Tasks
WHERE task_id = {}
""".format(task_id)
status, response = g.connection.send_request()
if status is False:
return response
if not response:
return utils.json_str_error("Non-existent task number!")
branch, user_id = response[0][1], response[0][2]
all_rebuilds = [i[0] for i in response]
g.connection.request_line = QM.task_info_get_task_content.format(
id=task_id)
if try_iteration:
g.connection.request_line = QM.task_info_get_task_content_rebuild.format(
id=task_id, ti=try_iteration)
status, response = g.connection.send_request()
if status is False:
return response
src_pkgs = response
task_status = src_pkgs[0][1]
try_iteration = src_pkgs[0][3]
pkg_subtask = {pkg[0]: pkg[2] for pkg in src_pkgs}
pkg_hshs = [
val for sublist in [[i[0]] + i[4] for i in response] for val in sublist
]
g.connection.request_line = """SELECT pkghash,
name,
version,
release,
arch,
description
FROM Package
WHERE pkghash IN {}
""".format(tuple(pkg_hshs))
status, response = g.connection.send_request()
if status is False:
return response
name_hsh = utils.tuplelist_to_dict(response, 5)
http = utils.HtmlParser(
'a',
['Name', 'Last modified', 'Size', 'Description', 'Parent Directory'])
for hsh, subtask in pkg_subtask.items():
result_list = []
for i in ['approved', 'disapproved']:
url = "http://git.altlinux.org/tasks/{task}/acl/{act}/{subtask}/" \
"".format(task=task_id, subtask=subtask, act=i)
result = http.parse_html(url)
if result:
result = result.split('::')
msg_format = [result[0].strip()] + \
[k.strip() for k in result[1].split('\n')][:-1]
else:
msg_format = ''
result_list.append(msg_format or '')
pkg_subtask[hsh] = [pkg_subtask[hsh]] + result_list
beehive_result = requests.get(
"http://bb.ipa.basealt.ru/RESULT/{}/check-beehive-result.log"
"".format(task_id))
beehive_result = "" if beehive_result.status_code != 200 \
else beehive_result.content.decode()
task_msg_req = requests.get(
'http://git.altlinux.org/tasks/{task}/logs/events.{ti}.log'
''.format(task=task_id, ti=try_iteration))
task_msg = ''
if task_msg_req.status_code == 200:
message = re.findall(r'message:(.*)', task_msg_req.content.decode())
if message:
task_msg = message[0].strip()
result_list = []
for pkg in src_pkgs:
pkg = [
*name_hsh[pkg[0]][:-2], branch, user_id, task_status,
*pkg_subtask[pkg[0]], task_msg, try_iteration,
sorted(all_rebuilds),
utils.tuplelist_to_dict([(name_hsh[hsh][3], name_hsh[hsh][0])
for hsh in pkg[4]],
1), name_hsh[pkg[0]][-1], beehive_result
]
if pkg not in result_list:
result_list.append(pkg)
fields = [
'src_pkg', 'version', 'release', 'branch', 'user', 'status', 'subtask',
'approve', 'disapprove', 'task_msg', 'current_rebuild', 'all_rebuilds',
'task_content', 'description', 'beehive_check'
]
return utils.convert_to_json(fields, sorted(result_list,
key=itemgetter(6)))
def scrape_feed(self):
p = self.ig_parser
pf = self.post_filter
# open instagram main page, so that method can be used in loop
self.open_link('https://www.instagram.com/')
random_sleep()
# scroll feed and gather post links from timestamp element
post_links = self.fetch_posts(config.num_posts)
random_sleep()
# loop through post links
print('Parsing posts...')
print('-------------------------------------------------------------------------------')
for i, post_link in enumerate(post_links):
self.open_link(post_link)
post = p.post_parser(post_link)
filtered_post = pf.filter_post(post, config, post_keywords)
is_relevant_post = pf.is_relevant_post(filtered_post)
random_sleep()
# like post
if config.like_post == True and pt.like_counter < config.num_likes:
like = self.click_button(xpaths.like_button, 'like button')
if like == True:
pt.like_counter += 1
else:
pt.already_liked_counter += 1
# breaks if only old posts
if pt.already_liked_counter > config.max_already_liked:
break
# skips irrelevant posts
if is_relevant_post == False:
continue
# if matched post, save and try to find location
if filtered_post.matches_keyword == True:
pt.matched_post_counter += 1
if config.save_post == True:
self.click_button(xpaths.save_button, 'save button')
pt.save_counter += 1
as_json = convert_to_json(post)
pt.all_posts.append(as_json)
if post.location != '':
try:
position = self.find_lat_long(post.location)
except ValueError:
pass
if position != '':
self.lat_lon_to_gcs(position, post)
point = Point((post.lon, post.lat))
date_as_string = post.date.isoformat()
short_date = date_as_string[:11]
setattr(post, 'date', short_date)
pt.posts_with_location.append(Feature(geometry=point, properties = asdict(post)))
# try:
# self.open_link(post.author)
# profile = p.profile_parser(post.author, location_keywords)
# setattr(post, 'lat', profile.lat)
# setattr(post, 'lon', profile.lon)
# except ValueError:
# pass
print('Making json files.')
with open("matched_posts.json", "w") as outfile:
json.dump(pt.all_posts, outfile, indent = 4)
with open("matched_posts_locations.json", "w") as outfile:
json.dump(pt.posts_with_location, outfile, indent = 4)
print('-------------------------------------------------------------------------------')
