def call_hello(url):
req = urlopen(url)
txt = req.read()
buf = BytesIO(txt)
tree = xml_parse(buf)
root = tree.getroot()
return root
def load_labels(self, annotations_file):
annotations_path = join(self.data_dir, annotations_file)
data_dir, _ = split(annotations_path)
annotations = xml_parse(annotations_path)
root = annotations.getroot()
labels = {}
for image in root.findall('image'):
name = image.attrib.get('name')
if not exists(join(self.images_dir, name)):
continue
boxes = []
classes = []
scores = []
for box in image:
boxes.append(
[
box.get('xtl'),
box.get('ytl'),
box.get('xbr'),
box.get('ybr'),
]
)
attributes = box.iter('attribute')
attribute = next(attributes)
classes.append(MAP_IDS[attribute.text])
try:
attribute = next(attributes)
scores.append(float(attribute.text))
except StopIteration:
scores.append(1.)
boxes = np.array(boxes, dtype=np.float16)
classes = np.array(classes, dtype=int)
scores = np.array(scores, dtype=np.float16)
labels[name] = (boxes, classes, scores)
return labels
def parse_from_xml(cls, source, tag=None, parser=None):
"""Create an instance from an XML file object."""
tag = tag or cls.standard_tag
tree = xml_parse(source, parser)
element = tree.getroot()
if element.tag != tag:
raise ParseError("expected '%s' element, got '%s'"
% (tag, element.tag))
return cls.from_xml(element)
def hugexml_bad_memory_parse():
doc = xml_parse('items_bigsize.xml')
items_by_zip = Counter()
for item in doc.iterfind('row/row'):
items_by_zip[item.findtext('zip')] +=1
for zipcode, num in items_by_zip.most_common():
print(zipcode, num)
def _stencil_from_svg(cls, path: str) -> Optional[Stencil]:
"""
Read SVG and fetch metadata
:param path:
:return: Stencil insance or None in case of parsing error
"""
# Get library path
group = os.path.basename(os.path.dirname(path))
file_name = os.path.basename(path)
s_path = "%s/%s" % (group, file_name[:-4])
# Parse SVG
try:
doc = xml_parse(path)
except XMLParseError as e:
logger.info("[%s] Cannot parse stencil: %s", s_path, e)
return None
svg = doc.getroot()
# Get attributes
# width
width = svg.attrib.get("width")
if not width:
logger.info(
"[%s] Cannot parse stencil: <svg> tag contains no 'width' attribute",
s_path)
return None
# height
height = svg.attrib.get("height")
if not height:
logger.info(
"[%s] Cannot parse stencil: <svg> tag contains no 'height' attribute",
s_path)
return None
# Get title from <title> tag
title_el = svg.find("{http://www.w3.org/2000/svg}title")
if title_el is None or title_el.text is None:
title = None
else:
title = title_el.text.strip()
if title:
title = "%s | %s" % (group, title)
else:
logger.info(
"[%s] Missed <title> tag, generating title from file name",
s_path)
title = "%s | %s" % (group, os.path.basename(path)[:-4])
# Build Stencil structure
return Stencil(id=s_path,
title=title,
width=float(width),
height=float(height),
path=s_path)
def initUser_dx(request):
"""
"""
new_url = request.META["QUERY_STRING"]
mReturnURL = None
info = None
print(new_url)
new_url = urllib.parse.unquote(new_url)
print(new_url)
for i in new_url.split("&"):
info_self = re.findall("INFO=.*", i)
mReturnURL = re.findall("ReturnURL=.*", i)
if info_self:
info = info_self[0]
if mReturnURL:
mReturnURL = mReturnURL[0][10:]
request.session["INFO"] = info[5:]
info = urllib.parse.unquote(info[5:])
print(info, 'info--------------')
# 包月大厅拦截
request.session["home_index"] = "home_index"
if ("back_epg_url" in info) and ("back_hall_url" not in info):
# if "back_epg_url" in info:
# 从epg到游戏专区，需要转化成从大厅到游戏专区的参数
info = info.replace("back_epg_url", "back_hall_url")
# GET = SXML.parseString("<root>%s</root>" % info)
with open('info.txt', 'w') as f:
f.write(info)
GET = xml_parse("info.txt")
try:
back_epg_url = GET["back_epg_url"].asString
except:
back_epg_url = GET["back_hall_url"].asString
# 记录来源
try:
request.session["epgPlatform"] = GET["epgPlatform"].asString
except:
request.session["epgPlatform"] = 1
request.session["back_epg_url"] = back_epg_url
TokenExpiretime = GET["TokenExpiretime"].asString
GroupId = GET["GroupId"].asString
back_hall_url = None
if mReturnURL:
mReturnURL = urllib.unquote(mReturnURL)
# logger.info("mReturnURL2--------: %s", mReturnURL)
if mReturnURL.find('backUrl') != -1:
for i in mReturnURL.split("?"):
back_hall_url = re.findall("backUrl=.*", i)
if back_hall_url:
back_hall_url = back_hall_url[0][8:]
break
back_hall_url = urllib.unquote(back_hall_url)
else:
back_hall_url = mReturnURL
# logger.info('back_hall_url:%s'% back_hall_url)
if not back_hall_url:
# back_hall_url = "http://110.190.90.140:8296/main/sichuan/gqgameshall/gqgameshall.jsp"
back_hall_url = "http://110.190.90.140:8296/main/scgameshall/init_gqgameshall.action"
# back_hall_url = ReturnURL
areaCode = GET["areaCode"].asString
stbId = GET["stbId"].asString
userIP = GET["userIP"].asString
try:
vas_to_epg = GET["VAStoEPG"].asString
except:
vas_to_epg = "" # "http://58.223.143.84:8080/iptvepg/frame129/third_to_epg.jsp"
request.session["vas_to_epg"] = vas_to_epg.split('third_to_epg')[0]
user_message = GET["userId"].asString
try:
key = GET["key"].asString
if key == "null":
raise Exception(info)
userid = getOriginalInfo(key, user_message)
token_message = GET["userToken"].asString
userToken = getOriginalInfo(key, token_message)
except:
userid = user_message
# logger.info("getOriginalInfo(): userid '%s'; '%s'" % (userid, info))
token_message = GET["userToken"].asString
userToken = token_message
# 给页面游戏使用的返回地址
# gamereturnurl = "INFO=<userId>%s</userId><userToken>%s</userToken><TokenExpiretime>%s</TokenExpiretime><GroupId>%s</GroupId><userIP>%s</userIP><petName></petName><areaCode>%s</areaCode><TradeId>null</TradeId><stbId>%s</stbId><back_hall_url>%s</back_hall_url>" % (
# userid, userToken, TokenExpiretime, GroupId, userIP, areaCode, stbId,
# back_hall_url)
# 传给新大厅的参数
# iptvhall_param = "INFO=<userId>%s</userId><userToken>%s</userToken><TokenExpiretime>%s</TokenExpiretime><GroupId>%s</GroupId><userIP>%s</userIP><petName></petName><areaCode>%s</areaCode><TradeId>null</TradeId><stbId>%s</stbId><back_hall_url>%s</back_hall_url><from>%s</from>"%(userid, userToken, TokenExpiretime, GroupId, userIP, areaCode, stbId, back_hall_url, request.GET.get("from", ""))
backurl_param = (GET["userId"].asString, GET["userToken"].asString,
GET["TokenExpiretime"].asString, GET["GroupId"].asString,
GET["userIP"].asString, GET["areaCode"].asString,
GET["key"].asString, GET["stbId"].asString, "11111",
GET["epgPlatform"].asString)
backurl_info = "<userId>%s</userId><userToken>%s</userToken><TokenExpiretime>%s</TokenExpiretime><GroupId>%s</GroupId><userIP>%s</userIP><areaCode>%s</areaCode><TradeId></TradeId><key>%s</key><stbId>%s</stbId><optFlag>GAME</optFlag><SPID>%s</SPID><epgPlatform>%s</epgPlatform>" % backurl_param
backurl_info = urllib.quote(backurl_info)
if back_hall_url.find("?") != -1:
returnURL = back_hall_url + "&INFO=" + backurl_info
else:
returnURL = back_hall_url + "?INFO=" + backurl_info
# returnURL = returnURL + "&ReturnURL=" + mReturnURL
request.session["back_epg_url"] = returnURL
# request.session["gamereturnurl"] = gamereturnurl #20180813 注释
request.session["new_url"] = new_url
# request.session["iptvhall_param"] = iptvhall_param
# 电信采集数据
request.session["areacode"] = areaCode
request.session["stbid"] = stbId
# 把需要保存的数据写入session
request.session["return_url"] = returnURL
request.session["userid"] = userid # 用户帐号
request.session["username"] = "******"
request.session["usertoken"] = userToken.encode('utf-8') # 20180813
request.session["epg_full_url"] = request.get_full_path()
request.session["dx_key"] = key
# request.session["home_index"] = "home_index"
#20180801修改，根据判断条件不一样
# if userid[0] in "hH": # 华为
# stbtype = "hw"
# else: # userID[0] in "pP" 中兴
# stbtype = "ztebw"
request.session["stbtype"] = ""
# 20180731创建参数信息统计表
return True
def refresh_db(self) -> None:
"""Generate database from configured layout and data sources."""
# Stop processing, if DB is not available
if self.app.db_cursor is None:
self.app.log.error("No database connection")
return
# get required params from app config
all_data_configs: List[Dict] = self.app.config.get("yager", "data")
# get required params from CLI
xml_files: str = self.app.pargs.xml_list
# init local vars
database_query: Optional[Cursor] = None
# make backup of main DB
database_query = self._query_db("PRAGMA database_list;")
if database_query:
db_list: List = database_query.fetchall()
db_name_dict: Dict = dict(map(lambda x: (x[1], x[2]), db_list))
main_db_path: str = Path(db_name_dict["main"])
db_backup_name: str = "{}.{}.bak".format(
main_db_path.name, int(datetime.utcnow().timestamp()))
self.app.log.info("Backup DB '{}' as '{}'".format(
main_db_path, db_backup_name))
copyfile(
main_db_path,
main_db_path.with_name(db_backup_name),
)
# delete all tables from existing DB but leave excluded ones
database_query = self._query_db(
"SELECT name FROM sqlite_master WHERE type == 'table';")
if database_query:
table_list: List[str] = database_query.fetchall()
for record_tuple in table_list:
table = record_tuple[0]
if table not in all_data_configs["exclude_from_refresh"]:
self.app.log.info("Deleting table '{}'".format(table))
self._query_db("DROP TABLE {};".format(table))
else:
self.app.log.info(
"Leaving table '{}' untouched".format(table))
# create table layout
for table in all_data_configs["layout"]:
table_name: str = table["name"]
table_columns: str = table["columns"].replace("\n", "")
data_source_type: str = table["data_source"].split(":")[0]
data_source: str = table["data_source"].split(":")[1]
self.app.log.info("Creating table '{}'".format(table_name))
database_query = self._query_db(
"CREATE TABLE IF NOT EXISTS {} ({})".format(
table_name, table_columns))
self.app.log.debug("Using data source '{}:{}'".format(
data_source_type, data_source))
if data_source_type == "csv":
data_source_path: Path = Path(data_source)
row_count: int = 0
self.app.log.info("Inserting data from CSV file '{}'".format(
data_source_path))
# read data from CSV
with open(data_source_path, "r") as csv_file:
csv_data = DictReader(csv_file)
self._query_db("BEGIN TRANSACTION")
for row in csv_data:
self._query_db(
"INSERT OR IGNORE INTO {} ({}) VALUES ('{}')".
format(
table_name,
",".join(row.keys()),
"','".join(row.values()),
))
row_count += 1
# Commit to DB every 100 records
if row_count % 100 == 0:
self._query_db("COMMIT")
self.app.log.info(
"Inserted {} records...".format(row_count))
self._query_db("BEGIN TRANSACTION")
self._query_db("COMMIT")
self.app.log.info(
"Total records inserted: {}".format(row_count))
elif data_source_type == "xml":
if xml_files:
for file_path in xml_files:
element_count: int = 0
self.app.log.info(
"Inserting data from XML file '{}'".format(
file_path))
xml_root: Element = xml_parse(
file_path).getroot() # noqa: S314
self.app.log.info(
"Finding all elements matching XPath '{}'".format(
data_source))
xml_data: List[Element] = xml_root.findall(data_source)
self.app.log.info("Found {} elements".format(
len(xml_data)))
self._query_db("BEGIN TRANSACTION")
for element in xml_data:
input_map: Dict[str] = table["input_map"]
input_map_parametrized: Dict[str] = table.get(
"input_map_parametrized", {})
row: Dict[str] = {}
# Expand straightforward input paramenters into values
for input_key, input_value_map in input_map.items(
):
element_mapped: Optional[
Element] = element.find(input_value_map)
if element_mapped is not None:
input_value: str = element_mapped.text
else:
input_value = "undefined"
row.update({input_key: input_value})
# Expand parametrized input paramenters into values
for (
input_key,
input_value_map,
) in input_map_parametrized.items():
input_value_map_expanded = input_value_map.format(
**row)
element_mapped: Optional[
Element] = xml_root.find(
input_value_map_expanded)
if element_mapped is not None:
input_value: str = element_mapped.text
else:
input_value = "undefined"
row.update({input_key: input_value})
self._query_db(
"INSERT OR IGNORE INTO {} ({}) VALUES ('{}')".
format(
table_name,
",".join(row.keys()),
"','".join(row.values()),
))
element_count += 1
# Commit to DB every 100 records
if element_count % 100 == 0:
self._query_db("COMMIT")
self.app.log.info(
"Inserted {} records...".format(
element_count))
self._query_db("BEGIN TRANSACTION")
self._query_db("COMMIT")
self.app.log.info(
"Total records inserted: {}".format(element_count))
else:
self.app.log.error("No XML files specified for data input")
else:
self.app.log.error(
"Unknown data source type '{}'".format(data_source_type))
# Import libraries
from xml.etree.ElementTree import parse as xml_parse
from pprint import pprint
# Read in the xml file with the parsed story
e = xml_parse("story.txt.xml").getroot()
# path_tags is a list of tuples: [(tag, attribute, attribute_id), ...]
def get_xml_struct(root, path_tags):
current = root
for tag, attribute, attribute_value in path_tags:
for c in current:
if c.tag == tag:
if attribute is not None and attribute_value is not None:
if c.attrib.get(attribute) == attribute_value:
current = c
break
else:
current = c
break
return current
def get_attrib_value(root, dep_type, dependent=True):
identifiers = {}
for c in root:
if c.tag == "dep":
if c.attrib.get("type", "") == dep_type:
for d in c:
if dependent and d.tag == "dependent":
return d.attrib.get("idx", "")
identifiers[d.tag] = d.attrib.get("idx", "")
def parse_install(self):
from xml.etree.ElementTree import parse as xml_parse
tree = xml_parse(self.install_log_path).getroot()
self.parse_mods(tree)
self.parse_files(tree)
def parse_install(self):
from xml.etree.ElementTree import parse as xml_parse
tree = xml_parse(self.install_log_path).getroot()
self.parse_mods(tree)
self.parse_files(tree)
def _get_root():
global _FILE_PATH
path = os.path.join(os.path.dirname(__file__), _FILE_PATH)
tree = xml_parse(path)
return tree.getroot()
def as_svg(self):
"""
:return:
"""
template = path.join(path.abspath(path.dirname(__file__)),
self.__template__)
xml = xml_parse(template)
root = xml.getroot()
if not self.is_clean:
self._clean_all()
# Full name
for item in root.findall(
".//{http://www.w3.org/2000/svg}text[@id='card_full_name']"):
item.text = self.name
# Title
for item in root.findall(
".//{http://www.w3.org/2000/svg}text[@id='card_title']"):
item.text = self.title
# Company
for item in root.findall(
".//{http://www.w3.org/2000/svg}text[@id='card_company']"):
item.text = self.company
# Work Phone
for item in root.findall(
".//{http://www.w3.org/2000/svg}text[@id='card_work_phone_number']"
):
item.text = self.work_phone
# Cell Phone
for item in root.findall(
".//{http://www.w3.org/2000/svg}text[@id='card_cell_phone_number']"
):
item.text = self.cell_phone
# E-Mail
for item in root.findall(
".//{http://www.w3.org/2000/svg}text[@id='card_email']"):
item.text = self.email
# QR Code
qrcode = b64encode(self.as_qrcode()).decode("ascii")
for item in root.findall(
".//{http://www.w3.org/2000/svg}image[@id='card_qr_code']"):
# item.set('{http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd}absref', qrcode)
# item.set('{http://www.w3.org/1999/xlink}href', r"file://" + quote(qrcode))
item.set('{http://www.w3.org/1999/xlink}href',
"data:image/png;base64," + qrcode)
# convert svg to pdf
with BytesIO() as svg:
xml.write(svg, encoding='utf-8', xml_declaration=True)
svg.seek(0)
bytes = svg.getvalue()
return bytes
