def call_hello(url): req = urlopen(url) txt = req.read() buf = BytesIO(txt) tree = xml_parse(buf) root = tree.getroot() return root
def load_labels(self, annotations_file): annotations_path = join(self.data_dir, annotations_file) data_dir, _ = split(annotations_path) annotations = xml_parse(annotations_path) root = annotations.getroot() labels = {} for image in root.findall('image'): name = image.attrib.get('name') if not exists(join(self.images_dir, name)): continue boxes = [] classes = [] scores = [] for box in image: boxes.append( [ box.get('xtl'), box.get('ytl'), box.get('xbr'), box.get('ybr'), ] ) attributes = box.iter('attribute') attribute = next(attributes) classes.append(MAP_IDS[attribute.text]) try: attribute = next(attributes) scores.append(float(attribute.text)) except StopIteration: scores.append(1.) boxes = np.array(boxes, dtype=np.float16) classes = np.array(classes, dtype=int) scores = np.array(scores, dtype=np.float16) labels[name] = (boxes, classes, scores) return labels
def parse_from_xml(cls, source, tag=None, parser=None): """Create an instance from an XML file object.""" tag = tag or cls.standard_tag tree = xml_parse(source, parser) element = tree.getroot() if element.tag != tag: raise ParseError("expected '%s' element, got '%s'" % (tag, element.tag)) return cls.from_xml(element)
def hugexml_bad_memory_parse(): doc = xml_parse('items_bigsize.xml') items_by_zip = Counter() for item in doc.iterfind('row/row'): items_by_zip[item.findtext('zip')] +=1 for zipcode, num in items_by_zip.most_common(): print(zipcode, num)
def _stencil_from_svg(cls, path: str) -> Optional[Stencil]: """ Read SVG and fetch metadata :param path: :return: Stencil insance or None in case of parsing error """ # Get library path group = os.path.basename(os.path.dirname(path)) file_name = os.path.basename(path) s_path = "%s/%s" % (group, file_name[:-4]) # Parse SVG try: doc = xml_parse(path) except XMLParseError as e: logger.info("[%s] Cannot parse stencil: %s", s_path, e) return None svg = doc.getroot() # Get attributes # width width = svg.attrib.get("width") if not width: logger.info( "[%s] Cannot parse stencil: <svg> tag contains no 'width' attribute", s_path) return None # height height = svg.attrib.get("height") if not height: logger.info( "[%s] Cannot parse stencil: <svg> tag contains no 'height' attribute", s_path) return None # Get title from <title> tag title_el = svg.find("{http://www.w3.org/2000/svg}title") if title_el is None or title_el.text is None: title = None else: title = title_el.text.strip() if title: title = "%s | %s" % (group, title) else: logger.info( "[%s] Missed <title> tag, generating title from file name", s_path) title = "%s | %s" % (group, os.path.basename(path)[:-4]) # Build Stencil structure return Stencil(id=s_path, title=title, width=float(width), height=float(height), path=s_path)
def initUser_dx(request): """ """ new_url = request.META["QUERY_STRING"] mReturnURL = None info = None print(new_url) new_url = urllib.parse.unquote(new_url) print(new_url) for i in new_url.split("&"): info_self = re.findall("INFO=.*", i) mReturnURL = re.findall("ReturnURL=.*", i) if info_self: info = info_self[0] if mReturnURL: mReturnURL = mReturnURL[0][10:] request.session["INFO"] = info[5:] info = urllib.parse.unquote(info[5:]) print(info, 'info--------------') # 包月大厅拦截 request.session["home_index"] = "home_index" if ("back_epg_url" in info) and ("back_hall_url" not in info): # if "back_epg_url" in info: # 从epg到游戏专区,需要转化成从大厅到游戏专区的参数 info = info.replace("back_epg_url", "back_hall_url") # GET = SXML.parseString("<root>%s</root>" % info) with open('info.txt', 'w') as f: f.write(info) GET = xml_parse("info.txt") try: back_epg_url = GET["back_epg_url"].asString except: back_epg_url = GET["back_hall_url"].asString # 记录来源 try: request.session["epgPlatform"] = GET["epgPlatform"].asString except: request.session["epgPlatform"] = 1 request.session["back_epg_url"] = back_epg_url TokenExpiretime = GET["TokenExpiretime"].asString GroupId = GET["GroupId"].asString back_hall_url = None if mReturnURL: mReturnURL = urllib.unquote(mReturnURL) # logger.info("mReturnURL2--------: %s", mReturnURL) if mReturnURL.find('backUrl') != -1: for i in mReturnURL.split("?"): back_hall_url = re.findall("backUrl=.*", i) if back_hall_url: back_hall_url = back_hall_url[0][8:] break back_hall_url = urllib.unquote(back_hall_url) else: back_hall_url = mReturnURL # logger.info('back_hall_url:%s'% back_hall_url) if not back_hall_url: # back_hall_url = "http://110.190.90.140:8296/main/sichuan/gqgameshall/gqgameshall.jsp" back_hall_url = "http://110.190.90.140:8296/main/scgameshall/init_gqgameshall.action" # back_hall_url = ReturnURL areaCode = GET["areaCode"].asString stbId = GET["stbId"].asString userIP = GET["userIP"].asString try: vas_to_epg = GET["VAStoEPG"].asString except: vas_to_epg = "" # "http://58.223.143.84:8080/iptvepg/frame129/third_to_epg.jsp" request.session["vas_to_epg"] = vas_to_epg.split('third_to_epg')[0] user_message = GET["userId"].asString try: key = GET["key"].asString if key == "null": raise Exception(info) userid = getOriginalInfo(key, user_message) token_message = GET["userToken"].asString userToken = getOriginalInfo(key, token_message) except: userid = user_message # logger.info("getOriginalInfo(): userid '%s'; '%s'" % (userid, info)) token_message = GET["userToken"].asString userToken = token_message # 给页面游戏使用的返回地址 # gamereturnurl = "INFO=<userId>%s</userId><userToken>%s</userToken><TokenExpiretime>%s</TokenExpiretime><GroupId>%s</GroupId><userIP>%s</userIP><petName></petName><areaCode>%s</areaCode><TradeId>null</TradeId><stbId>%s</stbId><back_hall_url>%s</back_hall_url>" % ( # userid, userToken, TokenExpiretime, GroupId, userIP, areaCode, stbId, # back_hall_url) # 传给新大厅的参数 # iptvhall_param = "INFO=<userId>%s</userId><userToken>%s</userToken><TokenExpiretime>%s</TokenExpiretime><GroupId>%s</GroupId><userIP>%s</userIP><petName></petName><areaCode>%s</areaCode><TradeId>null</TradeId><stbId>%s</stbId><back_hall_url>%s</back_hall_url><from>%s</from>"%(userid, userToken, TokenExpiretime, GroupId, userIP, areaCode, stbId, back_hall_url, request.GET.get("from", "")) backurl_param = (GET["userId"].asString, GET["userToken"].asString, GET["TokenExpiretime"].asString, GET["GroupId"].asString, GET["userIP"].asString, GET["areaCode"].asString, GET["key"].asString, GET["stbId"].asString, "11111", GET["epgPlatform"].asString) backurl_info = "<userId>%s</userId><userToken>%s</userToken><TokenExpiretime>%s</TokenExpiretime><GroupId>%s</GroupId><userIP>%s</userIP><areaCode>%s</areaCode><TradeId></TradeId><key>%s</key><stbId>%s</stbId><optFlag>GAME</optFlag><SPID>%s</SPID><epgPlatform>%s</epgPlatform>" % backurl_param backurl_info = urllib.quote(backurl_info) if back_hall_url.find("?") != -1: returnURL = back_hall_url + "&INFO=" + backurl_info else: returnURL = back_hall_url + "?INFO=" + backurl_info # returnURL = returnURL + "&ReturnURL=" + mReturnURL request.session["back_epg_url"] = returnURL # request.session["gamereturnurl"] = gamereturnurl #20180813 注释 request.session["new_url"] = new_url # request.session["iptvhall_param"] = iptvhall_param # 电信采集数据 request.session["areacode"] = areaCode request.session["stbid"] = stbId # 把需要保存的数据写入session request.session["return_url"] = returnURL request.session["userid"] = userid # 用户帐号 request.session["username"] = "******" request.session["usertoken"] = userToken.encode('utf-8') # 20180813 request.session["epg_full_url"] = request.get_full_path() request.session["dx_key"] = key # request.session["home_index"] = "home_index" #20180801修改,根据判断条件不一样 # if userid[0] in "hH": # 华为 # stbtype = "hw" # else: # userID[0] in "pP" 中兴 # stbtype = "ztebw" request.session["stbtype"] = "" # 20180731创建参数信息统计表 return True
def refresh_db(self) -> None: """Generate database from configured layout and data sources.""" # Stop processing, if DB is not available if self.app.db_cursor is None: self.app.log.error("No database connection") return # get required params from app config all_data_configs: List[Dict] = self.app.config.get("yager", "data") # get required params from CLI xml_files: str = self.app.pargs.xml_list # init local vars database_query: Optional[Cursor] = None # make backup of main DB database_query = self._query_db("PRAGMA database_list;") if database_query: db_list: List = database_query.fetchall() db_name_dict: Dict = dict(map(lambda x: (x[1], x[2]), db_list)) main_db_path: str = Path(db_name_dict["main"]) db_backup_name: str = "{}.{}.bak".format( main_db_path.name, int(datetime.utcnow().timestamp())) self.app.log.info("Backup DB '{}' as '{}'".format( main_db_path, db_backup_name)) copyfile( main_db_path, main_db_path.with_name(db_backup_name), ) # delete all tables from existing DB but leave excluded ones database_query = self._query_db( "SELECT name FROM sqlite_master WHERE type == 'table';") if database_query: table_list: List[str] = database_query.fetchall() for record_tuple in table_list: table = record_tuple[0] if table not in all_data_configs["exclude_from_refresh"]: self.app.log.info("Deleting table '{}'".format(table)) self._query_db("DROP TABLE {};".format(table)) else: self.app.log.info( "Leaving table '{}' untouched".format(table)) # create table layout for table in all_data_configs["layout"]: table_name: str = table["name"] table_columns: str = table["columns"].replace("\n", "") data_source_type: str = table["data_source"].split(":")[0] data_source: str = table["data_source"].split(":")[1] self.app.log.info("Creating table '{}'".format(table_name)) database_query = self._query_db( "CREATE TABLE IF NOT EXISTS {} ({})".format( table_name, table_columns)) self.app.log.debug("Using data source '{}:{}'".format( data_source_type, data_source)) if data_source_type == "csv": data_source_path: Path = Path(data_source) row_count: int = 0 self.app.log.info("Inserting data from CSV file '{}'".format( data_source_path)) # read data from CSV with open(data_source_path, "r") as csv_file: csv_data = DictReader(csv_file) self._query_db("BEGIN TRANSACTION") for row in csv_data: self._query_db( "INSERT OR IGNORE INTO {} ({}) VALUES ('{}')". format( table_name, ",".join(row.keys()), "','".join(row.values()), )) row_count += 1 # Commit to DB every 100 records if row_count % 100 == 0: self._query_db("COMMIT") self.app.log.info( "Inserted {} records...".format(row_count)) self._query_db("BEGIN TRANSACTION") self._query_db("COMMIT") self.app.log.info( "Total records inserted: {}".format(row_count)) elif data_source_type == "xml": if xml_files: for file_path in xml_files: element_count: int = 0 self.app.log.info( "Inserting data from XML file '{}'".format( file_path)) xml_root: Element = xml_parse( file_path).getroot() # noqa: S314 self.app.log.info( "Finding all elements matching XPath '{}'".format( data_source)) xml_data: List[Element] = xml_root.findall(data_source) self.app.log.info("Found {} elements".format( len(xml_data))) self._query_db("BEGIN TRANSACTION") for element in xml_data: input_map: Dict[str] = table["input_map"] input_map_parametrized: Dict[str] = table.get( "input_map_parametrized", {}) row: Dict[str] = {} # Expand straightforward input paramenters into values for input_key, input_value_map in input_map.items( ): element_mapped: Optional[ Element] = element.find(input_value_map) if element_mapped is not None: input_value: str = element_mapped.text else: input_value = "undefined" row.update({input_key: input_value}) # Expand parametrized input paramenters into values for ( input_key, input_value_map, ) in input_map_parametrized.items(): input_value_map_expanded = input_value_map.format( **row) element_mapped: Optional[ Element] = xml_root.find( input_value_map_expanded) if element_mapped is not None: input_value: str = element_mapped.text else: input_value = "undefined" row.update({input_key: input_value}) self._query_db( "INSERT OR IGNORE INTO {} ({}) VALUES ('{}')". format( table_name, ",".join(row.keys()), "','".join(row.values()), )) element_count += 1 # Commit to DB every 100 records if element_count % 100 == 0: self._query_db("COMMIT") self.app.log.info( "Inserted {} records...".format( element_count)) self._query_db("BEGIN TRANSACTION") self._query_db("COMMIT") self.app.log.info( "Total records inserted: {}".format(element_count)) else: self.app.log.error("No XML files specified for data input") else: self.app.log.error( "Unknown data source type '{}'".format(data_source_type))
# Import libraries from xml.etree.ElementTree import parse as xml_parse from pprint import pprint # Read in the xml file with the parsed story e = xml_parse("story.txt.xml").getroot() # path_tags is a list of tuples: [(tag, attribute, attribute_id), ...] def get_xml_struct(root, path_tags): current = root for tag, attribute, attribute_value in path_tags: for c in current: if c.tag == tag: if attribute is not None and attribute_value is not None: if c.attrib.get(attribute) == attribute_value: current = c break else: current = c break return current def get_attrib_value(root, dep_type, dependent=True): identifiers = {} for c in root: if c.tag == "dep": if c.attrib.get("type", "") == dep_type: for d in c: if dependent and d.tag == "dependent": return d.attrib.get("idx", "") identifiers[d.tag] = d.attrib.get("idx", "")
def parse_install(self): from xml.etree.ElementTree import parse as xml_parse tree = xml_parse(self.install_log_path).getroot() self.parse_mods(tree) self.parse_files(tree)
def _get_root(): global _FILE_PATH path = os.path.join(os.path.dirname(__file__), _FILE_PATH) tree = xml_parse(path) return tree.getroot()
def as_svg(self): """ :return: """ template = path.join(path.abspath(path.dirname(__file__)), self.__template__) xml = xml_parse(template) root = xml.getroot() if not self.is_clean: self._clean_all() # Full name for item in root.findall( ".//{http://www.w3.org/2000/svg}text[@id='card_full_name']"): item.text = self.name # Title for item in root.findall( ".//{http://www.w3.org/2000/svg}text[@id='card_title']"): item.text = self.title # Company for item in root.findall( ".//{http://www.w3.org/2000/svg}text[@id='card_company']"): item.text = self.company # Work Phone for item in root.findall( ".//{http://www.w3.org/2000/svg}text[@id='card_work_phone_number']" ): item.text = self.work_phone # Cell Phone for item in root.findall( ".//{http://www.w3.org/2000/svg}text[@id='card_cell_phone_number']" ): item.text = self.cell_phone # E-Mail for item in root.findall( ".//{http://www.w3.org/2000/svg}text[@id='card_email']"): item.text = self.email # QR Code qrcode = b64encode(self.as_qrcode()).decode("ascii") for item in root.findall( ".//{http://www.w3.org/2000/svg}image[@id='card_qr_code']"): # item.set('{http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd}absref', qrcode) # item.set('{http://www.w3.org/1999/xlink}href', r"file://" + quote(qrcode)) item.set('{http://www.w3.org/1999/xlink}href', "data:image/png;base64," + qrcode) # convert svg to pdf with BytesIO() as svg: xml.write(svg, encoding='utf-8', xml_declaration=True) svg.seek(0) bytes = svg.getvalue() return bytes