def map_programme(el: ElementTree, channels_dict): category = el.find("category") return Programme( el.find("title").text, "" if category is None else category.text, channels_dict[el.get("channel")], datetime.strptime(el.get("start"), time_format), datetime.strptime(el.get("stop"), time_format))
def _from_etree(cls, el: ElementTree, base_href: str) -> LessonSection: title_el = el.find('./h2') if title_el is None or not title_el.text: raise LessonParseError( 'Lesson <section> needs a non-empty <h2> title') title = title_el.text steps_el = el.find('./ol[@class="steps"]') if steps_el is None or not steps_el: steps = list() else: steps = [ LessonSectionStep._from_etree(el, base_href) for el in steps_el ] # Now get the rest of the HTML, minus the <h1> and <ol> el.remove(title_el) # hacky mutation if steps_el is not None: el.remove(steps_el) # hacky mutation html = _build_inner_html(el, base_href) # Look for "fullscreen" class on section, set fullscreen flag if so is_full_screen = el.find('[@class="fullscreen"]') is not None return cls(title, html, steps, is_full_screen)
def _read_image(self, mrb_handle: MrbHandle, mrml: ElementTree) -> Image: volume_node = mrml.find("./Volume") volume_storage_node = mrml.find( "./VolumeArchetypeStorage[@id='{}']".format( volume_node.attrib["storageNodeRef"])) nrrd_data, nrrd_header = mrb_handle.read_nrrd( urllib.parse.unquote(volume_storage_node.attrib["fileName"])) assert nrrd_header["space"] == "left-posterior-superior",\ "Unsupported nrrd image volume space: " + nrrd_header["space"] assert [np.sign(x) for x in self._get_space_directions(nrrd_header["space directions"])] == [1., 1., 1.],\ "Unsupported nrrd image volume directions." # type-cast nrrd_data = nrrd_data.astype(np.int32) # switch sagittal-coronal-axial axis-order to axial-coronal-sagittal nrrd_data = np.swapaxes(nrrd_data, 0, 2) # posterior towards anterior nrrd_data = np.flip(nrrd_data, 1) image = Image(image_data=nrrd_data, identifier=self._derive_identifier( volume_node.attrib["name"]), voxel_spacing=self._get_voxel_spacing( nrrd_header["space directions"])) return image
def _parse_gateway(gateway_child: ElementTree) -> Gateway: id = gateway_child.get('id') name = gateway_child.find('Name').text type = gateway_child.find('Type').text.lower() gates = [] distribution = None rule = None if type == GATEWAY_TYPES['choice']: distribution_child = gateway_child.find('Distribution') rule_child = gateway_child.find('Rule') if distribution_child is not None and len( list(distribution_child)): distribution = [] for gate in distribution_child: gates.append(gate.get('id')) distribution.append(float(gate.text)) elif rule_child is not None and len(list(rule_child)): for gate in rule_child: gates.append(gate.get('id')) rule = id else: raise ValueError( "For choice gateways, either rule or distribution must be present." ) elif type == GATEWAY_TYPES['parallel'] or type == GATEWAY_TYPES[ 'merge']: gates_child = gateway_child.find('Gates') for gate in gates_child: gates.append(gate.get('id')) return Gateway(id=id, name=name, type=type, gates=gates, distribution=distribution, rule=rule)
def from_xml(xml_tree : ElementTree): path = xml_tree.find('path').text description = xml_tree.find('description').text info = xml_tree.find('info') timestamp = info.get('time-stamp') cid = int(info.get('camera-id')) return Image(path, timestamp, cid, description)
def __init__(self, xml_str): if not xml_str: raise Exception("Empty string") xml_data = ET.fromstring(xml_str) self.to_user = ET.find('ToUserName').text self.from_user = ET.find('FromUserName').text self.content = ET.find('Content').text self.msg_id = ET.find('MsgId').text self.msg_type = ET.find('MsgType').text
def get_asd_reference_tuple(asd: ET): thing_id: ET = asd.find("ThingReferenceId") network_id: ET = asd.find("NetworkReferenceId") thing_key = thing_id.text network_key = network_id.text position_key = None if thing_key == '0' and network_key == '0': position: ET = asd.find('Position') position_key = vector_to_sortable_string(position) return thing_key, network_key, position_key, asd
def configure(self, config_tree: ElementTree): userinfo = config_tree.find('credentials') self.mail_user = userinfo.find('username').get('value') self.mail_password = userinfo.find('password').get('value') gateway = config_tree.find('gateway') self.gateway_address = gateway.get('address') self.gateway_port = gateway.get('port')
def parse_member(member: ElementTree, tree: ElementTree) -> CMember: e_type = member.find('./type') type_string = (member.text or '') + e_type.text + (e_type.tail or '') e_name = member.find('./name') name = e_name.text array_size: int = None if e_name.tail and e_name.tail.startswith('['): match = re.match('\[\s*(\d+)\s*\]', e_name.tail) if match: array_size = int(match.group(1)) else: e_enum = member.find('./enum') array_size = int( tree.find( f'./enums/enum[@name="{e_enum.text}"]').attrib['value']) type_strings = type_string.split('*') first_type = type_strings[0].split() is_const = False if 'const' in first_type: first_type.remove('const') is_const = True if 'struct' in first_type: first_type.remove('struct') c_type = CType(name=' '.join(first_type), const=is_const) pointers = type_strings[1:] lengths = member.attrib['len'].split(',') if 'len' in member.attrib else [] optionals = member.attrib['optional'].split( ',') if 'optional' in member.attrib else [] if len(optionals) > len(pointers): c_type.optional = optionals.pop(-1) == 'true' lengths = reversed(lengths[:len(pointers)]) optionals = reversed(optionals[:len(pointers)]) for pointer, length, optional in zip_longest(pointers, lengths, optionals): c_type = CType(pointer_to=c_type, length=length, const='const' in pointer, optional=optional == 'true') if array_size is not None: c_type = CType(array_of=c_type, length=array_size) values_string = member.get('values') values = values_string.split(',') if values_string else [] return CMember(name, c_type, values)
def _from_etree(cls, el: ElementTree, base_href: str) -> LessonFooter: title_el = el.find('./h2') if title_el is None or not title_el.text: raise LessonParseError( 'Lesson <footer> needs a non-empty <h2> title') title = title_el.text is_full_screen = el.find('[@class="fullscreen"]') is not None # Now get the rest of the HTML, minus the <h2> el.remove(title_el) # hacky mutation html = _build_inner_html(el, base_href) return cls(title, html, is_full_screen)
def _prep_xmltext(et: Et, key: str) -> str: """Wrapper to quickly get settings. :param et: Element Tree from parsed Xml :type et: xml.etree.ElementTree :param key: Name of the setting :type key: str :return: Parsed value for setting :rtype: str """ t = et.find(key).text.strip() if et.findtext(key) and ( et.find(key).text is not None) else None return None if not t or t == "" else t
def get_date(article: ElementTree) -> int: """ Extracts the year of the article. If ArticleDate exist use its year otherwise use the year form JournalIssue """ d = article.find("ArticleDate") if d is not None: return int(d.find("Year").text) d = article.find("Journal").find("JournalIssue").find("PubDate") y = d.find("Year") if y is not None: return int(y.text) return int(d.find("MedlineDate").text.split(" ")[0].split("-")[0])
def __init__(self): self.unitTechs = [] self.buildingTechs = [] self.mothershipTechs = [] tree = ElementTree(file="TechTree.xml") units = tree.find("Units") for u in units.findall("Tech"): self.unitTechs.append(Tech(self, u)) buildings = tree.find("Buildings") for b in buildings.findall("Tech"): self.buildingTechs.append(Tech(self, b)) motherships = tree.find("Mothership") for m in motherships.findall("Tech"): self.mothershipTechs.append(Tech(self, m))
def __init__(self, currency: ElementTree, date): self.date = date self.code = currency.get('Kod') temp = currency.find('Unit').text self.unit = int(temp) if temp else None self.name = currency.find('Isim').text self.currency_name = currency.find('CurrencyName').text temp = currency.find('ForexBuying').text self.forex_buying = float(temp) if temp else None temp = currency.find('ForexSelling').text self.forex_selling = float(temp) if temp else None temp = currency.find('BanknoteBuying').text self.banknote_buying = float(temp) if temp else None temp = currency.find('BanknoteSelling').text self.banknote_selling = float(temp) if temp else None cross_rate_usd = currency.find('CrossRateUSD').text self.cross_rate_usd = float(cross_rate_usd) if cross_rate_usd else None
def _common_check_response(xml_response: Tree, method_name: str, statistic: Statistic) -> Tree: logger = statistic.get_log().get_logger("scripts/xml/zmq") logger.info("was called (xml_response)") logger.debug("(" + Tree.tostring(xml_response).decode() + ")") if xml_response == -1: statistic.append_error(method_name + " завершился с ошибкой!", "ZMQ_ОТВЕТ") return -1 status = int(xml_response.find(".//status").text) if status != 0: status_descr = xml_response.find(".//statusDescr").text statistic.append_error("Статус ответа: " + str(status_descr), "ZMQ_ОТВЕТ") return -1 return xml_response
def _get_segmentation_volume( self, mrb_handle: MrbHandle, mrml: ElementTree, segmentation_node: ElementTree) -> Tuple[np.ndarray, dict]: segmentation_storage_node = mrml.find( "./SegmentationStorage[@id='{}']".format( segmentation_node.attrib["storageNodeRef"])) nrrd_data, nrrd_header = mrb_handle.read_nrrd( segmentation_storage_node.attrib["fileName"]) assert len( np.shape(nrrd_data) ) == 4, "Unexpected segmentation mask volume shape: {}".format( np.shape(nrrd_data)) assert nrrd_header[ "space"] == "right-anterior-superior", "Unsupported segmentation mask nrrd space: " + nrrd_header[ "space"] assert [ np.sign(x) for x in self._get_space_directions( nrrd_header["space directions"][1:]) ] == [1., -1., -1.] # switch sagittal-coronal-axial axis-order to axial-coronal-sagittal nrrd_data = np.swapaxes(nrrd_data, 1, 3) # posterior towards anterior nrrd_data = np.flip(nrrd_data, 2) return nrrd_data, nrrd_header
def _cfg_set_or_create(root: ElementTree, match: str, new_value: str) -> None: ele = root.find(match) if ele is None: ele = ElementTree.SubElement(root, match) ele.text = new_value
def parse_bitmasks(self, tree: ElementTree): for e_bitmask in tree.findall('./types/type[@category="bitmask"]'): if 'alias' in e_bitmask.attrib: continue bitmask_name = e_bitmask.find('./name').text if self.should_ignore(type_=bitmask_name): continue c_bitmask = CBitmask(bitmask_name) requires = e_bitmask.get('requires') if requires: cases = {} e_enum = tree.find(f'./enums[@name="{requires}"]') for e_case in e_enum.findall('./enum'): if 'alias' in e_case.attrib: continue cases[e_case.attrib['name']] = parse_enum_value(e_case) for extension in self.extensions: if extension.supported == 'disabled' or extension.platform: continue for ext_enum in extension.enums: if ext_enum.name == requires: for ext_case in ext_enum.cases: cases[ext_case.name] = ext_case.value c_bitmask.enum = CEnum( requires, [CEnum.Case(name, value) for name, value in cases.items()]) self.bitmasks.append(c_bitmask)
def set_shape_text(shape: ET, text: str): t = shape.find(f"{namespace}Text") # type: Element if t is not None: if t.text: t.text = text else: t[0].tail = text
def get_shape_text(shape: ET) -> str: # technically the below is not an exact replacement of the above... text = "" text_elem = shape.find(f"{namespace}Text") if text_elem is not None: text = "".join(text_elem.itertext()) return text
def copy_shape(self, shape: Element, page: ET, page_path: str) -> ET: """Insert shape into first Shapes tag in destination page, and return the copy. If destination page does not have a Shapes tag yet, create it. Parameters: shape (Element): The source shape to be copied. Use Shape.xml page (ElementTree): The page where the new Shape will be placed. Use Page.xml page_path (str): The filename of the page where the new Shape will be placed. Use Page.filename Returns: ElementTree: The new shape ElementTree """ new_shape = ET.fromstring(ET.tostring(shape)) self.set_page_max_id(page_path) # find or create Shapes tag shapes_tag = page.find(f"{namespace}Shapes") if shapes_tag is None: shapes_tag = Element(f"{namespace}Shapes") page.getroot().append(shapes_tag) id_map = self.increment_shape_ids(new_shape, page_path) self.update_ids(new_shape, id_map) shapes_tag.append(new_shape) self.pages[page_path] = page return new_shape
def vector_to_sortable_string(vector:ET): result = '' for v in vector_components: text = vector.find(v).text sortable_string = number_string_to_sortable_string(text) result += sortable_string return result
def parse_image(tree: ElementTree): """Parses XML""" image_url = f"https://bing.com{tree.find('url').text}" image_base = f"https://bing.com{tree.find('urlBase').text}" image_date_raw = tree.find('startdate').text image_date = f"{image_date_raw[:4]}-{image_date_raw[4:6]}-{image_date_raw[6:]}" headline = tree.find('headline').text _copyright = tree.find('copyright').text return { 'image': image_url, 'image_base': image_base, 'title': headline, 'date': image_date, 'copyright': _copyright, }
def setup_streets(net: sumolib.net.Net, xml: ElementTree, pop_noise: NoiseSampler, work_noise: NoiseSampler): """ Create a street for each edge in the network and calculate its population and workplaces based on modified Perlin noise from NoiseSamplers :param net: the SUMO network :param xml: the statistics XML for the network :param pop_noise: NoiseSampler to use for population :param work_noise: NoiseSample to use for workplaces """ streets = xml.find("streets") if streets is None: streets = ET.SubElement(xml.getroot(), "streets") # Some edges might already have a street, so we want to ignore those known_streets = {street.attrib["edge"]: street for street in streets.findall("street")} for edge in net.getEdges(): eid = edge.getID() if eid not in known_streets: # This edge is missing a street entry. Find population and industry for this edge pos = get_edge_pair_centroid(edge.getShape()) population = pop_noise.sample(pos) industry = work_noise.sample(pos) logging.debug( f"[perlin] Adding street with eid: {eid},\t population: {population:.4f}, industry: {industry:.4f}") ET.SubElement(streets, "street", { "edge": eid, "population": str(population), "workPosition": str(industry) })
def getPocketPos(self, sWhichPocket, sWhichPosition): #print("%s-%s" % (sWhichPocket, sWhichPosition)) oXML = self.oXMLroot.find('Pockets/Pocket[@id="%s"]' % sWhichPocket) oXML2 = oXML.find('%s' % sWhichPosition) posX = oXML2.get("x") posY = oXML2.get("y") posZ = oXML2.get("z") return float(posX), float(posY), float(posZ)
def _cfg_load_stashes(root: ElementTree) -> list: stashes = root.find('stashes') if stashes is None: return [{'name': "", 'type': "normal"}] return [{ 'name': x.text, 'type': x.attrib['type'] } for x in list(stashes)]
def _parse_transition(self, transition_child: ElementTree) -> Transition: source = transition_child.get('source') destination = transition_child.get('destination') sgate = transition_child.get('source_gate') dgate = transition_child.get('destination_gate') duration_child = transition_child.find('Duration') distribution_child = transition_child.find('Duration/Distribution') if distribution_child is not None: distribution = self._parse_distribution(distribution_child) else: distribution = int( duration_child.text) if duration_child is not None else 0 return Transition(source=source, destination=destination, sgate=sgate, dgate=dgate, distribution=distribution)
def getSVO(sentence): """主語、述語、目的語を抽出""" dependencies = sentence.find( "dependencies[@type='collapsed-dependencies']") nsubj = dependencies.findall("dep[@type='nsubj']") dobj = dependencies.findall("dep[@type='dobj']") gove = lambda x: x.find("governor").text depa = lambda x: x.find("dependent").text answer = [] for ns in nsubj: for db in dobj: if gove(ns) == gove(db): answer.append((depa(ns), gove(ns), depa(db))) return answer
def set_game_info(self, info): self.info = info tree = ElementTree() tree.parse(info["path"]) link_node = tree.find("link") if link_node is not None: self.link_field.set_text(link_node.text.strip()) else: self.link_field.set_text("")
def from_etree(cls, obj: ElementTree) -> 'ObjectDef': return cls(name=_node_text(obj.find('Name')), description=_node_text(obj.find('Description1')), oid=int(_node_text(obj.find('ObjectID'))), urn=_node_text(obj.find('ObjectURN')), multiple={'Single': False, 'Multiple': True}[_node_text(obj.find('MultipleInstances'))], mandatory={'Optional': False, 'Mandatory': True}[_node_text(obj.find('Mandatory'))], resources=sorted([ResourceDef.from_etree(item) for item in obj.find('Resources').findall('Item')], key=operator.attrgetter('rid')))
def from_ET(et:ET) -> ThingSaveData: result_data = dict() for item in ThingSaveData._fields: value = et.find("./"+item) result_data[item] = value.text.strip() if value is not None and value.text is not None else None result_data['parsed_element'] = et result_data['xsi_type'] = et.get('{http://www.w3.org/2001/XMLSchema-instance}type') result = ThingSaveData(**result_data) return result
def _from_etree(cls, el: ElementTree, base_href: str) -> LessonHeader: title_el = el.find('./h1') if title_el is None or not title_el.text: raise LessonParseError( 'Lesson <header> needs a non-empty <h1> title') title = title_el.text # Now get the rest of the HTML, minus the <h1> el.remove(title_el) # hacky mutation html = _build_inner_html(el, base_href) return cls(title, html)
def setChannel(androidManifestFilePath,channel): ''' 修改AndroidManifest.xml中的UMENG_CHANNEL ''' if not os.path.exists(androidManifestFilePath): sys.exit(androidManifestFilePath+' 不存在') xml.etree.ElementTree.register_namespace("android","http://schemas.android.com/apk/res/android") tree = ElementTree() xmlRoot=tree.parse(androidManifestFilePath) channelNode=tree.find("./application/meta-data[@{http://schemas.android.com/apk/res/android}name='UMENG_CHANNEL']") channelNode.set("{http://schemas.android.com/apk/res/android}value",channel) tree.write(androidManifestFilePath, "UTF-8", True)
def parse_user_details(s): """Takes a string XML representation of a user's details and returns a dictionary of values we care about""" root = ET.find('./user') if not root: print 'aaaargh' return None user = { 'id': root.attrib['id'], 'username': root.attrib['display_name'] } try: user['lat'] = float(root.find('./home').attrib['lat']) user['lon'] = float(root.find('./home').attrib['lon']) except AttributeError: pass user['changesets'] = int(root.find('./changesets').attrib['count']) return user
class ConfigXmlWriter(RcbXmlReaderWriter): def __init__(self, createNew): Logutil.log('init ConfigXmlWriter', util.LOG_LEVEL_INFO) self.createNew = createNew if(createNew): configFile = os.path.join(util.getAddonInstallPath(), 'resources', 'database', 'config_template.xml') else: configFile = util.getConfigXmlPath() if(not os.path.isfile(configFile)): Logutil.log('File config.xml does not exist. Place a valid config file here: ' + str(configFile), util.LOG_LEVEL_ERROR) else: self.tree = ElementTree().parse(configFile) """ Functions for generating XML objects from objects """ def getXmlAttributesForRomCollection(self, rc): return {'id': str(rc.id), 'name': rc.name} def getXmlElementsForRomCollection(self, rc): elements = [] # String attributes for e in ['gameclient', 'emulatorCmd', 'emulatorParams', 'preCmd', 'postCmd', 'saveStatePath', 'saveStateParams']: try: el = Element(e, {}) el.text = getattr(rc, e) elements.append(el) except AttributeError as exc: # Skip any errors pass # Non-string attributes for e in ['useBuiltinEmulator', 'useEmuSolo', 'usePopen', 'ignoreOnScan', 'allowUpdate', 'autoplayVideoMain', 'autoplayVideoInfo', 'useFoldernameAsGamename', 'maxFolderDepth', 'doNotExtractZipFiles', 'makeLocalCopy', 'diskPrefix']: try: el = Element(e, {}) el.text = str(getattr(rc, e)) elements.append(el) except AttributeError as exc: # Skip any errors pass for romPath in rc.romPaths: el = Element('romPath', {}) el.text = romPath elements.append(el) return elements def getXmlAttributesForSite(self, site): attrs = {'name': site.name, 'path': site.path} return attrs def getXmlElementsForSite(self, site): """ Not needed """ pass def writeRomCollections(self, romCollections, isEdit): Logutil.log('write Rom Collections', util.LOG_LEVEL_INFO) romCollectionsXml = self.tree.find('RomCollections') #HACK: remove all Rom Collections and create new if(isEdit): for romCollectionXml in romCollectionsXml.findall('RomCollection'): romCollectionsXml.remove(romCollectionXml) for romCollection in romCollections.values(): Logutil.log('write Rom Collection: ' + str(romCollection.name), util.LOG_LEVEL_INFO) romCollectionXml = SubElement(romCollectionsXml, 'RomCollection', self.getXmlAttributesForRomCollection(romCollection)) for subel in self.getXmlElementsForRomCollection(romCollection): romCollectionXml.append(subel) for mediaPath in romCollection.mediaPaths: success, message = self.searchConfigObjects('FileTypes/FileType', mediaPath.fileType.name, 'FileType') if(not success): return False, message SubElement(romCollectionXml, 'mediaPath', {'type': mediaPath.fileType.name}).text = mediaPath.path #image placing if(not self.createNew): #in case of an update we have to create new options if(romCollection.name == 'MAME' and not self.createNew): self.addFileTypesForMame() self.addImagePlacingForMame() if(romCollection.imagePlacingMain != None and romCollection.imagePlacingMain.name != ''): success, message = self.searchConfigObjects('ImagePlacing/fileTypeFor', romCollection.imagePlacingMain.name, 'ImagePlacing') if(not success): return False, message SubElement(romCollectionXml, 'imagePlacingMain').text = romCollection.imagePlacingMain.name else: SubElement(romCollectionXml, 'imagePlacingMain').text = 'gameinfobig' if(romCollection.imagePlacingInfo != None and romCollection.imagePlacingInfo.name != ''): success, message = self.searchConfigObjects('ImagePlacing/fileTypeFor', romCollection.imagePlacingInfo.name, 'ImagePlacing') if(not success): return False, message SubElement(romCollectionXml, 'imagePlacingInfo').text = romCollection.imagePlacingInfo.name else: SubElement(romCollectionXml, 'imagePlacingInfo').text = 'gameinfosmall' if (romCollection.scraperSites == None or len(romCollection.scraperSites) == 0): #use thegamesdb.net as default scraper in online scraping scenario SubElement(romCollectionXml, 'scraper', {'name': 'thegamesdb.net'}) else: for scraperSite in romCollection.scraperSites: if(scraperSite == None): continue SubElement(romCollectionXml, 'scraper', {'name' : scraperSite.name, 'path' : scraperSite.path}) success, message = self.writeFile() return success, message def writeMissingFilter(self, showHideOption, artworkOrGroup, artworkAndGroup, infoOrGroup, infoAndGroup): Logutil.log('write Missing Info Filter', util.LOG_LEVEL_INFO) missingFilterXml = self.tree.find('MissingFilter') #HACK: remove MissingFilter-element if(missingFilterXml != None): self.tree.remove(missingFilterXml) missingFilterXml = SubElement(self.tree, 'MissingFilter') SubElement(missingFilterXml, 'showHideOption').text = showHideOption if(len(artworkOrGroup) > 0 or len(artworkAndGroup) > 0): missingArtworkXml = SubElement(missingFilterXml, 'missingArtworkFilter') self.addMissingFilterItems(missingArtworkXml, artworkOrGroup, 'orGroup') self.addMissingFilterItems(missingArtworkXml, artworkAndGroup, 'andGroup') if(len(infoOrGroup) > 0 or len(infoAndGroup) > 0): missingInfoXml = SubElement(missingFilterXml, 'missingInfoFilter') self.addMissingFilterItems(missingInfoXml, infoOrGroup, 'orGroup') self.addMissingFilterItems(missingInfoXml, infoAndGroup, 'andGroup') success, message = self.writeFile() return success, message def addMissingFilterItems(self, missingXml, group, groupName): if(len(group) > 0): groupXml = SubElement(missingXml, groupName) for item in group: SubElement(groupXml, 'item').text = item def searchConfigObjects(self, xPath, nameToCompare, objectType): objects = self.tree.findall(xPath) objectFound = False for obj in objects: objectName = obj.attrib.get('name') if(objectName == nameToCompare): objectFound = True break if(not objectFound): return False, util.localize(32009) % (objectType, nameToCompare) return True, '' def removeRomCollection(self, RCName): Logutil.log('removeRomCollection', util.LOG_LEVEL_INFO) configFile = util.getConfigXmlPath() self.tree = ElementTree().parse(configFile) romCollectionsXml = self.tree.find('RomCollections') for romCollectionXml in romCollectionsXml.findall('RomCollection'): name = romCollectionXml.attrib.get('name') if(name == RCName): romCollectionsXml.remove(romCollectionXml) success, message = self.writeFile() return success, message def addFileTypesForMame(self): Logutil.log('addFileTypesForMame', util.LOG_LEVEL_INFO) fileTypesXml = self.tree.find('FileTypes') #check if the MAME FileTypes already exist cabinetExists = False marqueeExists = False actionExists = False titleExists = False highestId = 0 fileTypeXml = fileTypesXml.findall('FileType') for fileType in fileTypeXml: name = fileType.attrib.get('name') if name == 'cabinet': cabinetExists = True elif name == 'marquee': marqueeExists = True elif name == 'action': actionExists = True elif name == 'title': titleExists = True id = fileType.attrib.get('id') if int(id) > highestId: highestId = int(id) if not cabinetExists: self.createFileType(fileTypesXml, str(highestId + 1), 'cabinet', 'image', 'game') if not marqueeExists: self.createFileType(fileTypesXml, str(highestId + 2), 'marquee', 'image', 'game') if not actionExists: self.createFileType(fileTypesXml, str(highestId + 3), 'action', 'image', 'game') if not titleExists: self.createFileType(fileTypesXml, str(highestId + 4), 'title', 'image', 'game') def createFileType(self, fileTypesXml, id, name, type, parent): fileType = SubElement(fileTypesXml, 'FileType', {'id' : str(id), 'name' : name}) SubElement(fileType, 'type').text = type SubElement(fileType, 'parent').text = parent def addImagePlacingForMame(self): Logutil.log('addImagePlacingForMame', util.LOG_LEVEL_INFO) imagePlacingXml = self.tree.find('ImagePlacing') #check if the MAME ImagePlacing options already exist cabinetExists = False marqueeExists = False fileTypeForXml = imagePlacingXml.findall('fileTypeFor') for fileTypeFor in fileTypeForXml: name = fileTypeFor.attrib.get('name') if name == 'gameinfomamecabinet': cabinetExists = True elif name == 'gameinfomamemarquee': marqueeExists = True if not cabinetExists: fileTypeFor = SubElement(imagePlacingXml, 'fileTypeFor', {'name' : 'gameinfomamecabinet'}) for imgtype in ['cabinet', 'boxfront', 'title']: SubElement(fileTypeFor, 'fileTypeForGameList').text = imgtype SubElement(fileTypeFor, 'fileTypeForGameListSelected').text = imgtype for imgtype in ['boxfront', 'title', 'action']: SubElement(fileTypeFor, 'fileTypeForMainViewBackground').text = imgtype SubElement(fileTypeFor, 'fileTypeForMainViewGameInfoUpperLeft').text = 'title' SubElement(fileTypeFor, 'fileTypeForMainViewGameInfoUpperRight').text = 'action' SubElement(fileTypeFor, 'fileTypeForMainViewGameInfoLower').text = 'marquee' if not marqueeExists: fileTypeFor = SubElement(imagePlacingXml, 'fileTypeFor', {'name' : 'gameinfomamemarquee'}) for imgtype in ['marquee', 'boxfront', 'title']: SubElement(fileTypeFor, 'fileTypeForGameList').text = imgtype SubElement(fileTypeFor, 'fileTypeForGameListSelected').text = imgtype for imgtype in ['boxfront', 'title', 'action']: SubElement(fileTypeFor, 'fileTypeForMainViewBackground').text = imgtype SubElement(fileTypeFor, 'fileTypeForMainViewGameInfoLeft').text = 'cabinet' SubElement(fileTypeFor, 'fileTypeForMainViewGameInfoUpperRight').text = 'action' SubElement(fileTypeFor, 'fileTypeForMainViewGameInfoLowerRight').text = 'title' # FIXME TODO This function is only called within this class - raise exception rather than return tuple def writeFile(self): Logutil.log('writeFile', util.LOG_LEVEL_INFO) #write file try: configFile = util.getConfigXmlPath() self.indentXml(self.tree) treeToWrite = ElementTree(self.tree) treeToWrite.write(configFile) return True, "" except Exception, (exc): print("Error: Cannot write config.xml: " + str(exc)) return False, util.localize(32008) + ": " + str(exc)
class ConfigXmlWriter: def __init__(self, createNew): Logutil.log('init ConfigXmlWriter', util.LOG_LEVEL_INFO) self.createNew = createNew if(createNew): configFile = os.path.join(util.getAddonInstallPath(), 'resources', 'database', 'config_template.xml') else: configFile = util.getConfigXmlPath() if(not os.path.isfile(configFile)): Logutil.log('File config.xml does not exist. Place a valid config file here: ' +str(configFile), util.LOG_LEVEL_ERROR) return False, util.localize(32003) self.tree = ElementTree().parse(configFile) def writeRomCollections(self, romCollections, isEdit): Logutil.log('write Rom Collections', util.LOG_LEVEL_INFO) romCollectionsXml = self.tree.find('RomCollections') #HACK: remove all Rom Collections and create new if(isEdit): for romCollectionXml in romCollectionsXml.findall('RomCollection'): romCollectionsXml.remove(romCollectionXml) for romCollection in romCollections.values(): Logutil.log('write Rom Collection: ' +str(romCollection.name), util.LOG_LEVEL_INFO) romCollectionXml = SubElement(romCollectionsXml, 'RomCollection', {'id' : str(romCollection.id), 'name' : romCollection.name}) SubElement(romCollectionXml, 'useBuiltinEmulator').text = str(romCollection.useBuiltinEmulator) SubElement(romCollectionXml, 'gameclient').text = romCollection.gameclient SubElement(romCollectionXml, 'emulatorCmd').text = romCollection.emulatorCmd SubElement(romCollectionXml, 'emulatorParams').text = romCollection.emulatorParams for romPath in romCollection.romPaths: SubElement(romCollectionXml, 'romPath').text = romPath SubElement(romCollectionXml, 'saveStatePath').text = romCollection.saveStatePath SubElement(romCollectionXml, 'saveStateParams').text = romCollection.saveStateParams for mediaPath in romCollection.mediaPaths: success, message = self.searchConfigObjects('FileTypes/FileType', mediaPath.fileType.name, 'FileType') if(not success): return False, message SubElement(romCollectionXml, 'mediaPath', {'type' : mediaPath.fileType.name}).text = mediaPath.path SubElement(romCollectionXml, 'preCmd').text = romCollection.preCmd SubElement(romCollectionXml, 'postCmd').text = romCollection.postCmd SubElement(romCollectionXml, 'useEmuSolo').text = str(romCollection.useEmuSolo) SubElement(romCollectionXml, 'usePopen').text = str(romCollection.usePopen) SubElement(romCollectionXml, 'ignoreOnScan').text = str(romCollection.ignoreOnScan) SubElement(romCollectionXml, 'allowUpdate').text = str(romCollection.allowUpdate) SubElement(romCollectionXml, 'autoplayVideoMain').text = str(romCollection.autoplayVideoMain) SubElement(romCollectionXml, 'autoplayVideoInfo').text = str(romCollection.autoplayVideoInfo) SubElement(romCollectionXml, 'useFoldernameAsGamename').text = str(romCollection.useFoldernameAsGamename) SubElement(romCollectionXml, 'maxFolderDepth').text = str(romCollection.maxFolderDepth) SubElement(romCollectionXml, 'doNotExtractZipFiles').text = str(romCollection.doNotExtractZipFiles) SubElement(romCollectionXml, 'diskPrefix').text = str(romCollection.diskPrefix) if (os.environ.get( "OS", "xbox" ) == "xbox"): SubElement(romCollectionXml, 'xboxCreateShortcut').text = str(romCollection.xboxCreateShortcut) SubElement(romCollectionXml, 'xboxCreateShortcutAddRomfile').text = str(romCollection.xboxCreateShortcutAddRomfile) SubElement(romCollectionXml, 'xboxCreateShortcutUseShortGamename').text = str(romCollection.xboxCreateShortcutUseShortGamename) #image placing if(not self.createNew): #in case of an update we have to create new options if(romCollection.name == 'MAME' and not self.createNew): self.addFileTypesForMame() self.addImagePlacingForMame() if(romCollection.imagePlacingMain != None and romCollection.imagePlacingMain.name != ''): success, message = self.searchConfigObjects('ImagePlacing/fileTypeFor', romCollection.imagePlacingMain.name, 'ImagePlacing') if(not success): return False, message SubElement(romCollectionXml, 'imagePlacingMain').text = romCollection.imagePlacingMain.name else: SubElement(romCollectionXml, 'imagePlacingMain').text = 'gameinfobig' if(romCollection.imagePlacingInfo != None and romCollection.imagePlacingInfo.name != ''): success, message = self.searchConfigObjects('ImagePlacing/fileTypeFor', romCollection.imagePlacingInfo.name, 'ImagePlacing') if(not success): return False, message SubElement(romCollectionXml, 'imagePlacingInfo').text = romCollection.imagePlacingInfo.name else: SubElement(romCollectionXml, 'imagePlacingInfo').text = 'gameinfosmall' if(romCollection.scraperSites == None or len(romCollection.scraperSites) == 0): SubElement(romCollectionXml, 'scraper', {'name' : 'thegamesdb.net', 'replaceKeyString' : '', 'replaceValueString' : ''}) SubElement(romCollectionXml, 'scraper', {'name' : 'archive.vg', 'replaceKeyString' : '', 'replaceValueString' : ''}) SubElement(romCollectionXml, 'scraper', {'name' : 'mobygames.com', 'replaceKeyString' : '', 'replaceValueString' : ''}) else: for scraperSite in romCollection.scraperSites: if(scraperSite == None): continue #HACK: use replaceKey and -Value only from first scraper firstScraper = scraperSite.scrapers[0] SubElement(romCollectionXml, 'scraper', {'name' : scraperSite.name, 'replaceKeyString' : firstScraper.replaceKeyString, 'replaceValueString' : firstScraper.replaceValueString}) #create Scraper element scrapersXml = self.tree.find('Scrapers') #check if the current scraper already exists siteExists = False sitesXml = scrapersXml.findall('Site') for site in sitesXml: name = site.attrib.get('name') if name == scraperSite.name: siteExists = True break if not siteExists: #HACK: this only covers the first scraper (for offline scrapers) site = SubElement(scrapersXml, 'Site', { 'name' : scraperSite.name, 'descFilePerGame' : str(scraperSite.descFilePerGame), 'searchGameByCRC' : str(scraperSite.searchGameByCRC), 'useFoldernameAsCRC' : str(scraperSite.useFoldernameAsCRC), 'useFilenameAsCRC' : str(scraperSite.useFilenameAsCRC) }) scraper = scraperSite.scrapers[0] SubElement(site, 'Scraper', { 'parseInstruction' : scraper.parseInstruction, 'source' : scraper.source, 'encoding' : scraper.encoding }) success, message = self.writeFile() return success, message def writeScrapers(self, scrapers): Logutil.log('write scraper sites', util.LOG_LEVEL_INFO) scraperSitesXml = self.tree.find('Scrapers') #HACK: remove all scrapers and create new for scraperSiteXml in scraperSitesXml.findall('Site'): scraperSitesXml.remove(scraperSiteXml) for scraperSite in scrapers.values(): Logutil.log('write scraper site: ' +str(scraperSite.name), util.LOG_LEVEL_INFO) #Don't write None-Scraper if(scraperSite.name == util.localize(32854)): Logutil.log('None scraper will be skipped', util.LOG_LEVEL_INFO) continue scraperSiteXml = SubElement(scraperSitesXml, 'Site', { 'name' : scraperSite.name, 'descFilePerGame' : str(scraperSite.descFilePerGame), 'searchGameByCRC' : str(scraperSite.searchGameByCRC), 'useFoldernameAsCRC' : str(scraperSite.useFoldernameAsCRC), 'useFilenameAsCRC' : str(scraperSite.useFilenameAsCRC) }) for scraper in scraperSite.scrapers: #check if we can use a relative path to parseInstructions rcbScraperPath = os.path.join(util.RCBHOME, 'resources', 'scraper') pathParts = os.path.split(scraper.parseInstruction) if(pathParts[0].upper() == rcbScraperPath.upper()): scraper.parseInstruction = pathParts[1] scraperXml = SubElement(scraperSiteXml, 'Scraper', { 'parseInstruction' : scraper.parseInstruction, 'source' : scraper.source, 'encoding' : scraper.encoding, 'returnUrl' : str(scraper.returnUrl) }) success, message = self.writeFile() return success, message def writeMissingFilter(self, showHideOption, artworkOrGroup, artworkAndGroup, infoOrGroup, infoAndGroup): Logutil.log('write Missing Info Filter', util.LOG_LEVEL_INFO) missingFilterXml = self.tree.find('MissingFilter') #HACK: remove MissingFilter-element if(missingFilterXml != None): self.tree.remove(missingFilterXml) missingFilterXml = SubElement(self.tree, 'MissingFilter') SubElement(missingFilterXml, 'showHideOption').text = showHideOption if(len(artworkOrGroup) > 0 or len(artworkAndGroup) > 0): missingArtworkXml = SubElement(missingFilterXml, 'missingArtworkFilter') self.addMissingFilterItems(missingArtworkXml, artworkOrGroup, 'orGroup') self.addMissingFilterItems(missingArtworkXml, artworkAndGroup, 'andGroup') if(len(infoOrGroup) > 0 or len(infoAndGroup) > 0): missingInfoXml = SubElement(missingFilterXml, 'missingInfoFilter') self.addMissingFilterItems(missingInfoXml, infoOrGroup, 'orGroup') self.addMissingFilterItems(missingInfoXml, infoAndGroup, 'andGroup') success, message = self.writeFile() return success, message def addMissingFilterItems(self, missingXml, group, groupName): if(len(group) > 0): groupXml = SubElement(missingXml, groupName) for item in group: SubElement(groupXml, 'item').text = item def searchConfigObjects(self, xPath, nameToCompare, objectType): objects = self.tree.findall(xPath) objectFound = False for obj in objects: objectName = obj.attrib.get('name') if(objectName == nameToCompare): objectFound = True break if(not objectFound): return False, util.localize(32009) %(objectType, nameToCompare) return True, '' def removeRomCollection(self, RCName): Logutil.log('removeRomCollection', util.LOG_LEVEL_INFO) configFile = util.getConfigXmlPath() self.tree = ElementTree().parse(configFile) romCollectionsXml = self.tree.find('RomCollections') for romCollectionXml in romCollectionsXml.findall('RomCollection'): name = romCollectionXml.attrib.get('name') if(name == RCName): romCollectionsXml.remove(romCollectionXml) success, message = self.writeFile() return success, message def addFileTypesForMame(self): Logutil.log('addFileTypesForMame', util.LOG_LEVEL_INFO) fileTypesXml = self.tree.find('FileTypes') #check if the MAME FileTypes already exist cabinetExists = False marqueeExists = False actionExists = False titleExists = False highestId = 0 fileTypeXml = fileTypesXml.findall('FileType') for fileType in fileTypeXml: name = fileType.attrib.get('name') if name == 'cabinet': cabinetExists = True elif name == 'marquee': marqueeExists = True elif name == 'action': actionExists = True elif name == 'title': titleExists = True id = fileType.attrib.get('id') if int(id) > highestId: highestId = int(id) if not cabinetExists: self.createFileType(fileTypesXml, str(highestId +1), 'cabinet', 'image', 'game') if not marqueeExists: self.createFileType(fileTypesXml, str(highestId +2), 'marquee', 'image', 'game') if not actionExists: self.createFileType(fileTypesXml, str(highestId +3), 'action', 'image', 'game') if not titleExists: self.createFileType(fileTypesXml, str(highestId +4), 'title', 'image', 'game') def createFileType(self, fileTypesXml, id, name, type, parent): fileType = SubElement(fileTypesXml, 'FileType', {'id' : str(id), 'name' : name}) SubElement(fileType, 'type').text = type SubElement(fileType, 'parent').text = parent def addImagePlacingForMame(self): Logutil.log('addImagePlacingForMame', util.LOG_LEVEL_INFO) imagePlacingXml = self.tree.find('ImagePlacing') #check if the MAME ImagePlacing options already exist cabinetExists = False marqueeExists = False fileTypeForXml = imagePlacingXml.findall('fileTypeFor') for fileTypeFor in fileTypeForXml: name = fileTypeFor.attrib.get('name') if name == 'gameinfomamecabinet': cabinetExists = True elif name == 'gameinfomamemarquee': marqueeExists = True if not cabinetExists: fileTypeFor = SubElement(imagePlacingXml, 'fileTypeFor', {'name' : 'gameinfomamecabinet'}) SubElement(fileTypeFor, 'fileTypeForGameList').text = 'cabinet' SubElement(fileTypeFor, 'fileTypeForGameList').text = 'boxfront' SubElement(fileTypeFor, 'fileTypeForGameList').text = 'title' SubElement(fileTypeFor, 'fileTypeForGameListSelected').text = 'cabinet' SubElement(fileTypeFor, 'fileTypeForGameListSelected').text = 'boxfront' SubElement(fileTypeFor, 'fileTypeForGameListSelected').text = 'title' SubElement(fileTypeFor, 'fileTypeForMainViewBackground').text = 'boxfront' SubElement(fileTypeFor, 'fileTypeForMainViewBackground').text = 'title' SubElement(fileTypeFor, 'fileTypeForMainViewBackground').text = 'action' SubElement(fileTypeFor, 'fileTypeForMainViewGameInfoUpperLeft').text = 'title' SubElement(fileTypeFor, 'fileTypeForMainViewGameInfoUpperRight').text = 'action' SubElement(fileTypeFor, 'fileTypeForMainViewGameInfoLower').text = 'marquee' if not marqueeExists: fileTypeFor = SubElement(imagePlacingXml, 'fileTypeFor', {'name' : 'gameinfomamemarquee'}) SubElement(fileTypeFor, 'fileTypeForGameList').text = 'marquee' SubElement(fileTypeFor, 'fileTypeForGameList').text = 'boxfront' SubElement(fileTypeFor, 'fileTypeForGameList').text = 'title' SubElement(fileTypeFor, 'fileTypeForGameListSelected').text = 'marquee' SubElement(fileTypeFor, 'fileTypeForGameListSelected').text = 'boxfront' SubElement(fileTypeFor, 'fileTypeForGameListSelected').text = 'title' SubElement(fileTypeFor, 'fileTypeForMainViewBackground').text = 'boxfront' SubElement(fileTypeFor, 'fileTypeForMainViewBackground').text = 'title' SubElement(fileTypeFor, 'fileTypeForMainViewBackground').text = 'action' SubElement(fileTypeFor, 'fileTypeForMainViewGameInfoLeft').text = 'cabinet' SubElement(fileTypeFor, 'fileTypeForMainViewGameInfoUpperRight').text = 'action' SubElement(fileTypeFor, 'fileTypeForMainViewGameInfoLowerRight').text = 'title' def writeFile(self): Logutil.log('writeFile', util.LOG_LEVEL_INFO) #write file try: configFile = util.getConfigXmlPath() util.indentXml(self.tree) treeToWrite = ElementTree(self.tree) treeToWrite.write(configFile) return True, "" except Exception, (exc): print("Error: Cannot write config.xml: " +str(exc)) return False, util.localize(32008) +": " +str(exc)
def makeContent(pages,media,p,tdir): pf=p['href'] if debugMode(): print("makeContent",pf) if pf==None: return False # strip off id references in path pp = pf.find("#") if pp>-1: f = pf[0:pp] else: f = pf # can we use extensions? extf = get_extensions() ff = os.path.normpath(tdir + os.sep + f) tree = ElementTree() tree.parse(ff) body = tree.find("body") if body==None: print("Error: no body in",f) return False # erase topic title line bh1 = body.find("h1") if bh1!=None: body.remove(bh1) # get a list of all the keywords in the file key_list = [] head = tree.find("head") if head==None: print("Error: no head in",f) return False # init field info fieldlist = {} # make a list of all <meta> elements and then find # the keywords in DC.subject. metas = head.findall("meta") for meta in metas: mname = meta.get("name") if mname!=None and mname=="DC.subject": DCcontent = meta.get("content") if DCcontent!=None: content_tags = DCcontent.split(",") for ctag in content_tags: ctag = ctag.strip() if not ctag in key_list: key_list.append(ctag) # add tag to global tag list if not ctag in get_tag_list(): add_tag(ctag) # if we can, add this tag to the vocabulary if extf and get_tagflag(): vcab = get_vocabulary() addVocabularyTerm(vcab,ctag) print(" add",vcab,"term",ctag) # look for fields provided in the input # this is a local extension elif mname!=None and mname.startswith("FIELD."): fpos = mname.find(".") fieldname = mname[fpos+1:] fieldvalue = meta.get("content") if debugMode(): print("field: ",fieldname,fieldvalue) fieldlist[fieldname] = fieldvalue # store the tags (keywords) p['keywords'] = key_list if len(fieldlist)>0: p['fields'] = fieldlist if debugMode(): print("key_list",key_list) print(get_tag_list()) # get a list of parentlinks and remove them if get_sourcetype()=="DITAXHTML": plinks = [] removeParentTopicLinks(body, plinks,1) # if not doing outline processing for DITA content, # move DITA parent links to top if not get_outline() and get_sourcetype()=="DITAXHTML": if len(plinks)>0: # move deleted parent links to the top for plink in plinks: body.insert(0,plink) elif get_cms()=="Drupal": # for Drupal, if there is no parent link, add one parent = p['parent'] if parent!=None: pp = pages[parent] if debugMode(): print("add parent",parent,pp['title']) # manufacture a parent link ltext='<div class="parentlink"><strong> Parent topic: </strong><a href="'+pp['href']+'">'+pp['title']+'</a></div>' body.insert(0,XML(ltext)) elif get_linkflag() and (get_sourcetype()=="DITAXHTML"): # also remove related task and concept links, previous and next topics divl = body.findall("div") for d in divl: c = d.get("class") if c!=None: if ("relinfo" in c) or ("familylinks" in c) or ("related-links" in c): pmap = parentMap(tree) pd = pmap[d] pd.remove(d) print(" removing div of class",c) # also remove olchildlink/ulchildlink <li> elements lil = body.findall("li") for l in lil: c = l.get("class") if c!=None: if ("olchildlink" in c) or ("ulchildlink" in c): pmap = parentMap(tree) pl = pmap[l] pl.remove(l) print(" removing li of class",c) # also remove all links from top TOC page if p['parent'] == None: flg = True while(flg): ule = body.find("ul") if ule!=None: pmap = parentMap(tree) pul = pmap[ule] pul.remove(ule) else: flg = False elif get_outline(): # when doing Drupal outline, delete TOC links from ditamaps parent_map = parentMap(tree) ullist = tree.find("body/div/ul") if ullist!=None: if ullist.get("class")=="ullinks": pnode = parent_map[ullist] pnode.remove(ullist) print(" remove page ullinks") # change the src attribute for any img elements to point to image URL images = body.iterfind("*//img") for img in images: src = img.get("src") if not isURL(src): if debugMode(): print("src",src) # get the URL where Drupal has stored the image murl = getMediaURL(media,f,src) img.set("src",murl) # delete <a> elements that have no href. they cause formatting # problems when the CMS styles the page. alist = [] remove_empty_anchors(body,1,alist) # turn the XML into a string bodystr = tostring(body) # save the edited page as a string if not isinstance(bodystr,tp_str): bodystr = bodystr.decode() p["content"] = bodystr return True
def parse_document(self, etree, systemdate): query_queue = collections.defaultdict(list) systemdate_sqlite_str = systemdate.strftime('%Y-%m-%d %H:%M:%S.%f') snapshot_utc_date = datetime.datetime.utcfromtimestamp( int(etree.find('PinnacleFeedTime').text) / 1e3 ) snapshot_sqlite_str = snapshot_utc_date.strftime('%Y-%m-%d %H:%M:%S.%f') for event_el in etree.findall('events/event'): islive = coalesce(event_el, 'IsLive', 1) eventid = event_el.find('gamenumber').text is_contest = len(event_el.findall('contest_maximum')) != 0 evdate_str = event_el.find('event_datetimeGMT').text evdate_dt = datetime.datetime.strptime(evdate_str, '%Y-%m-%d %H:%M') # For unknown reasons, pinnacle's xml lists some matches as starting on weird # times like 19:59 and 21:44. This will fix this. if evdate_dt.minute % 5 == 4: LOGGER.debug('Adding 1 minute to datetime: %s', evdate_dt) evdate_dt += datetime.timedelta(minutes = 1) def add_args(*args, **kwargs): if args[0] == 'odds': query_queue[eventid].append([ BEFORE_SQL, [ kwargs['periodnumber'], kwargs['type'], kwargs['eventid']] ]) args_to_append = cl.get_insert_args_mysql(*args, **kwargs) query_queue[eventid].append(args_to_append) if args[0] == 'odds': query_queue[eventid].append([ AFTER_SQL1, [ kwargs['periodnumber'], kwargs['type'], kwargs['eventid']] ]) query_queue[eventid].append([ AFTER_SQL2, [ kwargs['periodnumber'], kwargs['type'], kwargs['eventid']] ]) add_args( 'events', 'replace', id = eventid, date = cl.datetime_to_sqlite_str(evdate_dt), sporttype = event_el.find('sporttype').text, league = event_el.find('league').text, islive = (1 if islive == 'Yes' else 0), description = coalesce(event_el, 'description', 1), ) for participant_el in event_el.findall('participants/participant'): contestantnum = participant_el.find('contestantnum').text rotnum = participant_el.find('rotnum').text vhd = coalesce(participant_el, 'visiting_home_draw', 1) pitcher = coalesce(participant_el, 'pitcher', 1) add_args( 'participants', 'replace', eventid = eventid, contestantnum = contestantnum, rotnum = rotnum, vhdou = VHDOU_VAL[vhd], pitcher = pitcher, name = participant_el.find('participant_name').text, ) if is_contest: add_args( 'snapshots', 'replace', eventid = eventid, date = snapshot_sqlite_str, systemdate = systemdate_sqlite_str, mlmax = int(coalesce(event_el, 'contest_maximum', 1)), ) add_args( 'odds', 'replace', eventid = eventid, periodnumber = None, snapshotdate = snapshot_sqlite_str, type = 'm', threshold = 0, vhdou = None, price = dec_odds( coalesce( participant_el, 'odds/moneyline_value', 1 ) ), to_base = fix_to_base(coalesce( participant_el, 'odds/to_base', 1 )), contestantnum = contestantnum, rotnum = rotnum, ) if not is_contest: for period_el in event_el.findall('periods/period'): periodnumber = period_el.find('period_number').text add_args( 'periods', 'replace', eventid = eventid, number = periodnumber, description = period_el.find('period_description').text, cutoff = period_el.find('periodcutoff_datetimeGMT').text, ) add_args( 'snapshots', 'replace', eventid = eventid, periodnumber = periodnumber, date = snapshot_sqlite_str, systemdate = systemdate_sqlite_str, status = period_el.find('period_status').text, upd = period_el.find('period_update').text, spreadmax = int(float(period_el.find('spread_maximum').text)), mlmax = int(float(period_el.find('moneyline_maximum').text)), totalmax = int(float(period_el.find('total_maximum').text)), ) moneyline_el = coalesce(period_el, 'moneyline') if moneyline_el is not None: for vhd in ['home', 'visiting', 'draw']: if len(moneyline_el.findall('moneyline_' + vhd)) > 0: contestantnum, rotnum = period_el_to_contestantnum_rotnum(period_el, event_el, vhd) add_args( 'odds', None, eventid = eventid, periodnumber = periodnumber, snapshotdate = snapshot_sqlite_str, type = 'm', threshold = 0, vhdou = VHDOU_VAL[vhd], price = dec_odds( coalesce( moneyline_el, 'moneyline_' + vhd, 1 ) ), contestantnum = contestantnum, rotnum = rotnum, ) spread_el = coalesce(period_el, 'spread') if spread_el is not None: for vhd in ['home', 'visiting']: if len(spread_el.findall('spread_' + vhd)) > 0: contestantnum, rotnum = period_el_to_contestantnum_rotnum(period_el, event_el, vhd) add_args( 'odds', None, eventid = eventid, periodnumber = periodnumber, snapshotdate = snapshot_sqlite_str, type = 's', vhdou = VHDOU_VAL[vhd], threshold = float(spread_el.find('spread_' + vhd).text), price = dec_odds( coalesce( spread_el, 'spread_adjust_' + vhd, 1 ) ), contestantnum = contestantnum, rotnum = rotnum, ) total_el = coalesce(period_el, 'total') if total_el is not None: for ou in ['over', 'under']: contestantnum, rotnum = period_el_to_contestantnum_rotnum(period_el, event_el, ou) add_args( 'odds', None, eventid = eventid, periodnumber = periodnumber, snapshotdate = snapshot_sqlite_str, type = 't', vhdou = VHDOU_VAL[ou], threshold = float(total_el.find('total_points').text), price = dec_odds( coalesce( total_el, ou + '_adjust', 1 ) ), contestantnum = contestantnum, rotnum = rotnum, ) with get_conn_mysql() as conn: LOGGER.info('Queue holds %s queries. Starting execution...', sum(len(al) for al in query_queue.itervalues())) for argslist in query_queue.itervalues(): for args in argslist: try: if args[0] != AFTER_SQL1 and args[0] != AFTER_SQL2: conn.execute(*args) else: odds_ids = [r[0] for r in conn.execute(args[0], args[1]).fetchall()] for odds_id in odds_ids: odds_update_sql = ''' update odds set {col} = 1 where id = %s '''.format(col = 'opening' if args[0] == AFTER_SQL1 else 'latest') conn.execute(odds_update_sql, [odds_id]) except Exception as e: print 'Query that caused exception:' print args[0] print args[1] print raise conn.commit() LOGGER.info('Queue execution done.') if self.call_when_update_done is not None: self.call_when_update_done(systemdate)