Example #1
0
def map_programme(el: ElementTree, channels_dict):
    category = el.find("category")
    return Programme(
        el.find("title").text, "" if category is None else category.text,
        channels_dict[el.get("channel")],
        datetime.strptime(el.get("start"), time_format),
        datetime.strptime(el.get("stop"), time_format))
Example #2
0
    def _from_etree(cls, el: ElementTree, base_href: str) -> LessonSection:
        title_el = el.find('./h2')
        if title_el is None or not title_el.text:
            raise LessonParseError(
                'Lesson <section> needs a non-empty <h2> title')
        title = title_el.text

        steps_el = el.find('./ol[@class="steps"]')
        if steps_el is None or not steps_el:
            steps = list()
        else:
            steps = [
                LessonSectionStep._from_etree(el, base_href) for el in steps_el
            ]

        # Now get the rest of the HTML, minus the <h1> and <ol>
        el.remove(title_el)  # hacky mutation
        if steps_el is not None:
            el.remove(steps_el)  # hacky mutation
        html = _build_inner_html(el, base_href)

        # Look for "fullscreen" class on section, set fullscreen flag if so
        is_full_screen = el.find('[@class="fullscreen"]') is not None

        return cls(title, html, steps, is_full_screen)
Example #3
0
    def _read_image(self, mrb_handle: MrbHandle, mrml: ElementTree) -> Image:

        volume_node = mrml.find("./Volume")
        volume_storage_node = mrml.find(
            "./VolumeArchetypeStorage[@id='{}']".format(
                volume_node.attrib["storageNodeRef"]))

        nrrd_data, nrrd_header = mrb_handle.read_nrrd(
            urllib.parse.unquote(volume_storage_node.attrib["fileName"]))

        assert nrrd_header["space"] == "left-posterior-superior",\
            "Unsupported nrrd image volume space: " + nrrd_header["space"]
        assert [np.sign(x) for x in self._get_space_directions(nrrd_header["space directions"])] == [1., 1., 1.],\
            "Unsupported nrrd image volume directions."

        # type-cast
        nrrd_data = nrrd_data.astype(np.int32)

        # switch sagittal-coronal-axial axis-order to axial-coronal-sagittal
        nrrd_data = np.swapaxes(nrrd_data, 0, 2)

        # posterior towards anterior
        nrrd_data = np.flip(nrrd_data, 1)

        image = Image(image_data=nrrd_data,
                      identifier=self._derive_identifier(
                          volume_node.attrib["name"]),
                      voxel_spacing=self._get_voxel_spacing(
                          nrrd_header["space directions"]))

        return image
Example #4
0
 def _parse_gateway(gateway_child: ElementTree) -> Gateway:
     id = gateway_child.get('id')
     name = gateway_child.find('Name').text
     type = gateway_child.find('Type').text.lower()
     gates = []
     distribution = None
     rule = None
     if type == GATEWAY_TYPES['choice']:
         distribution_child = gateway_child.find('Distribution')
         rule_child = gateway_child.find('Rule')
         if distribution_child is not None and len(
                 list(distribution_child)):
             distribution = []
             for gate in distribution_child:
                 gates.append(gate.get('id'))
                 distribution.append(float(gate.text))
         elif rule_child is not None and len(list(rule_child)):
             for gate in rule_child:
                 gates.append(gate.get('id'))
             rule = id
         else:
             raise ValueError(
                 "For choice gateways, either rule or distribution must be present."
             )
     elif type == GATEWAY_TYPES['parallel'] or type == GATEWAY_TYPES[
             'merge']:
         gates_child = gateway_child.find('Gates')
         for gate in gates_child:
             gates.append(gate.get('id'))
     return Gateway(id=id,
                    name=name,
                    type=type,
                    gates=gates,
                    distribution=distribution,
                    rule=rule)
Example #5
0
    def from_xml(xml_tree : ElementTree):
        path = xml_tree.find('path').text
        description = xml_tree.find('description').text

        info = xml_tree.find('info')
        timestamp = info.get('time-stamp')
        cid = int(info.get('camera-id'))

        return Image(path, timestamp, cid, description)
Example #6
0
 def __init__(self, xml_str):
     if not xml_str:
         raise Exception("Empty string")
     xml_data = ET.fromstring(xml_str)
     self.to_user = ET.find('ToUserName').text
     self.from_user = ET.find('FromUserName').text
     self.content = ET.find('Content').text
     self.msg_id = ET.find('MsgId').text
     self.msg_type = ET.find('MsgType').text
Example #7
0
def get_asd_reference_tuple(asd: ET):
    thing_id: ET = asd.find("ThingReferenceId")
    network_id: ET = asd.find("NetworkReferenceId")
    thing_key = thing_id.text
    network_key = network_id.text
    position_key = None
    if thing_key == '0' and network_key == '0':
        position: ET = asd.find('Position')
        position_key = vector_to_sortable_string(position)
    return thing_key, network_key, position_key, asd
Example #8
0
    def configure(self, config_tree: ElementTree):
        userinfo = config_tree.find('credentials')

        self.mail_user = userinfo.find('username').get('value')
        self.mail_password = userinfo.find('password').get('value')

        gateway = config_tree.find('gateway')

        self.gateway_address = gateway.get('address')
        self.gateway_port = gateway.get('port')
Example #9
0
def parse_member(member: ElementTree, tree: ElementTree) -> CMember:
    e_type = member.find('./type')
    type_string = (member.text or '') + e_type.text + (e_type.tail or '')

    e_name = member.find('./name')
    name = e_name.text

    array_size: int = None
    if e_name.tail and e_name.tail.startswith('['):
        match = re.match('\[\s*(\d+)\s*\]', e_name.tail)
        if match:
            array_size = int(match.group(1))
        else:
            e_enum = member.find('./enum')
            array_size = int(
                tree.find(
                    f'./enums/enum[@name="{e_enum.text}"]').attrib['value'])

    type_strings = type_string.split('*')

    first_type = type_strings[0].split()
    is_const = False
    if 'const' in first_type:
        first_type.remove('const')
        is_const = True
    if 'struct' in first_type:
        first_type.remove('struct')
    c_type = CType(name=' '.join(first_type), const=is_const)

    pointers = type_strings[1:]
    lengths = member.attrib['len'].split(',') if 'len' in member.attrib else []
    optionals = member.attrib['optional'].split(
        ',') if 'optional' in member.attrib else []

    if len(optionals) > len(pointers):
        c_type.optional = optionals.pop(-1) == 'true'

    lengths = reversed(lengths[:len(pointers)])
    optionals = reversed(optionals[:len(pointers)])

    for pointer, length, optional in zip_longest(pointers, lengths, optionals):
        c_type = CType(pointer_to=c_type,
                       length=length,
                       const='const' in pointer,
                       optional=optional == 'true')

    if array_size is not None:
        c_type = CType(array_of=c_type, length=array_size)

    values_string = member.get('values')
    values = values_string.split(',') if values_string else []

    return CMember(name, c_type, values)
Example #10
0
    def _from_etree(cls, el: ElementTree, base_href: str) -> LessonFooter:
        title_el = el.find('./h2')
        if title_el is None or not title_el.text:
            raise LessonParseError(
                'Lesson <footer> needs a non-empty <h2> title')
        title = title_el.text
        is_full_screen = el.find('[@class="fullscreen"]') is not None

        # Now get the rest of the HTML, minus the <h2>
        el.remove(title_el)  # hacky mutation
        html = _build_inner_html(el, base_href)

        return cls(title, html, is_full_screen)
 def _prep_xmltext(et: Et, key: str) -> str:
     """Wrapper to quickly get settings.
     
     :param et: Element Tree from parsed Xml
     :type et: xml.etree.ElementTree
     :param key: Name of the setting
     :type key: str
     :return: Parsed value for setting
     :rtype: str
     """
     t = et.find(key).text.strip() if et.findtext(key) and (
         et.find(key).text is not None) else None
     return None if not t or t == "" else t
Example #12
0
def get_date(article: ElementTree) -> int:
    """
    Extracts the year of the article. If ArticleDate exist use its year
    otherwise use the year form JournalIssue
    """
    d = article.find("ArticleDate")
    if d is not None:
        return int(d.find("Year").text)
    d = article.find("Journal").find("JournalIssue").find("PubDate")
    y = d.find("Year")
    if y is not None:
        return int(y.text)
    return int(d.find("MedlineDate").text.split(" ")[0].split("-")[0])
Example #13
0
 def __init__(self):
     self.unitTechs = []
     self.buildingTechs = []
     self.mothershipTechs = []
     tree = ElementTree(file="TechTree.xml")
     units = tree.find("Units")
     for u in units.findall("Tech"):
         self.unitTechs.append(Tech(self, u))
     buildings = tree.find("Buildings")
     for b in buildings.findall("Tech"):
         self.buildingTechs.append(Tech(self, b))
     motherships = tree.find("Mothership")
     for m in motherships.findall("Tech"):
         self.mothershipTechs.append(Tech(self, m))
Example #14
0
    def __init__(self, currency: ElementTree, date):
        self.date = date

        self.code = currency.get('Kod')

        temp = currency.find('Unit').text
        self.unit = int(temp) if temp else None

        self.name = currency.find('Isim').text

        self.currency_name = currency.find('CurrencyName').text

        temp = currency.find('ForexBuying').text
        self.forex_buying = float(temp) if temp else None

        temp = currency.find('ForexSelling').text
        self.forex_selling = float(temp) if temp else None

        temp = currency.find('BanknoteBuying').text
        self.banknote_buying = float(temp) if temp else None

        temp = currency.find('BanknoteSelling').text
        self.banknote_selling = float(temp) if temp else None

        cross_rate_usd = currency.find('CrossRateUSD').text
        self.cross_rate_usd = float(cross_rate_usd) if cross_rate_usd else None
Example #15
0
def _common_check_response(xml_response: Tree, method_name: str, statistic: Statistic) -> Tree:
    logger = statistic.get_log().get_logger("scripts/xml/zmq")
    logger.info("was called (xml_response)")
    logger.debug("(" + Tree.tostring(xml_response).decode() + ")")
    if xml_response == -1:
        statistic.append_error(method_name + " завершился с ошибкой!", "ZMQ_ОТВЕТ")
        return -1

    status = int(xml_response.find(".//status").text)
    if status != 0:
        status_descr = xml_response.find(".//statusDescr").text
        statistic.append_error("Статус ответа: " + str(status_descr), "ZMQ_ОТВЕТ")
        return -1

    return xml_response
Example #16
0
    def _get_segmentation_volume(
            self, mrb_handle: MrbHandle, mrml: ElementTree,
            segmentation_node: ElementTree) -> Tuple[np.ndarray, dict]:

        segmentation_storage_node = mrml.find(
            "./SegmentationStorage[@id='{}']".format(
                segmentation_node.attrib["storageNodeRef"]))

        nrrd_data, nrrd_header = mrb_handle.read_nrrd(
            segmentation_storage_node.attrib["fileName"])

        assert len(
            np.shape(nrrd_data)
        ) == 4, "Unexpected segmentation mask volume shape: {}".format(
            np.shape(nrrd_data))
        assert nrrd_header[
            "space"] == "right-anterior-superior", "Unsupported segmentation mask nrrd space: " + nrrd_header[
                "space"]
        assert [
            np.sign(x) for x in self._get_space_directions(
                nrrd_header["space directions"][1:])
        ] == [1., -1., -1.]

        # switch sagittal-coronal-axial axis-order to axial-coronal-sagittal
        nrrd_data = np.swapaxes(nrrd_data, 1, 3)

        # posterior towards anterior
        nrrd_data = np.flip(nrrd_data, 2)

        return nrrd_data, nrrd_header
Example #17
0
    def _cfg_set_or_create(root: ElementTree, match: str,
                           new_value: str) -> None:
        ele = root.find(match)
        if ele is None:
            ele = ElementTree.SubElement(root, match)

        ele.text = new_value
Example #18
0
    def parse_bitmasks(self, tree: ElementTree):
        for e_bitmask in tree.findall('./types/type[@category="bitmask"]'):
            if 'alias' in e_bitmask.attrib:
                continue

            bitmask_name = e_bitmask.find('./name').text

            if self.should_ignore(type_=bitmask_name):
                continue

            c_bitmask = CBitmask(bitmask_name)

            requires = e_bitmask.get('requires')
            if requires:
                cases = {}
                e_enum = tree.find(f'./enums[@name="{requires}"]')
                for e_case in e_enum.findall('./enum'):
                    if 'alias' in e_case.attrib:
                        continue
                    cases[e_case.attrib['name']] = parse_enum_value(e_case)

                for extension in self.extensions:
                    if extension.supported == 'disabled' or extension.platform:
                        continue
                    for ext_enum in extension.enums:
                        if ext_enum.name == requires:
                            for ext_case in ext_enum.cases:
                                cases[ext_case.name] = ext_case.value

                c_bitmask.enum = CEnum(
                    requires,
                    [CEnum.Case(name, value) for name, value in cases.items()])

            self.bitmasks.append(c_bitmask)
Example #19
0
 def set_shape_text(shape: ET, text: str):
     t = shape.find(f"{namespace}Text")  # type: Element
     if t is not None:
         if t.text:
             t.text = text
         else:
             t[0].tail = text
Example #20
0
 def get_shape_text(shape: ET) -> str:
     # technically the below is not an exact replacement of the above...
     text = ""
     text_elem = shape.find(f"{namespace}Text")
     if text_elem is not None:
         text = "".join(text_elem.itertext())
     return text
Example #21
0
    def copy_shape(self, shape: Element, page: ET, page_path: str) -> ET:
        """Insert shape into first Shapes tag in destination page, and return the copy.

        If destination page does not have a Shapes tag yet, create it.

        Parameters:
            shape (Element): The source shape to be copied. Use Shape.xml
            page (ElementTree): The page where the new Shape will be placed. Use Page.xml
            page_path (str): The filename of the page where the new Shape will be placed. Use Page.filename

        Returns:
            ElementTree: The new shape ElementTree

        """

        new_shape = ET.fromstring(ET.tostring(shape))

        self.set_page_max_id(page_path)

        # find or create Shapes tag
        shapes_tag = page.find(f"{namespace}Shapes")
        if shapes_tag is None:
            shapes_tag = Element(f"{namespace}Shapes")
            page.getroot().append(shapes_tag)

        id_map = self.increment_shape_ids(new_shape, page_path)
        self.update_ids(new_shape, id_map)
        shapes_tag.append(new_shape)

        self.pages[page_path] = page
        return new_shape
Example #22
0
def vector_to_sortable_string(vector:ET):
    result = ''
    for v in vector_components:
        text = vector.find(v).text
        sortable_string = number_string_to_sortable_string(text)
        result += sortable_string
    return result
Example #23
0
def parse_image(tree: ElementTree):
    """Parses XML"""
    image_url = f"https://bing.com{tree.find('url').text}"
    image_base = f"https://bing.com{tree.find('urlBase').text}"
    image_date_raw = tree.find('startdate').text
    image_date = f"{image_date_raw[:4]}-{image_date_raw[4:6]}-{image_date_raw[6:]}"
    headline = tree.find('headline').text
    _copyright = tree.find('copyright').text

    return {
        'image': image_url,
        'image_base': image_base,
        'title': headline,
        'date': image_date,
        'copyright': _copyright,
    }
Example #24
0
def setup_streets(net: sumolib.net.Net, xml: ElementTree, pop_noise: NoiseSampler, work_noise: NoiseSampler):
    """
    Create a street for each edge in the network and calculate its population and workplaces based on
    modified Perlin noise from NoiseSamplers
    :param net: the SUMO network
    :param xml: the statistics XML for the network
    :param pop_noise: NoiseSampler to use for population
    :param work_noise: NoiseSample to use for workplaces
    """

    streets = xml.find("streets")
    if streets is None:
        streets = ET.SubElement(xml.getroot(), "streets")

    # Some edges might already have a street, so we want to ignore those
    known_streets = {street.attrib["edge"]: street for street in streets.findall("street")}

    for edge in net.getEdges():
        eid = edge.getID()
        if eid not in known_streets:
            # This edge is missing a street entry. Find population and industry for this edge
            pos = get_edge_pair_centroid(edge.getShape())
            population = pop_noise.sample(pos)
            industry = work_noise.sample(pos)

            logging.debug(
                f"[perlin] Adding street with eid: {eid},\t population: {population:.4f}, industry: {industry:.4f}")
            ET.SubElement(streets, "street", {
                "edge": eid,
                "population": str(population),
                "workPosition": str(industry)
            })
 def getPocketPos(self, sWhichPocket, sWhichPosition):
     #print("%s-%s" % (sWhichPocket, sWhichPosition))
     oXML = self.oXMLroot.find('Pockets/Pocket[@id="%s"]' % sWhichPocket)
     oXML2 = oXML.find('%s' % sWhichPosition)
     posX = oXML2.get("x")
     posY = oXML2.get("y")
     posZ = oXML2.get("z")
     return float(posX), float(posY), float(posZ)
Example #26
0
 def _cfg_load_stashes(root: ElementTree) -> list:
     stashes = root.find('stashes')
     if stashes is None:
         return [{'name': "", 'type': "normal"}]
     return [{
         'name': x.text,
         'type': x.attrib['type']
     } for x in list(stashes)]
Example #27
0
    def _parse_transition(self, transition_child: ElementTree) -> Transition:
        source = transition_child.get('source')
        destination = transition_child.get('destination')
        sgate = transition_child.get('source_gate')
        dgate = transition_child.get('destination_gate')
        duration_child = transition_child.find('Duration')
        distribution_child = transition_child.find('Duration/Distribution')
        if distribution_child is not None:
            distribution = self._parse_distribution(distribution_child)
        else:
            distribution = int(
                duration_child.text) if duration_child is not None else 0

        return Transition(source=source,
                          destination=destination,
                          sgate=sgate,
                          dgate=dgate,
                          distribution=distribution)
Example #28
0
def getSVO(sentence):
    """主語、述語、目的語を抽出"""
    dependencies = sentence.find(
        "dependencies[@type='collapsed-dependencies']")

    nsubj = dependencies.findall("dep[@type='nsubj']")
    dobj = dependencies.findall("dep[@type='dobj']")

    gove = lambda x: x.find("governor").text
    depa = lambda x: x.find("dependent").text

    answer = []

    for ns in nsubj:
        for db in dobj:
            if gove(ns) == gove(db):
                answer.append((depa(ns), gove(ns), depa(db)))
    return answer
Example #29
0
 def set_game_info(self, info):
     self.info = info
     tree = ElementTree()
     tree.parse(info["path"])
     link_node = tree.find("link")
     if link_node is not None:
         self.link_field.set_text(link_node.text.strip())
     else:
         self.link_field.set_text("")
Example #30
0
 def from_etree(cls, obj: ElementTree) -> 'ObjectDef':
     return cls(name=_node_text(obj.find('Name')),
                description=_node_text(obj.find('Description1')),
                oid=int(_node_text(obj.find('ObjectID'))),
                urn=_node_text(obj.find('ObjectURN')),
                multiple={'Single': False, 'Multiple': True}[_node_text(obj.find('MultipleInstances'))],
                mandatory={'Optional': False, 'Mandatory': True}[_node_text(obj.find('Mandatory'))],
                resources=sorted([ResourceDef.from_etree(item) for item in obj.find('Resources').findall('Item')],
                                 key=operator.attrgetter('rid')))
def from_ET(et:ET) -> ThingSaveData:
    result_data = dict()

    for item in ThingSaveData._fields:
        value = et.find("./"+item)
        result_data[item] = value.text.strip() if value is not None and value.text is not None else None
    result_data['parsed_element'] = et
    result_data['xsi_type'] = et.get('{http://www.w3.org/2001/XMLSchema-instance}type')

    result = ThingSaveData(**result_data)
    return result
Example #32
0
    def _from_etree(cls, el: ElementTree, base_href: str) -> LessonHeader:
        title_el = el.find('./h1')
        if title_el is None or not title_el.text:
            raise LessonParseError(
                'Lesson <header> needs a non-empty <h1> title')
        title = title_el.text

        # Now get the rest of the HTML, minus the <h1>
        el.remove(title_el)  # hacky mutation
        html = _build_inner_html(el, base_href)

        return cls(title, html)
def setChannel(androidManifestFilePath,channel):
    '''
    修改AndroidManifest.xml中的UMENG_CHANNEL
    '''
    if not os.path.exists(androidManifestFilePath):
        sys.exit(androidManifestFilePath+' 不存在')

    xml.etree.ElementTree.register_namespace("android","http://schemas.android.com/apk/res/android")
    tree = ElementTree()   
    xmlRoot=tree.parse(androidManifestFilePath)
    channelNode=tree.find("./application/meta-data[@{http://schemas.android.com/apk/res/android}name='UMENG_CHANNEL']")
    channelNode.set("{http://schemas.android.com/apk/res/android}value",channel)
    tree.write(androidManifestFilePath, "UTF-8", True)
Example #34
0
def parse_user_details(s):
    """Takes a string XML representation of a user's details and
    returns a dictionary of values we care about"""
    root = ET.find('./user')
    if not root:
        print 'aaaargh'
        return None
    user = {
        'id': root.attrib['id'],
        'username': root.attrib['display_name']
    }
    try:
        user['lat'] = float(root.find('./home').attrib['lat'])
        user['lon'] = float(root.find('./home').attrib['lon'])
    except AttributeError:
        pass
    user['changesets'] = int(root.find('./changesets').attrib['count'])
    return user
class ConfigXmlWriter(RcbXmlReaderWriter):

	def __init__(self, createNew):

		Logutil.log('init ConfigXmlWriter', util.LOG_LEVEL_INFO)

		self.createNew = createNew

		if(createNew):
			configFile = os.path.join(util.getAddonInstallPath(), 'resources', 'database', 'config_template.xml')
		else:
			configFile = util.getConfigXmlPath()

		if(not os.path.isfile(configFile)):
			Logutil.log('File config.xml does not exist. Place a valid config file here: ' + str(configFile), util.LOG_LEVEL_ERROR)
		else:
			self.tree = ElementTree().parse(configFile)

	""" Functions for generating XML objects from objects """
	def getXmlAttributesForRomCollection(self, rc):
		return {'id': str(rc.id), 'name': rc.name}

	def getXmlElementsForRomCollection(self, rc):
		elements = []
		# String attributes
		for e in ['gameclient', 'emulatorCmd', 'emulatorParams', 'preCmd', 'postCmd',
				  'saveStatePath', 'saveStateParams']:
			try:
				el = Element(e, {})
				el.text = getattr(rc, e)
				elements.append(el)
			except AttributeError as exc:
				# Skip any errors
				pass

		# Non-string attributes
		for e in ['useBuiltinEmulator', 'useEmuSolo', 'usePopen', 'ignoreOnScan', 'allowUpdate', 'autoplayVideoMain',
				  'autoplayVideoInfo', 'useFoldernameAsGamename', 'maxFolderDepth', 'doNotExtractZipFiles',
				  'makeLocalCopy', 'diskPrefix']:
			try:
				el = Element(e, {})
				el.text = str(getattr(rc, e))
				elements.append(el)
			except AttributeError as exc:
				# Skip any errors
				pass

		for romPath in rc.romPaths:
			el = Element('romPath', {})
			el.text = romPath
			elements.append(el)

		return elements

	def getXmlAttributesForSite(self, site):
		attrs = {'name': site.name, 'path': site.path}
		return attrs

	def getXmlElementsForSite(self, site):
		""" Not needed """
		pass

	def writeRomCollections(self, romCollections, isEdit):

		Logutil.log('write Rom Collections', util.LOG_LEVEL_INFO)

		romCollectionsXml = self.tree.find('RomCollections')

		#HACK: remove all Rom Collections and create new
		if(isEdit):
			for romCollectionXml in romCollectionsXml.findall('RomCollection'):
				romCollectionsXml.remove(romCollectionXml)


		for romCollection in romCollections.values():

			Logutil.log('write Rom Collection: ' + str(romCollection.name), util.LOG_LEVEL_INFO)

			romCollectionXml = SubElement(romCollectionsXml, 'RomCollection', self.getXmlAttributesForRomCollection(romCollection))

			for subel in self.getXmlElementsForRomCollection(romCollection):
				romCollectionXml.append(subel)


			for mediaPath in romCollection.mediaPaths:

				success, message = self.searchConfigObjects('FileTypes/FileType', mediaPath.fileType.name, 'FileType')
				if(not success):
					return False, message

				SubElement(romCollectionXml, 'mediaPath', {'type': mediaPath.fileType.name}).text = mediaPath.path

			#image placing
			if(not self.createNew):
				#in case of an update we have to create new options
				if(romCollection.name == 'MAME' and not self.createNew):
					self.addFileTypesForMame()
					self.addImagePlacingForMame()

			if(romCollection.imagePlacingMain != None and romCollection.imagePlacingMain.name != ''):
				success, message = self.searchConfigObjects('ImagePlacing/fileTypeFor', romCollection.imagePlacingMain.name, 'ImagePlacing')
				if(not success):
					return False, message
				SubElement(romCollectionXml, 'imagePlacingMain').text = romCollection.imagePlacingMain.name
			else:
				SubElement(romCollectionXml, 'imagePlacingMain').text = 'gameinfobig'

			if(romCollection.imagePlacingInfo != None and romCollection.imagePlacingInfo.name != ''):
				success, message = self.searchConfigObjects('ImagePlacing/fileTypeFor', romCollection.imagePlacingInfo.name, 'ImagePlacing')
				if(not success):
					return False, message
				SubElement(romCollectionXml, 'imagePlacingInfo').text = romCollection.imagePlacingInfo.name
			else:
				SubElement(romCollectionXml, 'imagePlacingInfo').text = 'gameinfosmall'

			if (romCollection.scraperSites == None or len(romCollection.scraperSites) == 0):
				#use thegamesdb.net as default scraper in online scraping scenario
				SubElement(romCollectionXml, 'scraper', {'name': 'thegamesdb.net'})
			else:
				for scraperSite in romCollection.scraperSites:
					if(scraperSite == None):
						continue
					SubElement(romCollectionXml, 'scraper', {'name' : scraperSite.name, 'path' : scraperSite.path})

		success, message = self.writeFile()
		return success, message


	def writeMissingFilter(self, showHideOption, artworkOrGroup, artworkAndGroup, infoOrGroup, infoAndGroup):

		Logutil.log('write Missing Info Filter', util.LOG_LEVEL_INFO)

		missingFilterXml = self.tree.find('MissingFilter')

		#HACK: remove MissingFilter-element
		if(missingFilterXml != None):
			self.tree.remove(missingFilterXml)

		missingFilterXml = SubElement(self.tree, 'MissingFilter')
		SubElement(missingFilterXml, 'showHideOption').text = showHideOption

		if(len(artworkOrGroup) > 0 or len(artworkAndGroup) > 0):
			missingArtworkXml = SubElement(missingFilterXml, 'missingArtworkFilter')
			self.addMissingFilterItems(missingArtworkXml, artworkOrGroup, 'orGroup')
			self.addMissingFilterItems(missingArtworkXml, artworkAndGroup, 'andGroup')
		if(len(infoOrGroup) > 0 or len(infoAndGroup) > 0):
			missingInfoXml = SubElement(missingFilterXml, 'missingInfoFilter')
			self.addMissingFilterItems(missingInfoXml, infoOrGroup, 'orGroup')
			self.addMissingFilterItems(missingInfoXml, infoAndGroup, 'andGroup')

		success, message = self.writeFile()
		return success, message


	def addMissingFilterItems(self, missingXml, group, groupName):
		if(len(group) > 0):
			groupXml = SubElement(missingXml, groupName)
			for item in group:
				SubElement(groupXml, 'item').text = item


	def searchConfigObjects(self, xPath, nameToCompare, objectType):
		objects = self.tree.findall(xPath)
		objectFound = False
		for obj in objects:
			objectName = obj.attrib.get('name')
			if(objectName == nameToCompare):
				objectFound = True
				break

		if(not objectFound):
			return False, util.localize(32009) % (objectType, nameToCompare)

		return True, ''


	def removeRomCollection(self, RCName):

		Logutil.log('removeRomCollection', util.LOG_LEVEL_INFO)

		configFile = util.getConfigXmlPath()
		self.tree = ElementTree().parse(configFile)
		romCollectionsXml = self.tree.find('RomCollections')
		for romCollectionXml in romCollectionsXml.findall('RomCollection'):
			name = romCollectionXml.attrib.get('name')
			if(name == RCName):
				romCollectionsXml.remove(romCollectionXml)

		success, message = self.writeFile()
		return success, message

	def addFileTypesForMame(self):
		Logutil.log('addFileTypesForMame', util.LOG_LEVEL_INFO)

		fileTypesXml = self.tree.find('FileTypes')

		#check if the MAME FileTypes already exist
		cabinetExists = False
		marqueeExists = False
		actionExists = False
		titleExists = False
		highestId = 0
		fileTypeXml = fileTypesXml.findall('FileType')
		for fileType in fileTypeXml:
			name = fileType.attrib.get('name')
			if name == 'cabinet':
				cabinetExists = True
			elif name == 'marquee':
				marqueeExists = True
			elif name == 'action':
				actionExists = True
			elif name == 'title':
				titleExists = True

			id = fileType.attrib.get('id')
			if int(id) > highestId:
				highestId = int(id)

		if not cabinetExists:
			self.createFileType(fileTypesXml, str(highestId + 1), 'cabinet', 'image', 'game')
		if not marqueeExists:
			self.createFileType(fileTypesXml, str(highestId + 2), 'marquee', 'image', 'game')
		if not actionExists:
			self.createFileType(fileTypesXml, str(highestId + 3), 'action', 'image', 'game')
		if not titleExists:
			self.createFileType(fileTypesXml, str(highestId + 4), 'title', 'image', 'game')


	def createFileType(self, fileTypesXml, id, name, type, parent):
		fileType = SubElement(fileTypesXml, 'FileType', {'id' : str(id), 'name' : name})
		SubElement(fileType, 'type').text = type
		SubElement(fileType, 'parent').text = parent


	def addImagePlacingForMame(self):
		Logutil.log('addImagePlacingForMame', util.LOG_LEVEL_INFO)

		imagePlacingXml = self.tree.find('ImagePlacing')

		#check if the MAME ImagePlacing options already exist
		cabinetExists = False
		marqueeExists = False
		fileTypeForXml = imagePlacingXml.findall('fileTypeFor')
		for fileTypeFor in fileTypeForXml:
			name = fileTypeFor.attrib.get('name')
			if name == 'gameinfomamecabinet':
				cabinetExists = True
			elif name == 'gameinfomamemarquee':
				marqueeExists = True

		if not cabinetExists:
			fileTypeFor = SubElement(imagePlacingXml, 'fileTypeFor', {'name' : 'gameinfomamecabinet'})
			for imgtype in ['cabinet', 'boxfront', 'title']:
				SubElement(fileTypeFor, 'fileTypeForGameList').text = imgtype
				SubElement(fileTypeFor, 'fileTypeForGameListSelected').text = imgtype

			for imgtype in ['boxfront', 'title', 'action']:
				SubElement(fileTypeFor, 'fileTypeForMainViewBackground').text = imgtype

			SubElement(fileTypeFor, 'fileTypeForMainViewGameInfoUpperLeft').text = 'title'
			SubElement(fileTypeFor, 'fileTypeForMainViewGameInfoUpperRight').text = 'action'
			SubElement(fileTypeFor, 'fileTypeForMainViewGameInfoLower').text = 'marquee'

		if not marqueeExists:
			fileTypeFor = SubElement(imagePlacingXml, 'fileTypeFor', {'name' : 'gameinfomamemarquee'})
			for imgtype in ['marquee', 'boxfront', 'title']:
				SubElement(fileTypeFor, 'fileTypeForGameList').text = imgtype
				SubElement(fileTypeFor, 'fileTypeForGameListSelected').text = imgtype

			for imgtype in ['boxfront', 'title', 'action']:
				SubElement(fileTypeFor, 'fileTypeForMainViewBackground').text = imgtype

			SubElement(fileTypeFor, 'fileTypeForMainViewGameInfoLeft').text = 'cabinet'
			SubElement(fileTypeFor, 'fileTypeForMainViewGameInfoUpperRight').text = 'action'
			SubElement(fileTypeFor, 'fileTypeForMainViewGameInfoLowerRight').text = 'title'

	# FIXME TODO This function is only called within this class - raise exception rather than return tuple
	def writeFile(self):
		Logutil.log('writeFile', util.LOG_LEVEL_INFO)
		#write file
		try:
			configFile = util.getConfigXmlPath()

			self.indentXml(self.tree)
			treeToWrite = ElementTree(self.tree)
			treeToWrite.write(configFile)

			return True, ""

		except Exception, (exc):
			print("Error: Cannot write config.xml: " + str(exc))
			return False, util.localize(32008) + ": " + str(exc)
class ConfigXmlWriter:
	
	def __init__(self, createNew):
		
		Logutil.log('init ConfigXmlWriter', util.LOG_LEVEL_INFO)
		
		self.createNew = createNew
		
		if(createNew):
			configFile = os.path.join(util.getAddonInstallPath(), 'resources', 'database', 'config_template.xml')
		else:
			configFile = util.getConfigXmlPath()
		
		if(not os.path.isfile(configFile)):
			Logutil.log('File config.xml does not exist. Place a valid config file here: ' +str(configFile), util.LOG_LEVEL_ERROR)
			return False, util.localize(32003)
		
		self.tree = ElementTree().parse(configFile)
	
	
	def writeRomCollections(self, romCollections, isEdit):
				
		Logutil.log('write Rom Collections', util.LOG_LEVEL_INFO)
				
		romCollectionsXml = self.tree.find('RomCollections')
		
		#HACK: remove all Rom Collections and create new
		if(isEdit):
			for romCollectionXml in romCollectionsXml.findall('RomCollection'):				
				romCollectionsXml.remove(romCollectionXml)
				
		
		for romCollection in romCollections.values():
			
			Logutil.log('write Rom Collection: ' +str(romCollection.name), util.LOG_LEVEL_INFO)
			
			romCollectionXml = SubElement(romCollectionsXml, 'RomCollection', {'id' : str(romCollection.id), 'name' : romCollection.name})
			SubElement(romCollectionXml, 'useBuiltinEmulator').text = str(romCollection.useBuiltinEmulator)
			SubElement(romCollectionXml, 'gameclient').text = romCollection.gameclient
			SubElement(romCollectionXml, 'emulatorCmd').text = romCollection.emulatorCmd
			SubElement(romCollectionXml, 'emulatorParams').text = romCollection.emulatorParams
			
			for romPath in romCollection.romPaths:
				SubElement(romCollectionXml, 'romPath').text = romPath
							
			SubElement(romCollectionXml, 'saveStatePath').text = romCollection.saveStatePath
			SubElement(romCollectionXml, 'saveStateParams').text = romCollection.saveStateParams
				
			for mediaPath in romCollection.mediaPaths:
				
				success, message = self.searchConfigObjects('FileTypes/FileType', mediaPath.fileType.name, 'FileType')
				if(not success):
					return False, message								
												
				SubElement(romCollectionXml, 'mediaPath', {'type' : mediaPath.fileType.name}).text = mediaPath.path
				
			SubElement(romCollectionXml, 'preCmd').text = romCollection.preCmd
			SubElement(romCollectionXml, 'postCmd').text = romCollection.postCmd
			SubElement(romCollectionXml, 'useEmuSolo').text = str(romCollection.useEmuSolo)
			SubElement(romCollectionXml, 'usePopen').text = str(romCollection.usePopen)
			SubElement(romCollectionXml, 'ignoreOnScan').text = str(romCollection.ignoreOnScan)
			SubElement(romCollectionXml, 'allowUpdate').text = str(romCollection.allowUpdate)
			SubElement(romCollectionXml, 'autoplayVideoMain').text = str(romCollection.autoplayVideoMain)
			SubElement(romCollectionXml, 'autoplayVideoInfo').text = str(romCollection.autoplayVideoInfo)
			SubElement(romCollectionXml, 'useFoldernameAsGamename').text = str(romCollection.useFoldernameAsGamename)
			SubElement(romCollectionXml, 'maxFolderDepth').text = str(romCollection.maxFolderDepth)
			SubElement(romCollectionXml, 'doNotExtractZipFiles').text = str(romCollection.doNotExtractZipFiles)
			SubElement(romCollectionXml, 'diskPrefix').text = str(romCollection.diskPrefix)
			
			if (os.environ.get( "OS", "xbox" ) == "xbox"):
				SubElement(romCollectionXml, 'xboxCreateShortcut').text = str(romCollection.xboxCreateShortcut)
				SubElement(romCollectionXml, 'xboxCreateShortcutAddRomfile').text = str(romCollection.xboxCreateShortcutAddRomfile)
				SubElement(romCollectionXml, 'xboxCreateShortcutUseShortGamename').text = str(romCollection.xboxCreateShortcutUseShortGamename)
				
			#image placing
			if(not self.createNew):
				#in case of an update we have to create new options
				if(romCollection.name == 'MAME' and not self.createNew):
					self.addFileTypesForMame()
					self.addImagePlacingForMame()
					
			if(romCollection.imagePlacingMain != None and romCollection.imagePlacingMain.name != ''):
				success, message = self.searchConfigObjects('ImagePlacing/fileTypeFor', romCollection.imagePlacingMain.name, 'ImagePlacing')
				if(not success):
					return False, message
				SubElement(romCollectionXml, 'imagePlacingMain').text = romCollection.imagePlacingMain.name 
			else:
				SubElement(romCollectionXml, 'imagePlacingMain').text = 'gameinfobig'
				
			if(romCollection.imagePlacingInfo != None and romCollection.imagePlacingInfo.name != ''):
				success, message = self.searchConfigObjects('ImagePlacing/fileTypeFor', romCollection.imagePlacingInfo.name, 'ImagePlacing')
				if(not success):
					return False, message
				SubElement(romCollectionXml, 'imagePlacingInfo').text = romCollection.imagePlacingInfo.name 
			else:
				SubElement(romCollectionXml, 'imagePlacingInfo').text = 'gameinfosmall'
			
			if(romCollection.scraperSites == None or len(romCollection.scraperSites) == 0):
				SubElement(romCollectionXml, 'scraper', {'name' : 'thegamesdb.net', 'replaceKeyString' : '', 'replaceValueString' : ''})
				SubElement(romCollectionXml, 'scraper', {'name' : 'archive.vg', 'replaceKeyString' : '', 'replaceValueString' : ''})
				SubElement(romCollectionXml, 'scraper', {'name' : 'mobygames.com', 'replaceKeyString' : '', 'replaceValueString' : ''})
			else:
				for scraperSite in romCollection.scraperSites:
				
					if(scraperSite == None):
						continue
						
					#HACK: use replaceKey and -Value only from first scraper
					firstScraper = scraperSite.scrapers[0]
					SubElement(romCollectionXml, 'scraper', {'name' : scraperSite.name, 'replaceKeyString' : firstScraper.replaceKeyString, 'replaceValueString' : firstScraper.replaceValueString})
					
					#create Scraper element
					scrapersXml = self.tree.find('Scrapers')
					
					#check if the current scraper already exists
					siteExists = False
					sitesXml = scrapersXml.findall('Site')
					for site in sitesXml:
						name = site.attrib.get('name')
						if name == scraperSite.name:
							siteExists = True
							break
						
					if not siteExists:
						#HACK: this only covers the first scraper (for offline scrapers)
						site = SubElement(scrapersXml, 'Site', 
							{ 
							'name' : scraperSite.name,
							'descFilePerGame' : str(scraperSite.descFilePerGame),
							'searchGameByCRC' : str(scraperSite.searchGameByCRC),
							'useFoldernameAsCRC' : str(scraperSite.useFoldernameAsCRC),
							'useFilenameAsCRC' : str(scraperSite.useFilenameAsCRC)
							})
																		
						scraper = scraperSite.scrapers[0]
						
						SubElement(site, 'Scraper', 
							{ 
							'parseInstruction' : scraper.parseInstruction,
							'source' : scraper.source,
							'encoding' : scraper.encoding
							})
				
		success, message = self.writeFile()
		return success, message
	
	
	def writeScrapers(self, scrapers):
		
		Logutil.log('write scraper sites', util.LOG_LEVEL_INFO)
				
		scraperSitesXml = self.tree.find('Scrapers')
				
		#HACK: remove all scrapers and create new
		for scraperSiteXml in scraperSitesXml.findall('Site'):				
			scraperSitesXml.remove(scraperSiteXml)
			
		for scraperSite in scrapers.values():
			
			Logutil.log('write scraper site: ' +str(scraperSite.name), util.LOG_LEVEL_INFO)
			
			#Don't write None-Scraper
			if(scraperSite.name == util.localize(32854)):
				Logutil.log('None scraper will be skipped', util.LOG_LEVEL_INFO)
				continue
			
			scraperSiteXml = SubElement(scraperSitesXml, 'Site', 
					{ 
					'name' : scraperSite.name,
					'descFilePerGame' : str(scraperSite.descFilePerGame),
					'searchGameByCRC' : str(scraperSite.searchGameByCRC),
					'useFoldernameAsCRC' : str(scraperSite.useFoldernameAsCRC),
					'useFilenameAsCRC' : str(scraperSite.useFilenameAsCRC)
					})
			
			for scraper in scraperSite.scrapers:
				
				#check if we can use a relative path to parseInstructions
				rcbScraperPath = os.path.join(util.RCBHOME, 'resources', 'scraper')
				pathParts = os.path.split(scraper.parseInstruction)
				if(pathParts[0].upper() == rcbScraperPath.upper()):
					scraper.parseInstruction = pathParts[1]
				
				scraperXml = SubElement(scraperSiteXml, 'Scraper', 
					{ 
					'parseInstruction' : scraper.parseInstruction,
					'source' : scraper.source,
					'encoding' : scraper.encoding,
					'returnUrl' : str(scraper.returnUrl)
					})
		
		success, message = self.writeFile()
		return success, message
	
	
	def writeMissingFilter(self, showHideOption, artworkOrGroup, artworkAndGroup, infoOrGroup, infoAndGroup):
		
		Logutil.log('write Missing Info Filter', util.LOG_LEVEL_INFO)
		
		missingFilterXml = self.tree.find('MissingFilter')
		
		#HACK: remove MissingFilter-element
		if(missingFilterXml != None):				
			self.tree.remove(missingFilterXml)
		
		missingFilterXml = SubElement(self.tree, 'MissingFilter')
		SubElement(missingFilterXml, 'showHideOption').text = showHideOption
		
		if(len(artworkOrGroup) > 0 or len(artworkAndGroup) > 0):
			missingArtworkXml = SubElement(missingFilterXml, 'missingArtworkFilter')
			self.addMissingFilterItems(missingArtworkXml, artworkOrGroup, 'orGroup')
			self.addMissingFilterItems(missingArtworkXml, artworkAndGroup, 'andGroup')
		if(len(infoOrGroup) > 0 or len(infoAndGroup) > 0):
			missingInfoXml = SubElement(missingFilterXml, 'missingInfoFilter')
			self.addMissingFilterItems(missingInfoXml, infoOrGroup, 'orGroup')
			self.addMissingFilterItems(missingInfoXml, infoAndGroup, 'andGroup')
				
		success, message = self.writeFile()
		return success, message
		
		
	def addMissingFilterItems(self, missingXml, group, groupName):		
		if(len(group) > 0):
			groupXml = SubElement(missingXml, groupName)
			for item in group:
				SubElement(groupXml, 'item').text = item
		
	
	def searchConfigObjects(self, xPath, nameToCompare, objectType):		
		objects = self.tree.findall(xPath)
		objectFound = False
		for obj in objects:
			objectName = obj.attrib.get('name')
			if(objectName == nameToCompare):
				objectFound = True
				break
		
		if(not objectFound):
			return False,  util.localize(32009) %(objectType, nameToCompare)
		
		return True, ''
	
		
	def removeRomCollection(self, RCName):
		
		Logutil.log('removeRomCollection', util.LOG_LEVEL_INFO)
		
		configFile = util.getConfigXmlPath()
		self.tree = ElementTree().parse(configFile)
		romCollectionsXml = self.tree.find('RomCollections')
		for romCollectionXml in romCollectionsXml.findall('RomCollection'):
			name = romCollectionXml.attrib.get('name')
			if(name == RCName):
				romCollectionsXml.remove(romCollectionXml)	
				
		success, message = self.writeFile()
		return success, message
		
	def addFileTypesForMame(self):
		Logutil.log('addFileTypesForMame', util.LOG_LEVEL_INFO)
		
		fileTypesXml = self.tree.find('FileTypes')
				
		#check if the MAME FileTypes already exist
		cabinetExists = False
		marqueeExists = False
		actionExists = False
		titleExists = False
		highestId = 0
		fileTypeXml = fileTypesXml.findall('FileType')
		for fileType in fileTypeXml:
			name = fileType.attrib.get('name')
			if name == 'cabinet':
				cabinetExists = True
			elif name == 'marquee':
				marqueeExists = True
			elif name == 'action':
				actionExists = True
			elif name == 'title':
				titleExists = True
			
			id = fileType.attrib.get('id')
			if int(id) > highestId:
				highestId = int(id)
		
		if not cabinetExists:
			self.createFileType(fileTypesXml, str(highestId +1), 'cabinet', 'image', 'game')
		if not marqueeExists:
			self.createFileType(fileTypesXml, str(highestId +2), 'marquee', 'image', 'game')			
		if not actionExists:
			self.createFileType(fileTypesXml, str(highestId +3), 'action', 'image', 'game')
		if not titleExists:
			self.createFileType(fileTypesXml, str(highestId +4), 'title', 'image', 'game')			
			
		
	def createFileType(self, fileTypesXml, id, name, type, parent):
		fileType = SubElement(fileTypesXml, 'FileType', {'id' : str(id), 'name' : name})
		SubElement(fileType, 'type').text = type
		SubElement(fileType, 'parent').text = parent
		
		
	def addImagePlacingForMame(self):
		Logutil.log('addImagePlacingForMame', util.LOG_LEVEL_INFO)
		
		imagePlacingXml = self.tree.find('ImagePlacing')
		
		#check if the MAME ImagePlacing options already exist
		cabinetExists = False
		marqueeExists = False		
		fileTypeForXml = imagePlacingXml.findall('fileTypeFor')
		for fileTypeFor in fileTypeForXml:
			name = fileTypeFor.attrib.get('name')
			if name == 'gameinfomamecabinet':
				cabinetExists = True
			elif name == 'gameinfomamemarquee':
				marqueeExists = True
				
		if not cabinetExists:
			fileTypeFor = SubElement(imagePlacingXml, 'fileTypeFor', {'name' : 'gameinfomamecabinet'})
			SubElement(fileTypeFor, 'fileTypeForGameList').text = 'cabinet'
			SubElement(fileTypeFor, 'fileTypeForGameList').text = 'boxfront'
			SubElement(fileTypeFor, 'fileTypeForGameList').text = 'title'
			SubElement(fileTypeFor, 'fileTypeForGameListSelected').text = 'cabinet'
			SubElement(fileTypeFor, 'fileTypeForGameListSelected').text = 'boxfront'
			SubElement(fileTypeFor, 'fileTypeForGameListSelected').text = 'title'			
			SubElement(fileTypeFor, 'fileTypeForMainViewBackground').text = 'boxfront'
			SubElement(fileTypeFor, 'fileTypeForMainViewBackground').text = 'title'
			SubElement(fileTypeFor, 'fileTypeForMainViewBackground').text = 'action'
			SubElement(fileTypeFor, 'fileTypeForMainViewGameInfoUpperLeft').text = 'title'
			SubElement(fileTypeFor, 'fileTypeForMainViewGameInfoUpperRight').text = 'action'
			SubElement(fileTypeFor, 'fileTypeForMainViewGameInfoLower').text = 'marquee'
			
		if not marqueeExists:
			fileTypeFor = SubElement(imagePlacingXml, 'fileTypeFor', {'name' : 'gameinfomamemarquee'})
			SubElement(fileTypeFor, 'fileTypeForGameList').text = 'marquee'
			SubElement(fileTypeFor, 'fileTypeForGameList').text = 'boxfront'
			SubElement(fileTypeFor, 'fileTypeForGameList').text = 'title'
			SubElement(fileTypeFor, 'fileTypeForGameListSelected').text = 'marquee'
			SubElement(fileTypeFor, 'fileTypeForGameListSelected').text = 'boxfront'
			SubElement(fileTypeFor, 'fileTypeForGameListSelected').text = 'title'			
			SubElement(fileTypeFor, 'fileTypeForMainViewBackground').text = 'boxfront'
			SubElement(fileTypeFor, 'fileTypeForMainViewBackground').text = 'title'
			SubElement(fileTypeFor, 'fileTypeForMainViewBackground').text = 'action'
			SubElement(fileTypeFor, 'fileTypeForMainViewGameInfoLeft').text = 'cabinet'
			SubElement(fileTypeFor, 'fileTypeForMainViewGameInfoUpperRight').text = 'action'
			SubElement(fileTypeFor, 'fileTypeForMainViewGameInfoLowerRight').text = 'title'
		
						
	def writeFile(self):
		Logutil.log('writeFile', util.LOG_LEVEL_INFO)
		#write file
		try:
			configFile = util.getConfigXmlPath()
			
			util.indentXml(self.tree)
			treeToWrite = ElementTree(self.tree)			
			treeToWrite.write(configFile)
			
			return True, ""
			
		except Exception, (exc):
			print("Error: Cannot write config.xml: " +str(exc))
			return False, util.localize(32008) +": " +str(exc)
Example #37
0
def makeContent(pages,media,p,tdir):
    pf=p['href']
    if debugMode():
        print("makeContent",pf)

    if pf==None:
        return False

    # strip off id references in path
    pp = pf.find("#")
    if pp>-1:
        f = pf[0:pp]
    else:
        f = pf
        
    # can we use extensions?
    extf = get_extensions()

    ff = os.path.normpath(tdir + os.sep + f)
    tree = ElementTree()
    tree.parse(ff)
                
    body = tree.find("body")

    if body==None:
        print("Error: no body in",f)
        return False

    # erase topic title line
    bh1 = body.find("h1")
    if bh1!=None:
        body.remove(bh1)
    
    # get a list of all the keywords in the file
    key_list = []
    head = tree.find("head")
    if head==None:
        print("Error: no head in",f)
        return False
    
    # init field info
    fieldlist = {}
    
    # make a list of all <meta> elements and then find
    # the keywords in DC.subject.
    metas = head.findall("meta")
    for meta in metas:
        mname = meta.get("name")
        
        if mname!=None and mname=="DC.subject":
            DCcontent = meta.get("content")
            if DCcontent!=None:
                content_tags = DCcontent.split(",")
                for ctag in content_tags:
                    ctag = ctag.strip()
                    if not ctag in key_list:
                        key_list.append(ctag)
                        # add tag to global tag list
                        if not ctag in get_tag_list():
                            add_tag(ctag)
                            # if we can, add this tag to the vocabulary
                            if extf and get_tagflag():
                                vcab = get_vocabulary()
                                addVocabularyTerm(vcab,ctag)
                                print(" add",vcab,"term",ctag)
                                
        # look for fields provided in the input
        # this is a local extension
        elif mname!=None and mname.startswith("FIELD."):
            fpos = mname.find(".")
            fieldname = mname[fpos+1:]
            fieldvalue = meta.get("content")
            if debugMode():
              print("field: ",fieldname,fieldvalue)
            fieldlist[fieldname] = fieldvalue
                
                            
    # store the tags (keywords)
    p['keywords'] = key_list

    if len(fieldlist)>0:
        p['fields'] = fieldlist
        
    if debugMode():
        print("key_list",key_list)
        print(get_tag_list())
        
    # get a list of parentlinks and remove them
    if get_sourcetype()=="DITAXHTML":
      plinks = []
      removeParentTopicLinks(body, plinks,1)
 
            
    # if not doing outline processing for DITA content,
    # move DITA parent links to top
    if not get_outline() and get_sourcetype()=="DITAXHTML":
      if len(plinks)>0:
          # move deleted parent links to the top
          for plink in plinks:
              body.insert(0,plink)
      elif get_cms()=="Drupal":
          # for Drupal, if there is no parent link, add one
          parent = p['parent']
          if parent!=None:
              pp = pages[parent]
              if debugMode():
                  print("add parent",parent,pp['title'])
              # manufacture a parent link
              ltext='<div class="parentlink"><strong> Parent topic: </strong><a href="'+pp['href']+'">'+pp['title']+'</a></div>'
              body.insert(0,XML(ltext))
              
    elif get_linkflag() and (get_sourcetype()=="DITAXHTML"):
        
        # also remove related task and concept links, previous and next topics
        divl = body.findall("div")
        for d in divl:
            c = d.get("class")
            if c!=None:
                if ("relinfo" in c) or ("familylinks" in c) or ("related-links" in c):
                    pmap = parentMap(tree)
                    pd = pmap[d]
                    pd.remove(d)
                    print(" removing div of class",c)

        # also remove olchildlink/ulchildlink <li> elements
        lil = body.findall("li")
        for l in lil:
            c = l.get("class")
            if c!=None:
                if ("olchildlink" in c) or ("ulchildlink" in c):
                    pmap = parentMap(tree)
                    pl = pmap[l]
                    pl.remove(l)
                    print(" removing li of class",c)
        
                
        # also remove all links from top TOC page
        if p['parent'] == None:
            flg = True
            while(flg):
                ule = body.find("ul")
                if ule!=None:
                    pmap = parentMap(tree)
                    pul = pmap[ule]
                    pul.remove(ule)
                else:
                    flg = False
                    
    elif get_outline():
        # when doing Drupal outline, delete TOC links from ditamaps
        parent_map = parentMap(tree)
        ullist = tree.find("body/div/ul")
        if ullist!=None:
            if ullist.get("class")=="ullinks":
                pnode = parent_map[ullist]
                pnode.remove(ullist)
                print(" remove page ullinks")
        
    # change the src attribute for any img elements to point to image URL
    images = body.iterfind("*//img")
    for img in images:
        src = img.get("src")
        if not isURL(src):
            if debugMode():
                print("src",src)
            # get the URL where Drupal has stored the image
            murl = getMediaURL(media,f,src)
            img.set("src",murl)

    # delete <a> elements that have no href. they cause formatting
    # problems when the CMS styles the page.
    alist = []
    remove_empty_anchors(body,1,alist)
       
    # turn the XML into a string
    bodystr = tostring(body)

    # save the edited page as a string
    if not isinstance(bodystr,tp_str):
        bodystr = bodystr.decode()
    p["content"] = bodystr

    return True
	def parse_document(self, etree, systemdate):
		query_queue = collections.defaultdict(list)

		systemdate_sqlite_str = systemdate.strftime('%Y-%m-%d %H:%M:%S.%f')


		snapshot_utc_date = datetime.datetime.utcfromtimestamp(
				int(etree.find('PinnacleFeedTime').text) / 1e3
		)
		snapshot_sqlite_str = snapshot_utc_date.strftime('%Y-%m-%d %H:%M:%S.%f')
		for event_el in etree.findall('events/event'):
			islive = coalesce(event_el, 'IsLive', 1)
			eventid = event_el.find('gamenumber').text
			is_contest = len(event_el.findall('contest_maximum')) != 0
			evdate_str = event_el.find('event_datetimeGMT').text
			evdate_dt = datetime.datetime.strptime(evdate_str, '%Y-%m-%d %H:%M')
# For unknown reasons, pinnacle's xml lists some matches as starting on weird
# times like 19:59 and 21:44. This will fix this.
			if evdate_dt.minute % 5 == 4:
				LOGGER.debug('Adding 1 minute to datetime: %s', evdate_dt)
				evdate_dt += datetime.timedelta(minutes = 1)

			def add_args(*args, **kwargs):
				if args[0] == 'odds':
					query_queue[eventid].append([
								BEFORE_SQL,
								[ kwargs['periodnumber'],
								kwargs['type'],
								kwargs['eventid']]
					])
				args_to_append = cl.get_insert_args_mysql(*args, **kwargs)
				query_queue[eventid].append(args_to_append)
				if args[0] == 'odds':
					query_queue[eventid].append([
								AFTER_SQL1,
								[ kwargs['periodnumber'],
								kwargs['type'],
								kwargs['eventid']]
					])
					query_queue[eventid].append([
								AFTER_SQL2,
								[ kwargs['periodnumber'],
								kwargs['type'],
								kwargs['eventid']]
					])

			add_args(
				'events',
				'replace',
				id = eventid,
				date = cl.datetime_to_sqlite_str(evdate_dt),
				sporttype = event_el.find('sporttype').text,
				league = event_el.find('league').text,
				islive = (1 if islive == 'Yes' else 0),
				description = coalesce(event_el, 'description', 1),
			)
			for participant_el in event_el.findall('participants/participant'):
				contestantnum = participant_el.find('contestantnum').text
				rotnum = participant_el.find('rotnum').text
				vhd = coalesce(participant_el, 'visiting_home_draw', 1)
				pitcher = coalesce(participant_el, 'pitcher', 1)
				add_args(
					'participants',
					'replace',
					eventid = eventid,
					contestantnum = contestantnum,
					rotnum = rotnum,
					vhdou = VHDOU_VAL[vhd],
					pitcher = pitcher,	
					name = participant_el.find('participant_name').text,
				)
				if is_contest:
					add_args(
						'snapshots',
						'replace',
						eventid = eventid,
						date = snapshot_sqlite_str,
						systemdate = systemdate_sqlite_str,
						mlmax = int(coalesce(event_el, 'contest_maximum', 1)),
					)
					add_args(
						'odds',
						'replace',
						eventid = eventid,
						periodnumber = None,
						snapshotdate = snapshot_sqlite_str,
						type = 'm',
						threshold = 0,
						vhdou = None,
						price = dec_odds(
							coalesce(
								participant_el, 'odds/moneyline_value', 1
							)
						),
						to_base = fix_to_base(coalesce(
									participant_el, 'odds/to_base', 1
						)),
						contestantnum = contestantnum,
						rotnum = rotnum,

					)
				
			if not is_contest:
				for period_el in event_el.findall('periods/period'):
					periodnumber = period_el.find('period_number').text
					add_args(
						'periods',
						'replace',
						eventid = eventid,
						number = periodnumber,
						description = period_el.find('period_description').text,
						cutoff = period_el.find('periodcutoff_datetimeGMT').text,
					)
					add_args(
						'snapshots',
						'replace', 
						eventid = eventid,
						periodnumber = periodnumber,
						date = snapshot_sqlite_str,
						systemdate = systemdate_sqlite_str,
						status = period_el.find('period_status').text,
						upd = period_el.find('period_update').text,
						spreadmax = int(float(period_el.find('spread_maximum').text)),
						mlmax = int(float(period_el.find('moneyline_maximum').text)),
						totalmax = int(float(period_el.find('total_maximum').text)),
					)

					moneyline_el = coalesce(period_el, 'moneyline')
					if moneyline_el is not None:
						for vhd in ['home', 'visiting', 'draw']:
							if len(moneyline_el.findall('moneyline_' + vhd)) > 0:
								contestantnum, rotnum = period_el_to_contestantnum_rotnum(period_el, event_el, vhd)
								add_args(
									'odds',
									None,
									eventid = eventid,
									periodnumber = periodnumber,
									snapshotdate = snapshot_sqlite_str,
									type = 'm',
									threshold = 0,
									vhdou = VHDOU_VAL[vhd],
									price = dec_odds(
										coalesce(
											moneyline_el, 'moneyline_' + vhd, 1
										)
									),
									contestantnum = contestantnum,
									rotnum = rotnum,
								)

					spread_el = coalesce(period_el, 'spread')
					if spread_el is not None:
						for vhd in ['home', 'visiting']:
							if len(spread_el.findall('spread_' + vhd)) > 0:
								contestantnum, rotnum = period_el_to_contestantnum_rotnum(period_el, event_el, vhd)
								add_args(
									'odds',
									None,
									eventid = eventid,
									periodnumber = periodnumber,
									snapshotdate = snapshot_sqlite_str,
									type = 's',
									vhdou = VHDOU_VAL[vhd],
									threshold = float(spread_el.find('spread_' + vhd).text),
									price = dec_odds(
										coalesce(
											spread_el, 'spread_adjust_' + vhd, 1
										)
									),
									contestantnum = contestantnum,
									rotnum = rotnum,
								)

					total_el = coalesce(period_el, 'total')
					if total_el is not None:
						for ou in ['over', 'under']:
							contestantnum, rotnum = period_el_to_contestantnum_rotnum(period_el, event_el, ou)
							add_args(
								'odds',
								None,
								eventid = eventid,
								periodnumber = periodnumber,
								snapshotdate = snapshot_sqlite_str,
								type = 't',
								vhdou = VHDOU_VAL[ou],
								threshold = float(total_el.find('total_points').text),
								price = dec_odds(
									coalesce(
										total_el, ou + '_adjust', 1
									)
								),
								contestantnum = contestantnum,
								rotnum = rotnum,
							)

		with get_conn_mysql() as conn:
			LOGGER.info('Queue holds %s queries. Starting execution...', sum(len(al) for al in query_queue.itervalues()))
			for argslist in query_queue.itervalues():
				for args in argslist:
					try:
						if args[0] != AFTER_SQL1 and args[0] != AFTER_SQL2:
							conn.execute(*args)
						else:
							odds_ids = [r[0] for r in conn.execute(args[0], args[1]).fetchall()]
							for odds_id in odds_ids:
								odds_update_sql = '''
									update odds
									set {col} = 1
									where id = %s
								'''.format(col = 'opening' 
										if args[0] == AFTER_SQL1 
										else 'latest')
								conn.execute(odds_update_sql, [odds_id])
					except Exception as e:
						print 'Query that caused exception:'
						print args[0]
						print args[1]
						print
						raise
				conn.commit()
			LOGGER.info('Queue execution done.')
			if self.call_when_update_done is not None:
				self.call_when_update_done(systemdate)