def linkbaseDiscover(self, linkbaseElement, inInstance=False): for lbElement in linkbaseElement.childNodes: if lbElement.nodeType == 1: #element lbLn = lbElement.localName lbNs = lbElement.namespaceURI if lbNs == XbrlConst.link: if lbLn == "roleRef" or lbLn == "arcroleRef": href = self.discoverHref(lbElement) if href is None: self.modelXbrl.error( "Linkbase in {0} {1} href attribute missing or malformed".format( os.path.basename(self.uri), lbLn), "err", "xbrl:hrefMissing") else: self.hrefObjects.append(href) continue if lbElement.getAttributeNS(XbrlConst.xlink, "type") == "extended": self.schemalocateElementNamespace(lbElement) arcrolesFound = set() dimensionArcFound = False formulaArcFound = False euRenderingArcFound = False linkQn = qname(lbElement) linkrole = lbElement.getAttributeNS(XbrlConst.xlink, "role") modelLink = ModelObject.createLink(self, lbElement) if inInstance: #index footnote links even if no arc children baseSetKeys = (("XBRL-footnotes",None,None,None), ("XBRL-footnotes",linkrole,None,None)) for baseSetKey in baseSetKeys: self.modelXbrl.baseSets[baseSetKey].append(modelLink) for linkElement in lbElement.childNodes: if linkElement.nodeType == 1: #element self.schemalocateElementNamespace(linkElement) xlinkType = linkElement.getAttributeNS(XbrlConst.xlink, "type") modelResource = None if xlinkType == "locator": nonDTS = linkElement.namespaceURI != XbrlConst.link or linkElement.localName != "loc" # only link:loc elements are discovered or processed href = self.discoverHref(linkElement, nonDTS=nonDTS) if href is None: self.modelXbrl.error( "Linkbase in {0} {1} href attribute missing or malformed".format( os.path.basename(self.uri), lbLn), "err", "xbrl:hrefMissing") else: modelResource = ModelObject.createLocator(self, linkElement, href) elif xlinkType == "arc": arcQn = qname(linkElement) arcrole = linkElement.getAttributeNS(XbrlConst.xlink, "arcrole") if arcrole not in arcrolesFound: if linkrole == "": linkrole = XbrlConst.defaultLinkRole #index by both arcrole and linkrole#arcrole and dimensionsions if applicable baseSetKeys = [(arcrole, linkrole, linkQn, arcQn)] baseSetKeys.append((arcrole, linkrole, None, None)) baseSetKeys.append((arcrole, None, None, None)) if XbrlConst.isDimensionArcrole(arcrole) and not dimensionArcFound: baseSetKeys.append(("XBRL-dimensions", None, None, None)) baseSetKeys.append(("XBRL-dimensions", linkrole, None, None)) dimensionArcFound = True if XbrlConst.isFormulaArcrole(arcrole) and not formulaArcFound: baseSetKeys.append(("XBRL-formulae", None, None, None)) baseSetKeys.append(("XBRL-formulae", linkrole, None, None)) formulaArcFound = True if XbrlConst.isEuRenderingArcrole(arcrole) and not euRenderingArcFound: baseSetKeys.append(("EU-rendering", None, None, None)) baseSetKeys.append(("EU-rendering", linkrole, None, None)) euRenderingArcFound = True self.modelXbrl.hasEuRendering = True for baseSetKey in baseSetKeys: self.modelXbrl.baseSets[baseSetKey].append(modelLink) arcrolesFound.add(arcrole) elif xlinkType == "resource": # create resource and make accessible by id for document modelResource = ModelObject.createResource(self, linkElement) if modelResource is not None: if linkElement.hasAttribute("id"): self.idObjects[linkElement.getAttribute("id")] = modelResource modelLink.labeledResources[linkElement.getAttributeNS(XbrlConst.xlink, "label")] \ .append(modelResource) else: XmlUtil.markIdAttributes(linkElement)
def linkbaseDiscover(self, linkbaseElement, inInstance=False): for lbElement in linkbaseElement.childNodes: if lbElement.nodeType == 1: #element lbLn = lbElement.localName lbNs = lbElement.namespaceURI if lbNs == XbrlConst.link: if lbLn == "roleRef" or lbLn == "arcroleRef": href = self.discoverHref(lbElement) if href is None: self.modelXbrl.error( "Linkbase in {0} {1} href attribute missing or malformed" .format(os.path.basename(self.uri), lbLn), "err", "xbrl:hrefMissing") else: self.hrefObjects.append(href) continue if lbElement.getAttributeNS(XbrlConst.xlink, "type") == "extended": self.schemalocateElementNamespace(lbElement) arcrolesFound = set() dimensionArcFound = False formulaArcFound = False euRenderingArcFound = False linkQn = qname(lbElement) linkrole = lbElement.getAttributeNS( XbrlConst.xlink, "role") modelLink = ModelObject.createLink(self, lbElement) if inInstance: #index footnote links even if no arc children baseSetKeys = (("XBRL-footnotes", None, None, None), ("XBRL-footnotes", linkrole, None, None)) for baseSetKey in baseSetKeys: self.modelXbrl.baseSets[baseSetKey].append( modelLink) for linkElement in lbElement.childNodes: if linkElement.nodeType == 1: #element self.schemalocateElementNamespace(linkElement) xlinkType = linkElement.getAttributeNS( XbrlConst.xlink, "type") modelResource = None if xlinkType == "locator": nonDTS = linkElement.namespaceURI != XbrlConst.link or linkElement.localName != "loc" # only link:loc elements are discovered or processed href = self.discoverHref(linkElement, nonDTS=nonDTS) if href is None: self.modelXbrl.error( "Linkbase in {0} {1} href attribute missing or malformed" .format(os.path.basename(self.uri), lbLn), "err", "xbrl:hrefMissing") else: modelResource = ModelObject.createLocator( self, linkElement, href) elif xlinkType == "arc": arcQn = qname(linkElement) arcrole = linkElement.getAttributeNS( XbrlConst.xlink, "arcrole") if arcrole not in arcrolesFound: if linkrole == "": linkrole = XbrlConst.defaultLinkRole #index by both arcrole and linkrole#arcrole and dimensionsions if applicable baseSetKeys = [(arcrole, linkrole, linkQn, arcQn)] baseSetKeys.append( (arcrole, linkrole, None, None)) baseSetKeys.append( (arcrole, None, None, None)) if XbrlConst.isDimensionArcrole( arcrole) and not dimensionArcFound: baseSetKeys.append(("XBRL-dimensions", None, None, None)) baseSetKeys.append( ("XBRL-dimensions", linkrole, None, None)) dimensionArcFound = True if XbrlConst.isFormulaArcrole( arcrole) and not formulaArcFound: baseSetKeys.append(("XBRL-formulae", None, None, None)) baseSetKeys.append( ("XBRL-formulae", linkrole, None, None)) formulaArcFound = True if XbrlConst.isEuRenderingArcrole( arcrole ) and not euRenderingArcFound: baseSetKeys.append( ("EU-rendering", None, None, None)) baseSetKeys.append( ("EU-rendering", linkrole, None, None)) euRenderingArcFound = True self.modelXbrl.hasEuRendering = True for baseSetKey in baseSetKeys: self.modelXbrl.baseSets[ baseSetKey].append(modelLink) arcrolesFound.add(arcrole) elif xlinkType == "resource": # create resource and make accessible by id for document modelResource = ModelObject.createResource( self, linkElement) if modelResource is not None: if linkElement.hasAttribute("id"): self.idObjects[linkElement.getAttribute( "id")] = modelResource modelLink.labeledResources[linkElement.getAttributeNS(XbrlConst.xlink, "label")] \ .append(modelResource) else: XmlUtil.markIdAttributes(linkElement)
def load(modelXbrl, uri, base=None, isEntry=False, isIncluded=None, namespace=None, reloadCache=False): normalizedUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(uri, base) if isEntry: modelXbrl.uri = normalizedUri modelXbrl.uriDir = os.path.dirname(normalizedUri) for i in range(modelXbrl.modelManager.disclosureSystem.maxSubmissionSubdirectoryEntryNesting): modelXbrl.uriDir = os.path.dirname(modelXbrl.uriDir) if modelXbrl.modelManager.validateDisclosureSystem and \ not normalizedUri.startswith(modelXbrl.uriDir) and \ not modelXbrl.modelManager.disclosureSystem.hrefValid(normalizedUri): blocked = modelXbrl.modelManager.disclosureSystem.blockDisallowedReferences modelXbrl.error( "Prohibited file for filings{1}: {0}".format(normalizedUri, _(" blocked") if blocked else ""), "err", "EFM.6.22.02", "GFM.1.1.3", "SBR.NL.2.1.0.06") if blocked: return None if normalizedUri in modelXbrl.modelManager.disclosureSystem.mappedFiles: mappedUri = modelXbrl.modelManager.disclosureSystem.mappedFiles[normalizedUri] else: # handle mapped paths mappedUri = normalizedUri for mapFrom, mapTo in modelXbrl.modelManager.disclosureSystem.mappedPaths: if normalizedUri.startswith(mapFrom): mappedUri = mapTo + normalizedUri[len(mapFrom):] break if modelXbrl.fileSource.isInArchive(mappedUri): filepath = mappedUri else: filepath = modelXbrl.modelManager.cntlr.webCache.getfilename(mappedUri, reload=reloadCache) if filepath: uri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(filepath) if filepath is None: # error such as HTTPerror is already logged modelXbrl.error( "File can not be loaded: {0}".format( mappedUri), "err", "FileNotLoadable") type = Type.Unknown return None modelDocument = modelXbrl.urlDocs.get(mappedUri) if modelDocument: return modelDocument # load XML and determine type of model document modelXbrl.modelManager.showStatus(_("parsing {0}").format(uri)) file = None try: if modelXbrl.modelManager.validateDisclosureSystem: file = ValidateFilingText.checkfile(modelXbrl,filepath) else: file = modelXbrl.fileSource.file(filepath) xmlDocument = xml.dom.minidom.parse(file) file.close() except EnvironmentError as err: modelXbrl.error( "{0}: file error: {1}".format( os.path.basename(uri), err), "err", "IOerror") type = Type.Unknown if file: file.close() return None except (xml.parsers.expat.ExpatError, xml.dom.DOMException, ValueError) as err: # ValueError raised on bad format of qnames, xmlns'es, or parameters modelXbrl.error( "{0}: import error: {1}".format( os.path.basename(uri), err), "err", "XMLsyntax") type = Type.Unknown if file: file.close() return None # identify document #modelXbrl.modelManager.addToLog("discovery: {0}".format( # os.path.basename(uri))) modelXbrl.modelManager.showStatus(_("loading {0}").format(uri)) modelDocument = None for rootNode in xmlDocument.childNodes: if rootNode.nodeType == 1: #element ln = rootNode.localName ns = rootNode.namespaceURI # type classification if ns == XbrlConst.xsd and ln == "schema": type = Type.SCHEMA elif ns == XbrlConst.link: if ln == "linkbase": type = Type.LINKBASE elif ln == "xbrl": type = Type.INSTANCE elif ns == XbrlConst.xbrli: if ln == "xbrl": type = Type.INSTANCE elif ns == XbrlConst.xhtml and \ ln == "html" or ln == "xhtml": type = Type.Unknown for i in range(len(rootNode.attributes)): if rootNode.attributes.item(i).value == XbrlConst.ixbrl: type = Type.INLINEXBRL break XmlUtil.markIdAttributes(rootNode) # required for minidom searchability elif ln == "report" and ns == XbrlConst.ver: type = Type.VERSIONINGREPORT elif ln == "testcases" or ln == "documentation": type = Type.TESTCASESINDEX elif ln == "testcase": type = Type.TESTCASE elif ln == "registry" and ns == XbrlConst.registry: type = Type.REGISTRY elif ln == "rss": type = Type.RSSFEED else: type = Type.Unknown nestedInline = XmlUtil.descendant(rootNode, XbrlConst.xhtml, ("html", "xhtml")) if nestedInline: for i in range(len(nestedInline.attributes)): if nestedInline.attributes.item(i).value == XbrlConst.ixbrl: type = Type.INLINEXBRL rootNode = nestedInline break XmlUtil.markIdAttributes(rootNode) # required for minidom searchability #create modelDocument object or subtype as identified if type == Type.VERSIONINGREPORT: from arelle.ModelVersReport import ModelVersReport modelDocument = ModelVersReport(modelXbrl, type, mappedUri, filepath, xmlDocument) elif type == Type.RSSFEED: from arelle.ModelRssObject import ModelRssObject modelDocument = ModelRssObject(modelXbrl, type, mappedUri, filepath, xmlDocument) else: modelDocument = ModelDocument(modelXbrl, type, mappedUri, filepath, xmlDocument) modelDocument.xmlRootElement = rootNode modelDocument.schemaLocationElements.add(rootNode) if isEntry: modelDocument.inDTS = True # discovery (parsing) if type == Type.SCHEMA: modelDocument.schemaDiscover(rootNode, isIncluded, namespace) elif type == Type.LINKBASE: modelDocument.linkbaseDiscover(rootNode) elif type == Type.INSTANCE: modelDocument.instanceDiscover(rootNode) elif type == Type.INLINEXBRL: modelDocument.inlineXbrlDiscover(rootNode) elif type == Type.VERSIONINGREPORT: modelDocument.versioningReportDiscover(rootNode) elif type == Type.TESTCASESINDEX: modelDocument.testcasesIndexDiscover(xmlDocument) elif type == Type.TESTCASE: modelDocument.testcaseDiscover(rootNode) elif type == Type.REGISTRY: modelDocument.registryDiscover(rootNode) elif type == Type.VERSIONINGREPORT: modelDocument.versioningReportDiscover(rootNode) elif type == Type.RSSFEED: modelDocument.rssFeedDiscover(rootNode) break return modelDocument
def load(modelXbrl, uri, base=None, isEntry=False, isIncluded=None, namespace=None, reloadCache=False): normalizedUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( uri, base) if isEntry: modelXbrl.uri = normalizedUri modelXbrl.uriDir = os.path.dirname(normalizedUri) for i in range(modelXbrl.modelManager.disclosureSystem. maxSubmissionSubdirectoryEntryNesting): modelXbrl.uriDir = os.path.dirname(modelXbrl.uriDir) if modelXbrl.modelManager.validateDisclosureSystem and \ not normalizedUri.startswith(modelXbrl.uriDir) and \ not modelXbrl.modelManager.disclosureSystem.hrefValid(normalizedUri): blocked = modelXbrl.modelManager.disclosureSystem.blockDisallowedReferences modelXbrl.error( "Prohibited file for filings{1}: {0}".format( normalizedUri, _(" blocked") if blocked else ""), "err", "EFM.6.22.02", "GFM.1.1.3", "SBR.NL.2.1.0.06") if blocked: return None if normalizedUri in modelXbrl.modelManager.disclosureSystem.mappedFiles: mappedUri = modelXbrl.modelManager.disclosureSystem.mappedFiles[ normalizedUri] else: # handle mapped paths mappedUri = normalizedUri for mapFrom, mapTo in modelXbrl.modelManager.disclosureSystem.mappedPaths: if normalizedUri.startswith(mapFrom): mappedUri = mapTo + normalizedUri[len(mapFrom):] break if modelXbrl.fileSource.isInArchive(mappedUri): filepath = mappedUri else: filepath = modelXbrl.modelManager.cntlr.webCache.getfilename( mappedUri, reload=reloadCache) if filepath: uri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(filepath) if filepath is None: # error such as HTTPerror is already logged modelXbrl.error("File can not be loaded: {0}".format(mappedUri), "err", "FileNotLoadable") type = Type.Unknown return None modelDocument = modelXbrl.urlDocs.get(mappedUri) if modelDocument: return modelDocument # load XML and determine type of model document modelXbrl.modelManager.showStatus(_("parsing {0}").format(uri)) file = None try: if modelXbrl.modelManager.validateDisclosureSystem: file = ValidateFilingText.checkfile(modelXbrl, filepath) else: file = modelXbrl.fileSource.file(filepath) xmlDocument = xml.dom.minidom.parse(file) file.close() except EnvironmentError as err: modelXbrl.error( "{0}: file error: {1}".format(os.path.basename(uri), err), "err", "IOerror") type = Type.Unknown if file: file.close() return None except ( xml.parsers.expat.ExpatError, xml.dom.DOMException, ValueError ) as err: # ValueError raised on bad format of qnames, xmlns'es, or parameters modelXbrl.error( "{0}: import error: {1}".format(os.path.basename(uri), err), "err", "XMLsyntax") type = Type.Unknown if file: file.close() return None # identify document #modelXbrl.modelManager.addToLog("discovery: {0}".format( # os.path.basename(uri))) modelXbrl.modelManager.showStatus(_("loading {0}").format(uri)) modelDocument = None for rootNode in xmlDocument.childNodes: if rootNode.nodeType == 1: #element ln = rootNode.localName ns = rootNode.namespaceURI # type classification if ns == XbrlConst.xsd and ln == "schema": type = Type.SCHEMA elif ns == XbrlConst.link: if ln == "linkbase": type = Type.LINKBASE elif ln == "xbrl": type = Type.INSTANCE elif ns == XbrlConst.xbrli: if ln == "xbrl": type = Type.INSTANCE elif ns == XbrlConst.xhtml and \ ln == "html" or ln == "xhtml": type = Type.Unknown for i in range(len(rootNode.attributes)): if rootNode.attributes.item(i).value == XbrlConst.ixbrl: type = Type.INLINEXBRL break XmlUtil.markIdAttributes( rootNode) # required for minidom searchability elif ln == "report" and ns == XbrlConst.ver: type = Type.VERSIONINGREPORT elif ln == "testcases" or ln == "documentation": type = Type.TESTCASESINDEX elif ln == "testcase": type = Type.TESTCASE elif ln == "registry" and ns == XbrlConst.registry: type = Type.REGISTRY elif ln == "rss": type = Type.RSSFEED else: type = Type.Unknown nestedInline = XmlUtil.descendant(rootNode, XbrlConst.xhtml, ("html", "xhtml")) if nestedInline: for i in range(len(nestedInline.attributes)): if nestedInline.attributes.item( i).value == XbrlConst.ixbrl: type = Type.INLINEXBRL rootNode = nestedInline break XmlUtil.markIdAttributes( rootNode) # required for minidom searchability #create modelDocument object or subtype as identified if type == Type.VERSIONINGREPORT: from arelle.ModelVersReport import ModelVersReport modelDocument = ModelVersReport(modelXbrl, type, mappedUri, filepath, xmlDocument) elif type == Type.RSSFEED: from arelle.ModelRssObject import ModelRssObject modelDocument = ModelRssObject(modelXbrl, type, mappedUri, filepath, xmlDocument) else: modelDocument = ModelDocument(modelXbrl, type, mappedUri, filepath, xmlDocument) modelDocument.xmlRootElement = rootNode modelDocument.schemaLocationElements.add(rootNode) if isEntry: modelDocument.inDTS = True # discovery (parsing) if type == Type.SCHEMA: modelDocument.schemaDiscover(rootNode, isIncluded, namespace) elif type == Type.LINKBASE: modelDocument.linkbaseDiscover(rootNode) elif type == Type.INSTANCE: modelDocument.instanceDiscover(rootNode) elif type == Type.INLINEXBRL: modelDocument.inlineXbrlDiscover(rootNode) elif type == Type.VERSIONINGREPORT: modelDocument.versioningReportDiscover(rootNode) elif type == Type.TESTCASESINDEX: modelDocument.testcasesIndexDiscover(xmlDocument) elif type == Type.TESTCASE: modelDocument.testcaseDiscover(rootNode) elif type == Type.REGISTRY: modelDocument.registryDiscover(rootNode) elif type == Type.VERSIONINGREPORT: modelDocument.versioningReportDiscover(rootNode) elif type == Type.RSSFEED: modelDocument.rssFeedDiscover(rootNode) break return modelDocument