def generateInfoset(dts, infosetFile):
if dts.fileSource.isArchive:
return
import os, io
from arelle import XmlUtil, XbrlConst
from arelle.ValidateXbrlCalcs import inferredPrecision, inferredDecimals
XmlUtil.setXmlns(dts.modelDocument, "ptv", "http://www.xbrl.org/2003/ptv")
numFacts = 0
for fact in dts.facts:
try:
if fact.concept.periodType:
fact.set("{http://www.xbrl.org/2003/ptv}periodType", fact.concept.periodType)
if fact.concept.balance:
fact.set("{http://www.xbrl.org/2003/ptv}balance", fact.concept.balance)
if fact.isNumeric and not fact.isNil:
fact.set("{http://www.xbrl.org/2003/ptv}decimals", str(inferredDecimals(fact)))
fact.set("{http://www.xbrl.org/2003/ptv}precision", str(inferredPrecision(fact)))
numFacts += 1
except Exception as err:
dts.error("saveInfoset.exception",
_("Facts exception %(fact)s %(value)s %(error)s."),
modelObject=fact, fact=fact.qname, value=fact.effectiveValue, error = err)
fh = open(infosetFile, "w", encoding="utf-8")
XmlUtil.writexml(fh, dts.modelDocument.xmlDocument, encoding="utf-8")
fh.close()
dts.info("info:saveInfoset",
_("Infoset of %(entryFile)s has %(numberOfFacts)s facts in infoset file %(infosetOutputFile)s."),
modelObject=dts,
entryFile=dts.uri, numberOfFacts=numFacts, infosetOutputFile=infosetFile)
def lbTreeWalk(lbType, parentElt, lbList, roleRefs, locs=None, fromPrefix=None, fromName=None):
order = 1.0
for toPrefix, toName, rel, list in lbList:
if rel == "_ELR_":
role = "unspecified"
if toPrefix and toPrefix.startswith("http://"): # have a role specified
role = toPrefix
elif toName: #may be a definition
for linkroleUri, modelRoleTypes in dts.roleTypes.items():
definition = modelRoleTypes[0].definition
if toName == definition:
role = linkroleUri
break
if role != XbrlConst.defaultLinkRole and role in dts.roleTypes: # add roleRef
roleType = modelRoleTypes[0]
roleRef = ("roleRef", role, roleType.modelDocument.uri + "#" + roleType.id)
roleRefs.add(roleRef)
linkElt = XmlUtil.addChild(parentElt,
XbrlConst.link, lbType + "Link",
attributes=(("{http://www.w3.org/1999/xlink}type", "extended"),
("{http://www.w3.org/1999/xlink}role", role)))
locs = set()
lbTreeWalk(lbType, linkElt, list, roleRefs, locs)
else:
toHref = extensionHref(toPrefix, toName)
toLabel = toPrefix + "_" + toName
if toHref not in locs:
XmlUtil.addChild(parentElt,
XbrlConst.link, "loc",
attributes=(("{http://www.w3.org/1999/xlink}type", "locator"),
("{http://www.w3.org/1999/xlink}href", toHref),
("{http://www.w3.org/1999/xlink}label", toLabel)))
locs.add(toHref)
if rel != "_root_":
fromLabel = fromPrefix + "_" + fromName
if lbType == "calculation":
otherAttrs = ( ("weight", list), )
else:
otherAttrs = ( )
if rel == "_dimensions_": # pick proper consecutive arcrole
fromConcept = hrefConcept(fromPrefix, fromName)
toConcept = hrefConcept(toPrefix, toName)
if toConcept is not None and toConcept.isHypercubeItem:
rel = XbrlConst.all
elif toConcept is not None and toConcept.isDimensionItem:
rel = XbrlConst.hypercubeDimension
elif fromConcept is not None and fromConcept.isDimensionItem:
rel = XbrlConst.dimensionDomain
else:
rel = XbrlConst.domainMember
XmlUtil.addChild(parentElt,
XbrlConst.link, lbType + "Arc",
attributes=(("{http://www.w3.org/1999/xlink}type", "arc"),
("{http://www.w3.org/1999/xlink}arcrole", rel),
("{http://www.w3.org/1999/xlink}from", fromLabel),
("{http://www.w3.org/1999/xlink}to", toLabel),
("order", order)) + otherAttrs )
order += 1.0
if lbType != "calculation" or rel == "_root_":
lbTreeWalk(lbType, parentElt, list, roleRefs, locs, toPrefix, toName)
def factAspectValue(fact, aspect, view=False):
if aspect == Aspect.LOCATION:
parentQname = fact.getparent().qname
if parentQname == XbrlConst.qnXbrliXbrl: # not tuple
return NONE
return parentQname # tuple
elif aspect == Aspect.CONCEPT:
return fact.qname
elif fact.isTuple or fact.context is None:
return NONE #subsequent aspects don't exist for tuples
elif aspect == Aspect.UNIT:
if fact.unit is None:
return NONE
measures = fact.unit.measures
if measures[1]:
return "{0} / {1}".format(' '.join(str(m) for m in measures[0]),
' '.join(str(m) for m in measures[1]))
else:
return ' '.join(str(m) for m in measures[0])
else:
context = fact.context
if aspect == Aspect.PERIOD:
return ("forever" if context.isForeverPeriod else
XmlUtil.dateunionValue(context.instantDatetime, subtractOneDay=True) if context.isInstantPeriod else
XmlUtil.dateunionValue(context.startDatetime) + "-" + XmlUtil.dateunionValue(context.endDatetime, subtractOneDay=True))
elif aspect == Aspect.ENTITY_IDENTIFIER:
if view:
return context.entityIdentifier[1]
else:
return context.entityIdentifier # (scheme, identifier)
elif aspect in (Aspect.COMPLETE_SEGMENT, Aspect.COMPLETE_SCENARIO,
Aspect.NON_XDT_SEGMENT, Aspect.NON_XDT_SCENARIO):
return ''.join(XmlUtil.xmlstring(elt, stripXmlns=True, prettyPrint=True)
for elt in context.nonDimValues(aspect))
def reference(self):
efmNameElts = XmlUtil.children(self.getparent(), None, "name")
for efmNameElt in efmNameElts:
if efmNameElt is not None and efmNameElt.text.startswith("EDGAR"):
return efmNameElt.text
referenceElement = XmlUtil.descendant(self, None, "reference")
if referenceElement is not None: # formula test suite
return "{0}#{1}".format(
referenceElement.get("specification"),
referenceElement.get("id"))
referenceElement = XmlUtil.descendant(self, None,
"documentationReference")
if referenceElement is not None: # w3c test suite
return referenceElement.get("{http://www.w3.org/1999/xlink}href")
descriptionElement = XmlUtil.descendant(self, None, "description")
if descriptionElement is not None and descriptionElement.get(
"reference"):
return descriptionElement.get("reference") # xdt test suite
if self.getparent().get("description"):
return self.getparent().get(
"description") # base spec 2.1 test suite
functRegistryRefElt = XmlUtil.descendant(self.getparent(), None,
"reference")
if functRegistryRefElt is not None: # function registry
return functRegistryRefElt.get(
"{http://www.w3.org/1999/xlink}href")
return None
def expected(self):
if self.localName == "testcase":
return self.document.basename[:4] #starts with PASS or FAIL
errorElement = XmlUtil.descendant(self, None, "error")
if errorElement is not None:
return ModelValue.qname(errorElement, XmlUtil.text(errorElement))
resultElement = XmlUtil.descendant(self, None, "result")
if resultElement is not None:
expected = resultElement.get("expected")
if expected:
return expected
for assertElement in XmlUtil.children(resultElement, None, "assert"):
num = assertElement.get("num")
if len(num) == 5:
return "EFM.{0}.{1}.{2}".format(num[0],num[1:3],num[3:6])
asserTests = {}
for atElt in XmlUtil.children(resultElement, None, "assertionTests"):
try:
asserTests[atElt.get("assertionID")] = (_INT(atElt.get("countSatisfied")),_INT(atElt.get("countNotSatisfied")))
except ValueError:
pass
if asserTests:
return asserTests
elif self.get("result"):
return self.get("result")
return None
def explicitDims(self):
return {
(self.prefixedNameQname(e.get("dimension")), self.prefixedNameQname(XmlUtil.text(qn)))
for e in XmlUtil.children(self, XbrlConst.formula, "explicitDimension")
for m in XmlUtil.children(e, XbrlConst.formula, "member")
for qn in XmlUtil.children(m, XbrlConst.formula, "qname")
}
def primaryItemQname(self):
conceptRule = XmlUtil.child(self, XbrlConst.formula, "concept")
if conceptRule is not None:
qnameElt = XmlUtil.child(conceptRule, XbrlConst.formula, "qname")
if qnameElt is not None:
return qname(qnameElt, qnameElt.text)
return None
def file(self,filepath):
archiveFileSource = self.fileSourceContainingFilepath(filepath)
if archiveFileSource is not None:
if filepath.startswith(archiveFileSource.basefile):
archiveFileName = filepath[len(archiveFileSource.basefile) + 1:]
else: # filepath.startswith(self.baseurl)
archiveFileName = filepath[len(archiveFileSource.baseurl) + 1:]
if archiveFileSource.isZip:
b = archiveFileSource.fs.read(archiveFileName.replace("\\","/"))
encoding = XmlUtil.encoding(b)
return (io.TextIOWrapper(
io.BytesIO(b),
encoding=encoding), encoding)
elif archiveFileSource.isXfd:
for data in archiveFileSource.xfdDocument.iter(tag="data"):
outfn = data.findtext("filename")
if outfn == archiveFileName:
b64data = data.findtext("mimedata")
if b64data:
# convert to bytes
#byteData = []
#for c in b64data:
# byteData.append(ord(c))
b = base64.b64decode(b64data.encode("latin-1"))
# remove BOM codes if present
if len(b) > 3 and b[0] == 239 and b[1] == 187 and b[2] == 191:
start = 3;
length = len(b) - 3;
b = b[start:start + length]
else:
start = 0;
length = len(b);
# pass back as ascii
#str = ""
#for bChar in b[start:start + length]:
# str += chr( bChar )
#return str
return (io.TextIOWrapper(
io.BytesIO(b),
encoding=XmlUtil.encoding(b)), "latin-1")
return (None,None)
# check encoding
with open(filepath, 'rb') as fb:
hdrBytes = fb.peek(512)
encoding = XmlUtil.encoding(hdrBytes)
if encoding.lower() in ('utf-8','utf8'):
text = None
else:
text = fb.read().decode(encoding)
# allow filepath to close
# this may not be needed for Mac or Linux, needs confirmation!!!
if text is None: # ok to read as utf-8
return (open(filepath, 'rt', encoding='utf-8'), encoding)
else:
# strip XML declaration
xmlDeclarationMatch = XMLdeclaration.search(text)
if xmlDeclarationMatch: # remove it for lxml
start,end = xmlDeclarationMatch.span()
text = text[0:start] + text[end:]
return (io.StringIO(initial_value=text), encoding)
def generateAxis(newLinkElt, newAxisParentElt, srcAxisElt, axisMbrRelSet, visited):
for rel in axisMbrRelSet.fromModelObject(srcAxisElt):
tgtAxisElt = rel.toModelObject
if isinstance(tgtAxisElt, ModelEuAxisCoord) and tgtAxisElt not in visited:
visited.add(tgtAxisElt)
newAxisElt = etree.SubElement(newLinkElt, "{http://xbrl.org/2011/table}ruleAxis")
copyAttrs(tgtAxisElt, newAxisElt, ("id",
"abstract",
"{http://www.w3.org/1999/xlink}type",
"{http://www.w3.org/1999/xlink}label"))
if tgtAxisElt.primaryItemQname:
newRuleElt = etree.SubElement(newAxisElt, "{http://xbrl.org/2008/formula}concept")
newQnameElt = etree.SubElement(newRuleElt, "{http://xbrl.org/2008/formula}qname")
newQnameElt.text = XmlUtil.addQnameValue(docObj, tgtAxisElt.primaryItemQname)
for dimQname, memQname in tgtAxisElt.explicitDims:
newRuleElt = etree.SubElement(newAxisElt, "{http://xbrl.org/2008/formula}explicitDimension")
newRuleElt.set("dimension", XmlUtil.addQnameValue(docObj, dimQname))
newMbrElt = etree.SubElement(newRuleElt, "{http://xbrl.org/2008/formula}member")
newQnameElt = etree.SubElement(newMbrElt, "{http://xbrl.org/2008/formula}qname")
newQnameElt.text = XmlUtil.addQnameValue(docObj, memQname)
newArcElt = etree.SubElement(newLinkElt, "{http://xbrl.org/2011/table}axisArc")
copyAttrs(rel, newArcElt, ("id",
"{http://www.w3.org/1999/xlink}type",
"{http://www.w3.org/1999/xlink}from",
"{http://www.w3.org/1999/xlink}to",
"order"))
newArcElt.set("{http://www.w3.org/1999/xlink}arcrole", XbrlConst.tableAxisSubtree)
generateAxis(newLinkElt, newAxisElt, tgtAxisElt, axisMbrRelSet, visited)
visited.discard(tgtAxisElt)
def _durationFunction(node, sphinxContext, args):
hasArg(node, sphinxContext, args, 1)
startArg = args[0]
if isinstance(startArg, str):
startDateTime = Period(None, XmlUtil.datetimeValue(startArg, none=NONE))
elif isinstance(startArg, datetime.datetime):
startDateTime = startArg
elif isinstance(arg, datetime.date):
startDateTime = datetime.date(startArg.year, startArg.month, startArg.day)
endArg = args[1]
if isinstance(endArg, str):
endDateTime = Period(None, XmlUtil.datetimeValue(startArg, addOneDay=True, none=NONE))
elif isinstance(endArg, datetime.datetime):
endDateTime = endArg
elif isinstance(endArg, datetime.date):
endDateTime = datetime.date(endArg.year, endArg.month, endArg.day) + datetime.timedelta(1)
if startDateTime and endDateTime:
return Period(startDateTime, endDateTime)
raise SphinxException(
node,
"sphinx.functionArgumentsMismatch",
_("Function %(name)s requires two argument that are a date or datetime string or value: %(start)s and %(end)s"),
name=node.name,
start=startArg,
end=endArg,
)
def pubDate(self):
try:
return self._pubDate
except AttributeError:
from arelle.UrlUtil import parseRfcDatetime
self._pubDate = parseRfcDatetime(XmlUtil.text(XmlUtil.descendant(self, None, "pubDate")))
return self._pubDate
def startDatetime(self):
"""(datetime) -- startDate attribute"""
try:
return self._startDatetime
except AttributeError:
self._startDatetime = XmlUtil.datetimeValue(XmlUtil.child(self.period, XbrlConst.xbrli, "startDate"))
return self._startDatetime
def instantDatetime(self):
"""(datetime) -- instant attribute, with adjustment to end-of-day midnight as needed"""
try:
return self._instantDatetime
except AttributeError:
self._instantDatetime = XmlUtil.datetimeValue(XmlUtil.child(self.period, XbrlConst.xbrli, "instant"), addOneDay=True)
return self._instantDatetime
def __init__(self, savedOptions=None, xbrlInstance=None):
self.entityIdentScheme = ""
self.entityIdentValue = ""
self.startDate = "" # use string values so structure can be json-saved
self.endDate = ""
self.monetaryUnit = ""
self.monetaryDecimals = ""
self.nonMonetaryDecimals = ""
if savedOptions is not None:
self.__dict__.update(savedOptions)
elif xbrlInstance is not None:
for fact in xbrlInstance.facts:
cntx = fact.context
unit = fact.unit
if fact.isItem and cntx is not None:
if not self.entityIdentScheme:
self.entityIdentScheme, self.entityIdentValue = cntx.entityIdentifier
if not self.startDate and cntx.isStartEndPeriod:
self.startDate = XmlUtil.dateunionValue(cntx.startDatetime)
if not self.startDate and (cntx.isStartEndPeriod or cntx.isInstantPeriod):
self.endDate = XmlUtil.dateunionValue(cntx.endDatetime, subtractOneDay=True)
if fact.isNumeric and unit is not None:
if fact.concept.isMonetary:
if not self.monetaryUnit and unit.measures[0] and unit.measures[0][0].namespaceURI == XbrlConst.iso4217:
self.monetaryUnit = unit.measures[0][0].localName
if not self.monetaryDecimals:
self.monetaryDecimals = fact.decimals
elif not self.nonMonetaryDecimals:
self.nonMonetaryDecimals = fact.decimals
if self.entityIdentScheme and self.startDate and self.monetaryUnit and self.monetaryDecimals and self.nonMonetaryDecimals:
break
def resultTableUri(self):
result = XmlUtil.descendant(self, None, "result")
if result is not None:
child = XmlUtil.child(result, None, "table")
if child is not None:
return os.path.join(self.modelDocument.outpath, XmlUtil.text(child))
return None
def cfcnTest(self):
# tuple of (expression, element holding the expression)
try:
return self._cfcnTest
except AttributeError:
self._cfcnTest = None
if self.localName == "test-case": # xpath testcase
outputFileElement = XmlUtil.descendant(self, None, "output-file")
if outputFileElement is not None and outputFileElement.get("compare") == "Text":
filepath = (
self.modelDocument.filepathdir
+ "/"
+ "ExpectedTestResults/"
+ self.get("FilePath")
+ outputFileElement.text
)
if os.sep != "/":
filepath = filepath.replace("/", os.sep)
with io.open(filepath, "rt", encoding="utf-8") as f:
self._cfcnTest = ("xs:string($result) eq '{0}'".format(f.read()), self)
else:
testElement = XmlUtil.descendant(self, XbrlConst.cfcn, "test")
if testElement is not None:
self._cfcnTest = (XmlUtil.innerText(testElement), testElement)
elif self.namespaceURI == "http://xbrl.org/2011/conformance-rendering/transforms":
output = self.get("output")
if output:
self._cfcnTest = ("$result eq '{0}'".format(output.replace("'", "''")), self)
return self._cfcnTest
def resultIsTable(self):
result = XmlUtil.descendant(self, None, "result")
if result is not None :
child = XmlUtil.child(result, None, "table")
if child is not None and XmlUtil.text(child).endswith(".xml"):
return True
return False
def cfcnCall(self):
# tuple of (expression, element holding the expression)
try:
return self._cfcnCall
except AttributeError:
self._cfcnCall = None
if self.localName == "test-case": # xpath testcase
queryElement = XmlUtil.descendant(self, None, "query")
if queryElement is not None:
filepath = (
self.modelDocument.filepathdir
+ "/"
+ "Queries/XQuery/"
+ self.get("FilePath")
+ queryElement.get("name")
+ ".xq"
)
if os.sep != "/":
filepath = filepath.replace("/", os.sep)
with io.open(filepath, "rt", encoding="utf-8") as f:
self._cfcnCall = (f.read(), self)
else:
for callElement in XmlUtil.descendants(self, XbrlConst.cfcn, "call"):
self._cfcnCall = (XmlUtil.innerText(callElement), callElement)
break
if self._cfcnCall is None and self.namespaceURI == "http://xbrl.org/2011/conformance-rendering/transforms":
name = self.getparent().get("name")
input = self.get("input")
if name and input:
self._cfcnCall = ("{0}('{1}')".format(name, input.replace("'", "''")), self)
return self._cfcnCall
def generateInstanceInfoset(dts, instanceInfosetFile):
if dts.fileSource.isArchive:
return
import os, io
from arelle import XmlUtil, XbrlConst
from arelle.ValidateXbrlCalcs import inferredPrecision, inferredDecimals
XmlUtil.setXmlns(dts.modelDocument, u"ptv", u"http://www.xbrl.org/2003/ptv")
numFacts = 0
for fact in dts.facts:
try:
if fact.concept.periodType:
fact.set(u"{http://www.xbrl.org/2003/ptv}periodType", fact.concept.periodType)
if fact.concept.balance:
fact.set(u"{http://www.xbrl.org/2003/ptv}balance", fact.concept.balance)
if fact.isNumeric and not fact.isNil:
fact.set(u"{http://www.xbrl.org/2003/ptv}decimals", unicode(inferredDecimals(fact)))
fact.set(u"{http://www.xbrl.org/2003/ptv}precision", unicode(inferredPrecision(fact)))
numFacts += 1
except Exception, err:
dts.error(u"saveInfoset.exception",
_(u"Facts exception %(fact)s %(value)s %(error)s."),
modelObject=fact, fact=fact.qname, value=fact.effectiveValue, error = err)
def close(self, noWrite=False):
if self.type == CSV:
if not isinstance(self.outfile, FileNamedStringIO):
self.csvFile.close()
elif not noWrite:
fileType = TYPENAMES[self.type]
try:
from arelle import XmlUtil
if isinstance(self.outfile, FileNamedStringIO):
fh = self.outfile
else:
fh = open(self.outfile, "w", encoding="utf-8")
if self.type == JSON:
fh.write(json.dumps(self.jsonObject, ensure_ascii=False))
else:
XmlUtil.writexml(fh, self.xmlDoc, encoding="utf-8",
xmlcharrefreplace= (self.type == HTML) )
if not isinstance(self.outfile, FileNamedStringIO):
fh.close()
self.modelXbrl.info("info", _("Saved output %(type)s to %(file)s"), file=self.outfile, type=fileType)
except (IOError, EnvironmentError) as err:
self.modelXbrl.exception("arelle:htmlIOError", _("Failed to save output %(type)s to %(file)s: \s%(error)s"), file=self.outfile, type=fileType, error=err)
self.modelXbrl = None
if self.type == HTML:
self.tblElt = None
elif self.type == XML:
self.docEltLevels = None
self.__dict__.clear() # dereference everything after closing document
def saveInstance(self, overrideFilepath=None):
"""Saves current instance document file.
:param overrideFilepath: specify to override saving in instance's modelDocument.filepath
"""
with open( (overrideFilepath or self.modelDocument.filepath), "w", encoding='utf-8') as fh:
XmlUtil.writexml(fh, self.modelDocument.xmlDocument, encoding="utf-8")
def stepAxis(self, op, p, sourceSequence):
targetSequence = []
for node in sourceSequence:
if not isinstance(node,(ModelObject.ModelObject, xml.dom.Node)):
raise XPathException(p, 'err:XPTY0020', _('Axis step {0} context item is not a node: {1}').format(op, node))
targetNodes = []
if isinstance(node, ModelObject.ModelObject): node = node.element
if isinstance(p,QNameDef):
ns = p.namespaceURI; localname = p.localName
if p.isAttribute:
if p.unprefixed:
if node.hasAttribute(localname):
targetNodes.append(node.getAttribute(localname))
else:
if node.hasAttributeNS(ns,localname):
targetNodes.append(node.getAttributeNS(ns,localname))
elif op == '/' or op is None:
targetNodes = XmlUtil.children(node, ns, localname)
elif op == '//':
targetNodes = XmlUtil.descendants(node, ns, localname)
elif op == '..':
targetNodes = [ XmlUtil.parent(node) ]
elif isinstance(p, OperationDef) and isinstance(p.name,QNameDef):
if p.name.localName == "text":
targetNodes = [XmlUtil.text(node)]
# todo: add element, attribute, node, etc...
targetSequence.extend(targetNodes)
return targetSequence
def __init__(self, mainWin, options):
self.mainWin = mainWin
parent = mainWin.parent
super(DialogNewFactItemOptions, self).__init__(parent)
self.parent = parent
self.options = options
parentGeometry = re.match("(\d+)x(\d+)[+]?([-]?\d+)[+]?([-]?\d+)", parent.geometry())
dialogX = int(parentGeometry.group(3))
dialogY = int(parentGeometry.group(4))
self.accepted = False
self.transient(self.parent)
self.title(_("New Fact Item Options"))
frame = Frame(self)
label(frame, 1, 1, "Entity scheme:")
self.cellEntityIdentScheme = gridCell(frame, 2, 1, getattr(options,"entityIdentScheme",""), width=50)
ToolTip(self.cellEntityIdentScheme, text=_("Enter the scheme for the context entity identifier"), wraplength=240)
label(frame, 1, 2, "Entity identifier:")
self.cellEntityIdentValue = gridCell(frame, 2, 2, getattr(options,"entityIdentValue",""))
ToolTip(self.cellEntityIdentValue, text=_("Enter the entity identifier value (e.g., stock ticker)"), wraplength=240)
label(frame, 1, 3, "Start date:")
startDate = getattr(options,"startDate",None)
self.cellStartDate = gridCell(frame, 2, 3, XmlUtil.dateunionValue(startDate) if startDate else "")
ToolTip(self.cellStartDate, text=_("Enter the start date for the report period (e.g., 2010-01-01)"), wraplength=240)
label(frame, 1, 4, "End date:")
endDate = getattr(options,"endDate",None)
self.cellEndDate = gridCell(frame, 2, 4, XmlUtil.dateunionValue(endDate, subtractOneDay=True) if endDate else "")
ToolTip(self.cellEndDate, text=_("Enter the end date for the report period (e.g., 2010-12-31)"), wraplength=240)
label(frame, 1, 5, "Monetary unit:")
self.cellMonetaryUnit = gridCombobox(frame, 2, 5, getattr(options,"monetaryUnit",""), values=monetaryUnits)
ToolTip(self.cellMonetaryUnit, text=_("Select a monetary unit (e.g., EUR)"), wraplength=240)
label(frame, 1, 6, "Monetary decimals:")
self.cellMonetaryDecimals = gridCell(frame, 2, 6, getattr(options,"monetaryDecimals","2"))
ToolTip(self.cellMonetaryDecimals, text=_("Enter decimals for monetary items"), wraplength=240)
label(frame, 1, 7, "Non-monetary decimals:")
self.cellNonMonetaryDecimals = gridCell(frame, 2, 7, getattr(options,"nonMonetaryDecimals","0"))
ToolTip(self.cellNonMonetaryDecimals, text=_("Enter decimals for non-monetary items (e.g., stock shares)"), wraplength=240)
cancelButton = Button(frame, text=_("Cancel"), width=8, command=self.close)
ToolTip(cancelButton, text=_("Cancel operation, discarding changes and entries"))
okButton = Button(frame, text=_("OK"), width=8, command=self.ok)
ToolTip(okButton, text=_("Accept the options as entered above"))
cancelButton.grid(row=8, column=1, columnspan=3, sticky=E, pady=3, padx=3)
okButton.grid(row=8, column=1, columnspan=3, sticky=E, pady=3, padx=86)
frame.grid(row=0, column=0, sticky=(N,S,E,W))
frame.columnconfigure(2, weight=1)
window = self.winfo_toplevel()
window.columnconfigure(0, weight=1)
self.geometry("+{0}+{1}".format(dialogX+50,dialogY+100))
#self.bind("<Return>", self.ok)
#self.bind("<Escape>", self.close)
self.protocol("WM_DELETE_WINDOW", self.close)
self.grab_set()
self.wait_window(self)
def write(self):
try:
from arelle import XmlUtil
with open(self.outfile, "w") as fh:
XmlUtil.writexml(fh, self.htmlDoc, encoding="utf-8")
self.modelXbrl.info("info", _("Saved output html to %(file)s"), file=self.outfile)
except (IOError, EnvironmentError) as err:
self.modelXbrl.exception("arelle:htmlIOError", _("Failed to save output html to %(file)s: \s%(error)s"), file=self.outfile, error=err)
def resultIsInfoset(self):
if self.modelDocument.outpath:
result = XmlUtil.descendant(self, None, "result")
if result is not None:
return XmlUtil.child(result, None,
"file") is not None or XmlUtil.text(
result).endswith(".xml")
return False
def resultInfosetUri(self):
result = XmlUtil.descendant(self, None, "result")
if result is not None:
child = XmlUtil.child(result, None, "file")
return os.path.join(
self.modelDocument.outpath,
XmlUtil.text(child if child is not None else result))
return None
def dataUris(self):
try:
return self._dataUris
except AttributeError:
self._dataUris = defaultdict(list) # may contain instances, schemas, linkbases
for dataElement in XmlUtil.descendants(self, None, ("data", "input")):
for elt in XmlUtil.descendants(dataElement, None, ("xsd", "schema", "linkbase", "instance")):
self._dataUris["schema" if elt.localName == "xsd" else elt.localName].append(elt.textValue.strip())
return self._dataUris
def aspectsCovered(self):
aspectsCovered = set()
if XmlUtil.hasChild(self, XbrlConst.euRend, "primaryItem"):
aspectsCovered.add(Aspect.CONCEPT)
if XmlUtil.hasChild(self, XbrlConst.euRend, "timeReference"):
aspectsCovered.add(Aspect.INSTANT)
for e in XmlUtil.children(self, XbrlConst.euRend, "explicitDimCoord"):
aspectsCovered.add(self.prefixedNameQname(e.get("dimension")))
return aspectsCovered
def __repr__(self):
return ("modelContext[{0}, period: {1}, {2}{3} line {4}]"
.format(self.id,
"forever" if self.isForeverPeriod else
"instant " + XmlUtil.dateunionValue(self.instantDatetime, subtractOneDay=True) if self.isInstantPeriod else
"duration " + XmlUtil.dateunionValue(self.startDatetime) + " - " + XmlUtil.dateunionValue(self.endDatetime, subtractOneDay=True),
"dimensions: ({0}) {1},".format(len(self.qnameDims),
tuple(mem.propertyView for dim,mem in sorted(self.qnameDims.items())))
if self.qnameDims else "",
self.modelDocument.basename, self.sourceline))
def propertyView(self):
scheme, entityId = self.entityIdentifier
return ((("entity", entityId, (("scheme", scheme),)),) +
((("forever", ""),) if self.isForeverPeriod else
(("instant", XmlUtil.dateunionValue(self.instantDatetime, subtractOneDay=True)),) if self.isInstantPeriod else
(("startDate", XmlUtil.dateunionValue(self.startDatetime)),("endDate", XmlUtil.dateunionValue(self.endDatetime, subtractOneDay=True)))) +
(("dimensions", "({0})".format(len(self.qnameDims)),
tuple(mem.propertyView for dim,mem in sorted(self.qnameDims.items())))
if self.qnameDims else (),
))
def evaluate(self,
exprStack,
contextItem=None,
resultStack=None,
parentOp=None):
if resultStack is None: resultStack = []
if contextItem is None: contextItem = self.contextItem
setProgHeader = False
for p in exprStack:
result = None
if isinstance(p, QNameDef) or (p == '*' and parentOp in ('/', '//')
): # path step QName or wildcard
# step axis operation
if len(resultStack) == 0 or not self.isNodeSequence(
resultStack[-1]):
resultStack.append([
contextItem,
])
result = self.stepAxis(parentOp, p, resultStack.pop())
elif isinstance(p, _STR_NUM_TYPES):
result = p
elif isinstance(p, VariableRef):
if p.name in self.inScopeVars:
result = self.inScopeVars[p.name]
# uncomment to allow lambdas as variable values (for deferred processing if needed)
#if isinstance(result, LambdaType):
# result = result() # dereference lambda-valued variables
if result is None: # None atomic result is XPath empty sequence
result = [
] # subsequent processing discards None results
elif isinstance(p, OperationDef):
op = p.name
if isinstance(op, QNameDef): # function call
args = self.evaluate(p.args, contextItem=contextItem)
ns = op.namespaceURI
localname = op.localName
try:
from arelle import (FunctionXs, FunctionFn,
FunctionXfi, FunctionIxt,
FunctionCustom)
if op in self.modelXbrl.modelCustomFunctionSignatures:
result = FunctionCustom.call(
self, p, op, contextItem, args)
elif op.unprefixed and localname in {
'attribute', 'comment', 'document-node',
'element', 'item', 'node',
'processing-instruction', 'schema-attribute',
'schema-element', 'text'
}:
# step axis operation
if len(resultStack
) == 0 or not self.isNodeSequence(
resultStack[-1]):
if isinstance(contextItem, (tuple, list)):
resultStack.append(contextItem)
else:
resultStack.append([
contextItem,
])
result = self.stepAxis(parentOp, p,
resultStack.pop())
elif op.unprefixed or ns == XbrlConst.fn:
result = FunctionFn.call(self, p, localname,
contextItem, args)
elif ns == XbrlConst.xfi or ns == XbrlConst.xff:
result = FunctionXfi.call(self, p, localname, args)
elif ns == XbrlConst.xsd:
result = FunctionXs.call(self, p, localname, args)
elif ns in FunctionIxt.ixtNamespaceFunctions:
result = FunctionIxt.call(self, p, op, args)
elif op in self.modelXbrl.modelManager.customTransforms:
result = self.modelXbrl.modelManager.customTransforms[
op](args[0][0])
else:
raise XPathException(
p, 'err:XPST0017',
_('Function call not identified: {0}.').format(
op))
except FunctionNumArgs as err:
raise XPathException(p, err.errCode,
"{}: {}".format(err.errText, op))
except FunctionArgType as err:
raise XPathException(
p, err.errCode,
_('Argument {0} does not match expected type {1} for {2} {3}.'
).format(err.argNum, err.expectedType, op,
err.foundObject))
except FunctionNotAvailable:
raise XPathException(
p, 'err:XPST0017',
_('Function named {0} does not have a custom or built-in implementation.'
).format(op))
elif op in VALUE_OPS:
# binary arithmetic operations and value comparisons
s1 = self.atomize(
p, resultStack.pop()) if len(resultStack) > 0 else []
s2 = self.atomize(
p, self.evaluate(p.args, contextItem=contextItem))
# value comparisons
if len(s1) > 1 or len(s2) > 1:
raise XPathException(
p, 'err:XPTY0004',
_("Value operation '{0}' sequence length error").
format(op))
if len(s1) == 0 or len(s2) == 0:
result = []
else:
op1 = s1[0]
op2 = s2[0]
from arelle.FunctionUtil import (testTypeCompatiblity)
testTypeCompatiblity(self, p, op, op1, op2)
if type(op1) != type(op2) and op in ('+', '-', '*',
'div', 'idiv',
'mod'):
# check if type promotion needed (Decimal-float, not needed for integer-Decimal)
if isinstance(op1, Decimal) and isinstance(
op2, float):
op1 = float(
op1
) # per http://http://www.w3.org/TR/xpath20/#dt-type-promotion 1b
elif isinstance(op2, Decimal) and isinstance(
op1, float):
op2 = float(op2)
if op == '+':
result = op1 + op2
elif op == '-':
result = op1 - op2
elif op == '*':
result = op1 * op2
elif op in ('div', 'idiv', "mod"):
try:
if op == 'div':
result = op1 / op2
elif op == 'idiv':
result = op1 // op2
elif op == 'mod':
result = op1 % op2
except ZeroDivisionError:
raise XPathException(
p, 'err:FOAR0001',
_('Attempt to divide by zero: {0} {1} {2}.'
).format(op1, op, op2))
elif op == 'ge':
result = op1 >= op2
elif op == 'gt':
result = op1 > op2
elif op == 'le':
result = op1 <= op2
elif op == 'lt':
result = op1 < op2
elif op == 'eq':
result = op1 == op2
elif op == 'ne':
result = op1 != op2
elif op == 'to':
result = _RANGE(_INT(op1), _INT(op2) + 1)
elif op in GENERALCOMPARISON_OPS:
# general comparisons
s1 = self.atomize(
p, resultStack.pop()) if len(resultStack) > 0 else []
s2 = self.atomize(
p, self.evaluate(p.args, contextItem=contextItem))
result = []
for op1 in s1:
for op2 in s2:
if op == '>=':
result = op1 >= op2
elif op == '>':
result = op1 > op2
elif op == '<=':
result = op1 <= op2
elif op == '<':
result = op1 < op2
elif op == '=':
result = op1 == op2
elif op == '!=':
result = op1 != op2
if result:
break
if result:
break
elif op in NODECOMPARISON_OPS:
# node comparisons
s1 = resultStack.pop() if len(resultStack) > 0 else []
s2 = self.evaluate(p.args, contextItem=contextItem)
if len(s1) > 1 or len(s2) > 1 or not self.isNodeSequence(
s1) or not self.isNodeSequence(s2[0]):
raise XPathException(
p, 'err:XPTY0004',
_('Node comparison sequence error'))
if len(s1) == 0 or len(s2[0]) == 0:
result = []
else:
n1 = s1[0]
n2 = s2[0][0]
result = False
for op1 in s1:
for op2 in s2:
if op == 'is':
result = n1 == n2
elif op == '>>':
result = op1 > op2
elif op == '<<':
result = op1 <= op2
if result:
break
elif op in COMBINING_OPS:
# node comparisons
s1 = resultStack.pop() if len(resultStack) > 0 else []
s2 = self.flattenSequence(
self.evaluate(p.args, contextItem=contextItem))
if not self.isNodeSequence(s1) or not self.isNodeSequence(
s2):
raise XPathException(
p, 'err:XPTY0004',
_('Node operation sequence error'))
set1 = set(s1)
set2 = set(s2)
if op == 'intersect':
resultset = set1 & set2
elif op == 'except':
resultset = set1 - set2
elif op == 'union' or op == '|':
resultset = set1 | set2
# convert to a list in document order
result = self.documentOrderedNodes(resultset)
elif op in LOGICAL_OPS:
# general comparisons
if len(resultStack) == 0:
result = []
else:
op1 = self.effectiveBooleanValue(p, resultStack.pop(
)) if len(resultStack) > 0 else False
op2 = self.effectiveBooleanValue(
p, self.evaluate(p.args, contextItem=contextItem))
result = False
if op == 'and':
result = op1 and op2
elif op == 'or':
result = op1 or op2
elif op in UNARY_OPS:
s1 = self.atomize(
p, self.evaluate(p.args, contextItem=contextItem))
if len(s1) > 1:
raise XPathException(
p, 'err:XPTY0004',
_('Unary expression sequence length error'))
if len(s1) == 0:
result = []
else:
op1 = s1[0]
if op == 'u+':
result = op1
elif op == 'u-':
result = -op1
elif op == 'instance':
result = False
s1 = self.flattenSequence(
resultStack.pop()) if len(resultStack) > 0 else []
arity = len(s1)
if len(p.args) > 1:
occurenceIndicator = p.args[1]
if (occurenceIndicator == '?' and arity in (0,1) ) or \
(occurenceIndicator == '+' and arity >= 1) or \
(occurenceIndicator == '*'):
result = True
elif arity == 1:
result = True
if result and len(p.args) > 0:
t = p.args[0]
for x in s1:
if isinstance(t, QNameDef):
if t.namespaceURI == XbrlConst.xsd:
tType = {
"integer": _INT_TYPES,
"string": _STR_BASE,
"decimal": Decimal,
"double": float,
"float": float,
"boolean": bool,
"QName": QName,
"anyURI": AnyURI,
"date": DateTime,
"dateTime": DateTime,
}.get(t.localName)
if tType:
result = isinstance(x, tType)
if result and tType == DateTime:
result = x.dateOnly == (
t.localName == "date")
elif isinstance(t, OperationDef):
if t.name == "element":
if isinstance(x, ModelObject):
if len(t.args) >= 1:
qn = t.args[0]
if qn == '*' or (isinstance(
qn, QNameDef) and qn == x):
result = True
if len(t.args
) >= 2 and isinstance(
t.args[1],
QNameDef):
modelXbrl = x.modelDocument.modelXbrl
modelConcept = modelXbrl.qnameConcepts.get(
qname(x))
if not modelConcept.instanceOfType(
t.args[1]):
result = False
else:
result = False
# elif t.name == "item" comes here and result stays True
if not result:
break
elif op == 'sequence':
result = self.evaluate(p.args, contextItem=contextItem)
elif op == 'predicate':
result = self.predicate(
p, resultStack.pop()) if len(resultStack) > 0 else []
elif op in FORSOMEEVERY_OPS: # for, some, every
result = []
self.evaluateRangeVars(op, p.args[0], p.args[1:],
contextItem, result)
elif op == 'if':
test = self.effectiveBooleanValue(
p,
self.evaluate(p.args[0].expr[0],
contextItem=contextItem))
result = self.evaluate(p.args[1 if test else 2].args,
contextItem=contextItem)
elif op == '.':
result = contextItem
elif op == '..':
result = XmlUtil.parent(contextItem)
elif op in PATH_OPS:
if op in ('rootChild', 'rootDescendant'):
# fix up for multi-instance
resultStack.append([
self.inputXbrlInstance.xmlDocument,
])
op = '/' if op == 'rootChild' else '//'
# contains QNameDefs and predicates
if len(resultStack) > 0:
innerFocusNodes = resultStack.pop()
else:
innerFocusNodes = contextItem
navSequence = []
for innerFocusNode in self.flattenSequence(
innerFocusNodes):
navSequence += self.evaluate(
p.args, contextItem=innerFocusNode, parentOp=op)
result = self.documentOrderedNodes(
self.flattenSequence(navSequence))
elif isinstance(p, ProgHeader):
self.progHeader = p
from arelle.ModelFormulaObject import Trace
if p.traceType not in (Trace.MESSAGE, Trace.CUSTOM_FUNCTION):
self.traceType = p.traceType
setProgHeader = True
if result is not None: # note: result can be False which gets appended to resultStack
resultStack.append(self.flattenSequence(result))
if setProgHeader:
self.progHeader = None
return resultStack
def generateUpdatedTableLB(dts, updatedTableLinkbaseFile):
import os, io
from arelle import XmlUtil, XbrlConst
from arelle.ViewUtil import viewReferences, referenceURI
from arelle.ModelRenderingObject import ModelEuAxisCoord
if dts.fileSource.isArchive:
dts.error(
"genTblLB:outFileIsArchive",
_("Updated Table Linkbase file cannot be an archive: %(tableLBOutputFile)s."
),
modelObject=dts,
tableLBOutputFile=updatedTableLinkbaseFile)
return
tblAxisRelSet = dts.relationshipSet(XbrlConst.euTableAxis)
axisMbrRelSet = dts.relationshipSet(XbrlConst.euAxisMember)
if len(tblAxisRelSet.modelRelationships) == 0:
dts.error(
"genTblLB:noInputTables",
_("DTS does not contain Eurofiling 2010 tables and axes: %(entryFile)s."
),
modelObject=dts,
entryFile=dts.uri)
return
file = io.StringIO('''
<nsmap>
<link:linkbase
xmlns:label="http://xbrl.org/2008/label"
xmlns:gen="http://xbrl.org/2008/generic"
xmlns:df="http://xbrl.org/2008/filter/dimension"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:reference="http://xbrl.org/2008/reference"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:table="http://xbrl.org/2011/table"
xmlns:formula="http://xbrl.org/2008/formula"
xsi:schemaLocation="
http://www.xbrl.org/2003/linkbase http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd
http://xbrl.org/2008/generic http://www.xbrl.org/2008/generic-link.xsd
http://xbrl.org/2008/reference http://www.xbrl.org/2008/generic-reference.xsd
http://xbrl.org/2008/label http://www.xbrl.org/2008/generic-label.xsd
http://xbrl.org/2011/table http://www.xbrl.org/2011/table.xsd
http://xbrl.org/2008/filter/dimension http://www.xbrl.org/2008/dimension-filter.xsd">
<link:arcroleRef arcroleURI="http://xbrl.org/arcrole/2011/table-filter" xlink:type="simple"
xlink:href="http://www.xbrl.org/2011/table.xsd#table-filter"/>
<link:arcroleRef arcroleURI="http://xbrl.org/arcrole/2011/table-axis" xlink:type="simple"
xlink:href="http://www.xbrl.org/2011/table.xsd#table-axis"/>
<link:arcroleRef arcroleURI="http://xbrl.org/arcrole/2011/axis-subtree" xlink:type="simple"
xlink:href="http://www.xbrl.org/2011/table.xsd#axis-subtree"/>
<link:arcroleRef arcroleURI="http://xbrl.org/arcrole/2011/axis-filter" xlink:type="simple"
xlink:href="http://www.xbrl.org/2011/filter-axis.xsd#axis-filter"/>
</link:linkbase>
</nsmap>
<!-- Generated by Arelle(r) http://arelle.org -->
''')
from arelle.ModelObjectFactory import parser
parser, parserLookupName, parserLookupClass = parser(dts, None)
from lxml import etree
xmlDocument = etree.parse(file,
parser=parser,
base_url=updatedTableLinkbaseFile)
file.close()
nsmapElt = xmlDocument.getroot()
#xmlDocument.getroot().init(self) ## is this needed ??
for lbElement in xmlDocument.iter(
tag="{http://www.xbrl.org/2003/linkbase}linkbase"):
break
class DocObj: # fake ModelDocument for namespaces
def __init__(self):
self.xmlRootElement = lbElement
self.xmlDocument = xmlDocument
docObj = DocObj()
numELRs = 0
numTables = 0
def copyAttrs(fromElt, toElt, attrTags):
for attr in attrTags:
if fromElt.get(attr):
toElt.set(attr, fromElt.get(attr))
def generateTable(newLinkElt, newTblElt, srcTblElt, tblAxisRelSet,
axisMbrRelSet, visited):
if srcTblElt is not None:
for rel in tblAxisRelSet.fromModelObject(srcTblElt):
srcAxisElt = rel.toModelObject
if isinstance(srcAxisElt, ModelEuAxisCoord):
visited.add(srcAxisElt)
newAxisElt = etree.SubElement(
newLinkElt, "{http://xbrl.org/2011/table}ruleAxis")
copyAttrs(srcAxisElt, newAxisElt,
("id", "{http://www.w3.org/1999/xlink}type",
"{http://www.w3.org/1999/xlink}label"))
newAxisElt.set("abstract",
"true") # always true on root element
newArcElt = etree.SubElement(
newLinkElt, "{http://xbrl.org/2011/table}axisArc")
copyAttrs(rel, newArcElt,
("id", "{http://www.w3.org/1999/xlink}type",
"{http://www.w3.org/1999/xlink}from",
"{http://www.w3.org/1999/xlink}to", "order"))
newArcElt.set("{http://www.w3.org/1999/xlink}arcrole",
XbrlConst.tableBreakdown)
newArcElt.set("axisDisposition", rel.axisDisposition)
generateAxis(newLinkElt, newAxisElt, srcAxisElt,
axisMbrRelSet, visited)
visited.discard(srcAxisElt)
def generateAxis(newLinkElt, newAxisParentElt, srcAxisElt, axisMbrRelSet,
visited):
for rel in axisMbrRelSet.fromModelObject(srcAxisElt):
tgtAxisElt = rel.toModelObject
if isinstance(tgtAxisElt,
ModelEuAxisCoord) and tgtAxisElt not in visited:
visited.add(tgtAxisElt)
newAxisElt = etree.SubElement(
newLinkElt, "{http://xbrl.org/2011/table}ruleAxis")
copyAttrs(
tgtAxisElt, newAxisElt,
("id", "abstract", "{http://www.w3.org/1999/xlink}type",
"{http://www.w3.org/1999/xlink}label"))
if tgtAxisElt.primaryItemQname:
newRuleElt = etree.SubElement(
newAxisElt, "{http://xbrl.org/2008/formula}concept")
newQnameElt = etree.SubElement(
newRuleElt, "{http://xbrl.org/2008/formula}qname")
newQnameElt.text = XmlUtil.addQnameValue(
docObj, tgtAxisElt.primaryItemQname)
for dimQname, memQname in tgtAxisElt.explicitDims:
newRuleElt = etree.SubElement(
newAxisElt,
"{http://xbrl.org/2008/formula}explicitDimension")
newRuleElt.set("dimension",
XmlUtil.addQnameValue(docObj, dimQname))
newMbrElt = etree.SubElement(
newRuleElt, "{http://xbrl.org/2008/formula}member")
newQnameElt = etree.SubElement(
newMbrElt, "{http://xbrl.org/2008/formula}qname")
newQnameElt.text = XmlUtil.addQnameValue(docObj, memQname)
newArcElt = etree.SubElement(
newLinkElt, "{http://xbrl.org/2011/table}axisArc")
copyAttrs(rel, newArcElt,
("id", "{http://www.w3.org/1999/xlink}type",
"{http://www.w3.org/1999/xlink}from",
"{http://www.w3.org/1999/xlink}to", "order"))
newArcElt.set("{http://www.w3.org/1999/xlink}arcrole",
XbrlConst.tableAxisSubtree)
generateAxis(newLinkElt, newAxisElt, tgtAxisElt, axisMbrRelSet,
visited)
visited.discard(tgtAxisElt)
# sort URIs
linkroleUris = sorted(
[linkroleUri for linkroleUri in tblAxisRelSet.linkRoleUris])
firstNewLinkElt = None
roleRefUris = set()
for linkroleUri in linkroleUris:
numELRs += 1
newLinkElt = etree.SubElement(lbElement,
"{http://xbrl.org/2008/generic}link")
newLinkElt.set("{http://www.w3.org/1999/xlink}type", "extended")
newLinkElt.set("{http://www.w3.org/1999/xlink}role", linkroleUri)
if firstNewLinkElt is None: firstNewLinkElt = newLinkElt
# To do: add roleRef if needed
tblAxisRelSet = dts.relationshipSet(XbrlConst.euTableAxis, linkroleUri)
axisMbrRelSet = dts.relationshipSet(XbrlConst.euAxisMember,
linkroleUri)
for srcTblElt in tblAxisRelSet.rootConcepts:
if srcTblElt.tag == "{http://www.eurofiling.info/2010/rendering}table":
numTables += 1
newTblElt = etree.SubElement(
newLinkElt, "{http://xbrl.org/2011/table}table")
newTblElt.set("aspectModel", "dimensional")
copyAttrs(srcTblElt, newTblElt,
("id", "{http://www.w3.org/1999/xlink}type",
"{http://www.w3.org/1999/xlink}label"))
generateTable(newLinkElt, newTblElt, srcTblElt, tblAxisRelSet,
axisMbrRelSet, set())
if linkroleUri not in roleRefUris:
srcRoleRefElt = XmlUtil.descendant(srcTblElt.getroottree(),
XbrlConst.link,
"roleRef", "roleURI",
linkroleUri)
if srcRoleRefElt is not None:
roleRefUris.add(linkroleUri)
newRoleRefElt = etree.Element(
"{http://www.xbrl.org/2003/linkbase}roleRef")
copyAttrs(
srcRoleRefElt, newRoleRefElt,
("roleURI", "{http://www.w3.org/1999/xlink}type",
"{http://www.w3.org/1999/xlink}href"))
firstNewLinkElt.addprevious(newRoleRefElt)
fh = open(updatedTableLinkbaseFile, "w", encoding="utf-8")
XmlUtil.writexml(fh, xmlDocument, encoding="utf-8")
fh.close()
dts.info(
"info:updateTableLinkbase",
_("Updated Table Linkbase of %(entryFile)s has %(numberOfLinkroles)s linkroles, %(numberOfTables)s tables in file %(tableLBOutputFile)s."
),
modelObject=dts,
entryFile=dts.uri,
numberOfLinkroles=numELRs,
numberOfTables=numTables,
tableLBOutputFile=updatedTableLinkbaseFile)
def load(modelXbrl,
uri,
base=None,
isEntry=False,
isIncluded=None,
namespace=None,
reloadCache=False):
normalizedUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
uri, base)
if isEntry:
modelXbrl.uri = normalizedUri
modelXbrl.uriDir = os.path.dirname(normalizedUri)
for i in range(modelXbrl.modelManager.disclosureSystem.
maxSubmissionSubdirectoryEntryNesting):
modelXbrl.uriDir = os.path.dirname(modelXbrl.uriDir)
if modelXbrl.modelManager.validateDisclosureSystem and \
not normalizedUri.startswith(modelXbrl.uriDir) and \
not modelXbrl.modelManager.disclosureSystem.hrefValid(normalizedUri):
blocked = modelXbrl.modelManager.disclosureSystem.blockDisallowedReferences
modelXbrl.error(
"Prohibited file for filings{1}: {0}".format(
normalizedUri,
_(" blocked") if blocked else ""), "err", "EFM.6.22.02",
"GFM.1.1.3", "SBR.NL.2.1.0.06")
if blocked:
return None
if normalizedUri in modelXbrl.modelManager.disclosureSystem.mappedFiles:
mappedUri = modelXbrl.modelManager.disclosureSystem.mappedFiles[
normalizedUri]
else: # handle mapped paths
mappedUri = normalizedUri
for mapFrom, mapTo in modelXbrl.modelManager.disclosureSystem.mappedPaths:
if normalizedUri.startswith(mapFrom):
mappedUri = mapTo + normalizedUri[len(mapFrom):]
break
if modelXbrl.fileSource.isInArchive(mappedUri):
filepath = mappedUri
else:
filepath = modelXbrl.modelManager.cntlr.webCache.getfilename(
mappedUri, reload=reloadCache)
if filepath:
uri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(filepath)
if filepath is None: # error such as HTTPerror is already logged
modelXbrl.error("File can not be loaded: {0}".format(mappedUri), "err",
"FileNotLoadable")
type = Type.Unknown
return None
modelDocument = modelXbrl.urlDocs.get(mappedUri)
if modelDocument:
return modelDocument
# load XML and determine type of model document
modelXbrl.modelManager.showStatus(_("parsing {0}").format(uri))
file = None
try:
if modelXbrl.modelManager.validateDisclosureSystem:
file = ValidateFilingText.checkfile(modelXbrl, filepath)
else:
file = modelXbrl.fileSource.file(filepath)
xmlDocument = xml.dom.minidom.parse(file)
file.close()
except EnvironmentError as err:
modelXbrl.error(
"{0}: file error: {1}".format(os.path.basename(uri), err), "err",
"IOerror")
type = Type.Unknown
if file:
file.close()
return None
except (
xml.parsers.expat.ExpatError, xml.dom.DOMException, ValueError
) as err: # ValueError raised on bad format of qnames, xmlns'es, or parameters
modelXbrl.error(
"{0}: import error: {1}".format(os.path.basename(uri), err), "err",
"XMLsyntax")
type = Type.Unknown
if file:
file.close()
return None
# identify document
#modelXbrl.modelManager.addToLog("discovery: {0}".format(
# os.path.basename(uri)))
modelXbrl.modelManager.showStatus(_("loading {0}").format(uri))
modelDocument = None
for rootNode in xmlDocument.childNodes:
if rootNode.nodeType == 1: #element
ln = rootNode.localName
ns = rootNode.namespaceURI
# type classification
if ns == XbrlConst.xsd and ln == "schema":
type = Type.SCHEMA
elif ns == XbrlConst.link:
if ln == "linkbase":
type = Type.LINKBASE
elif ln == "xbrl":
type = Type.INSTANCE
elif ns == XbrlConst.xbrli:
if ln == "xbrl":
type = Type.INSTANCE
elif ns == XbrlConst.xhtml and \
ln == "html" or ln == "xhtml":
type = Type.Unknown
for i in range(len(rootNode.attributes)):
if rootNode.attributes.item(i).value == XbrlConst.ixbrl:
type = Type.INLINEXBRL
break
XmlUtil.markIdAttributes(
rootNode) # required for minidom searchability
elif ln == "report" and ns == XbrlConst.ver:
type = Type.VERSIONINGREPORT
elif ln == "testcases" or ln == "documentation":
type = Type.TESTCASESINDEX
elif ln == "testcase":
type = Type.TESTCASE
elif ln == "registry" and ns == XbrlConst.registry:
type = Type.REGISTRY
elif ln == "rss":
type = Type.RSSFEED
else:
type = Type.Unknown
nestedInline = XmlUtil.descendant(rootNode, XbrlConst.xhtml,
("html", "xhtml"))
if nestedInline:
for i in range(len(nestedInline.attributes)):
if nestedInline.attributes.item(
i).value == XbrlConst.ixbrl:
type = Type.INLINEXBRL
rootNode = nestedInline
break
XmlUtil.markIdAttributes(
rootNode) # required for minidom searchability
#create modelDocument object or subtype as identified
if type == Type.VERSIONINGREPORT:
from arelle.ModelVersReport import ModelVersReport
modelDocument = ModelVersReport(modelXbrl, type, mappedUri,
filepath, xmlDocument)
elif type == Type.RSSFEED:
from arelle.ModelRssObject import ModelRssObject
modelDocument = ModelRssObject(modelXbrl, type, mappedUri,
filepath, xmlDocument)
else:
modelDocument = ModelDocument(modelXbrl, type, mappedUri,
filepath, xmlDocument)
modelDocument.xmlRootElement = rootNode
modelDocument.schemaLocationElements.add(rootNode)
if isEntry:
modelDocument.inDTS = True
# discovery (parsing)
if type == Type.SCHEMA:
modelDocument.schemaDiscover(rootNode, isIncluded, namespace)
elif type == Type.LINKBASE:
modelDocument.linkbaseDiscover(rootNode)
elif type == Type.INSTANCE:
modelDocument.instanceDiscover(rootNode)
elif type == Type.INLINEXBRL:
modelDocument.inlineXbrlDiscover(rootNode)
elif type == Type.VERSIONINGREPORT:
modelDocument.versioningReportDiscover(rootNode)
elif type == Type.TESTCASESINDEX:
modelDocument.testcasesIndexDiscover(xmlDocument)
elif type == Type.TESTCASE:
modelDocument.testcaseDiscover(rootNode)
elif type == Type.REGISTRY:
modelDocument.registryDiscover(rootNode)
elif type == Type.VERSIONINGREPORT:
modelDocument.versioningReportDiscover(rootNode)
elif type == Type.RSSFEED:
modelDocument.rssFeedDiscover(rootNode)
break
return modelDocument
def customAttributeQname(self, eventName):
custAttrElt = XmlUtil.child(self, XbrlConst.verce, eventName)
if custAttrElt is not None and custAttrElt.get("name"):
return qname(custAttrElt, custAttrElt.get("name"))
return None
def fromConceptQname(self):
fromConcept = XmlUtil.child(self, XbrlConst.vercb, "fromConcept")
if fromConcept is not None and fromConcept.get("name"):
return qname(fromConcept, fromConcept.get("name"))
else:
return None
def checkFilingDTS(val, modelDocument, isEFM, isGFM, visited):
global targetNamespaceDatePattern, efmFilenamePattern, htmlFileNamePattern, roleTypePattern, arcroleTypePattern, \
arcroleDefinitionPattern, namePattern, linkroleDefinitionBalanceIncomeSheet
if targetNamespaceDatePattern is None:
targetNamespaceDatePattern = re.compile(
r"/([12][0-9]{3})-([01][0-9])-([0-3][0-9])|"
r"/([12][0-9]{3})([01][0-9])([0-3][0-9])|")
efmFilenamePattern = re.compile(
r"^[a-z0-9][a-zA-Z0-9_\.\-]*(\.xsd|\.xml|\.htm)$")
htmlFileNamePattern = re.compile(
r"^[a-zA-Z0-9][._a-zA-Z0-9-]*(\.htm)$")
roleTypePattern = re.compile(r"^.*/role/[^/\s]+$")
arcroleTypePattern = re.compile(r"^.*/arcrole/[^/\s]+$")
arcroleDefinitionPattern = re.compile(
r"^.*[^\\s]+.*$") # at least one non-whitespace character
namePattern = re.compile(
"[][()*+?\\\\/^{}|@#%^=~`\"';:,<>&$\u00a3\u20ac]"
) # u20ac=Euro, u00a3=pound sterling
linkroleDefinitionBalanceIncomeSheet = re.compile(
r"[^-]+-\s+Statement\s+-\s+.*(income|balance|financial\W+position)",
re.IGNORECASE)
nonDomainItemNameProblemPattern = re.compile(
r"({0})|(FirstQuarter|SecondQuarter|ThirdQuarter|FourthQuarter|[1-4]Qtr|Qtr[1-4]|ytd|YTD|HalfYear)(?:$|[A-Z\W])"
.format(re.sub(r"\W", "", (val.entityRegistrantName or "").title())))
visited.append(modelDocument)
for referencedDocument, modelDocumentReference in modelDocument.referencesDocument.items(
):
#6.07.01 no includes
if modelDocumentReference.referenceType == "include":
val.modelXbrl.error(
("EFM.6.07.01", "GFM.1.03.01"),
_("Taxonomy schema %(schema)s includes %(include)s, only import is allowed"
),
modelObject=modelDocumentReference.referringModelObject,
schema=modelDocument.basename,
include=referencedDocument.basename)
if referencedDocument not in visited and referencedDocument.inDTS: # ignore EdgarRenderer added non-DTS documents
checkFilingDTS(val, referencedDocument, isEFM, isGFM, visited)
if val.disclosureSystem.standardTaxonomiesDict is None:
pass
if isEFM:
if modelDocument.uri in val.disclosureSystem.standardTaxonomiesDict:
if modelDocument.targetNamespace:
# check for duplicates of us-types, dei, and rr taxonomies
for pattern, indexGroup in ((usNamespacesConflictPattern, 2),
(ifrsNamespacesConflictPattern,
2)):
match = pattern.match(modelDocument.targetNamespace)
if match is not None:
conflictClass = match.group(indexGroup)
if (conflictClass == 'us-gaap' and match.group(3) <
'2018') or conflictClass == 'srt':
val.standardNamespaceConflicts['srt+us-gaap'].add(
modelDocument
) # ignore non-srt multi-usgaap in Filing.py
if conflictClass == 'us-gaap' or conflictClass == 'ifrs-full':
val.standardNamespaceConflicts['ifrs+us-gaap'].add(
modelDocument)
if conflictClass not in ('us-gaap', 'srt'):
val.standardNamespaceConflicts[conflictClass].add(
modelDocument)
else:
if len(modelDocument.basename) > 32:
val.modelXbrl.error(
"EFM.5.01.01.tooManyCharacters",
_("Document file name %(filename)s must not exceed 32 characters."
),
modelObject=modelDocument,
filename=modelDocument.basename)
if modelDocument.type == ModelDocument.Type.INLINEXBRL:
if not htmlFileNamePattern.match(modelDocument.basename):
val.modelXbrl.error(
"EFM.5.01.01",
_("Document file name %(filename)s must start with a-z or 0-9, contain upper or lower case letters, ., -, _, and end with .htm."
),
modelObject=modelDocument,
filename=modelDocument.basename)
elif not efmFilenamePattern.match(modelDocument.basename):
val.modelXbrl.error(
"EFM.5.01.01",
_("Document file name %(filename)s must start with a-z or 0-9, contain upper or lower case letters, ., -, _, and end with .xsd or .xml."
),
modelObject=modelDocument,
filename=modelDocument.basename)
if (modelDocument.type == ModelDocument.Type.SCHEMA
and modelDocument.targetNamespace
not in val.disclosureSystem.baseTaxonomyNamespaces
and modelDocument.uri.startswith(val.modelXbrl.uriDir)):
val.hasExtensionSchema = True
# check schema contents types
# 6.7.3 check namespace for standard authority
targetNamespaceAuthority = UrlUtil.authority(
modelDocument.targetNamespace)
if targetNamespaceAuthority in val.disclosureSystem.standardAuthorities:
val.modelXbrl.error(
("EFM.6.07.03", "GFM.1.03.03"),
_("The target namespace, %(targetNamespace)s cannot have the same authority (%(targetNamespaceAuthority)s) as a standard "
"taxonomy, in %(schema)s. Please change your target namespace."
),
edgarCode="fs-0703-Extension-Has-Standard-Namespace-Authority",
modelObject=modelDocument,
schema=modelDocument.basename,
targetNamespace=modelDocument.targetNamespace,
targetNamespaceAuthority=UrlUtil.authority(
modelDocument.targetNamespace, includeScheme=False))
# 6.7.4 check namespace format
if modelDocument.targetNamespace is None or not modelDocument.targetNamespace.startswith(
"http://"):
match = None
else:
targetNamespaceDate = modelDocument.targetNamespace[
len(targetNamespaceAuthority):]
match = targetNamespaceDatePattern.match(targetNamespaceDate)
if match is not None:
try:
if match.lastindex == 3:
date = datetime.date(int(match.group(1)),
int(match.group(2)),
int(match.group(3)))
elif match.lastindex == 6:
date = datetime.date(int(match.group(4)),
int(match.group(5)),
int(match.group(6)))
else:
match = None
except ValueError:
match = None
if match is None:
val.modelXbrl.error(
("EFM.6.07.04", "GFM.1.03.04"),
_("You did not adhere to the requirements for target namespace, for %(targetNamespace)s in %(schema)s. "
"Please recheck your submission and adhere to the target namespace requirements."
),
edgarCode="cp-0704-Taxonomy-Valid-Target-Namespace",
modelObject=modelDocument,
schema=modelDocument.basename,
targetNamespace=modelDocument.targetNamespace)
elif val.fileNameDate and date > val.fileNameDate:
val.modelXbrl.info(
("EFM.6.07.06", "GFM.1.03.06"),
_("Warning: Taxonomy schema %(schema)s namespace %(targetNamespace)s has date later than document name date %(docNameDate)s"
),
modelObject=modelDocument,
schema=modelDocument.basename,
targetNamespace=modelDocument.targetNamespace,
docNameDate=val.fileNameDate)
if modelDocument.targetNamespace is not None:
# 6.7.5 check prefix for _
authority = UrlUtil.authority(modelDocument.targetNamespace)
if not re.match(r"(http://|https://|ftp://|urn:)\w+", authority):
val.modelXbrl.error(
("EFM.6.07.05", "GFM.1.03.05"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s must be a valid URI with a valid authority for the namespace."
),
modelObject=modelDocument,
schema=modelDocument.basename,
targetNamespace=modelDocument.targetNamespace)
# may be multiple prefixes for namespace
prefixes = [
(prefix if prefix is not None else "")
for prefix, NS in modelDocument.xmlRootElement.nsmap.items()
if NS == modelDocument.targetNamespace
]
if not prefixes:
prefix = None
val.modelXbrl.error(
("EFM.6.07.07", "GFM.1.03.07"),
_("The schema does not supply a prefix for %(targetNamespace)s without an underscore character, in file %(schema)s. "
"Please provide or change a prefix"),
edgarCode="du-0707-Recommended-Prefix-Disallowed",
modelObject=modelDocument,
schema=modelDocument.basename,
targetNamespace=modelDocument.targetNamespace)
else:
prefix = prefixes[0]
if len(prefixes) > 1:
val.modelXbrl.error(
("EFM.6.07.07", "GFM.1.03.07"),
_("The schema does not supply a prefix for %(targetNamespace)s without an underscore character, in file %(schema)s. "
"Please provide or change a prefix"),
edgarCode="du-0707-Recommended-Prefix-Disallowed",
modelObject=modelDocument,
schema=modelDocument.basename,
targetNamespace=modelDocument.targetNamespace,
prefix=", ".join(prefixes))
elif "_" in prefix:
val.modelXbrl.error(
("EFM.6.07.07", "GFM.1.03.07"),
_("The schema does not supply a prefix for %(targetNamespace)s without an underscore character, in file %(schema)s. "
"Please provide or change a prefix"),
edgarCode="du-0707-Recommended-Prefix-Disallowed",
modelObject=modelDocument,
schema=modelDocument.basename,
targetNamespace=modelDocument.targetNamespace,
prefix=prefix)
for modelConcept in modelDocument.xmlRootElement.iterdescendants(
tag="{http://www.w3.org/2001/XMLSchema}element"):
if isinstance(modelConcept, ModelConcept):
# 6.7.16 name not duplicated in standard taxonomies
name = modelConcept.get("name")
if name is None:
name = ""
if modelConcept.get("ref") is not None:
continue # don't validate ref's here
for c in val.modelXbrl.nameConcepts.get(name, []):
if c.modelDocument != modelDocument:
if not c.modelDocument.uri.startswith(
val.modelXbrl.uriDir):
val.modelXbrl.error(
("EFM.6.07.16", "GFM.1.03.18"),
_("Your extension taxonomy contains an element, %(concept)s, which has the same name as an element in the base taxonomy, "
"%(standardConcept)s. Please ensure that this extension is appropriate and if so, please change the extension concept name."
),
edgarCode=
"cp-0716-Element-Name-Same-As-Base",
modelObject=(modelConcept, c),
concept=modelConcept.qname,
standardSchema=c.modelDocument.basename,
standardConcept=c.qname)
# 6.7.17 id properly formed
_id = modelConcept.id
requiredId = (prefix
if prefix is not None else "") + "_" + name
if _id != requiredId:
val.modelXbrl.error(
("EFM.6.07.17", "GFM.1.03.19"),
_("You did not adhere to the declarations for concepts by containing an 'id' attribute whose value begins with the recommended "
"namespace prefix of the taxonomy, followed by an underscore, followed by an element name, for the concept %(concept)s. "
"Please recheck your submission."),
edgarCode="cp-0717-Element-Id",
modelObject=modelConcept,
concept=modelConcept.qname,
id=_id,
requiredId=requiredId)
# 6.7.18 nillable is true
nillable = modelConcept.get("nillable")
if nillable != "true" and modelConcept.isItem:
val.modelXbrl.error(
("EFM.6.07.18", "GFM.1.03.20"),
_("Element %(concept)s is declared without a 'true' value for the nillable attribute. Please set the value to 'true'."
),
edgarCode="du-0718-Nillable-Not-True",
modelObject=modelConcept,
schema=modelDocument.basename,
concept=name,
nillable=nillable)
# 6.7.19 not tuple
if modelConcept.isTuple:
val.modelXbrl.error(
("EFM.6.07.19", "GFM.1.03.21"),
_("You provided an extension concept which is a tuple, %(concept)s. Please remove tuples and check your submission."
),
edgarCode="cp-0719-No-Tuple-Element",
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.7.20 no typed domain ref
if modelConcept.isTypedDimension:
val.modelXbrl.error(
("EFM.6.07.20", "GFM.1.03.22"),
_("There is an xbrldt:typedDomainRef attribute on %(concept)s (%(typedDomainRef)s). Please remove it."
),
edgarCode="du-0720-Typed-Domain-Ref-Disallowed",
modelObject=modelConcept,
concept=modelConcept.qname,
typedDomainRef=modelConcept.typedDomainElement.
qname if modelConcept.typedDomainElement
is not None else modelConcept.typedDomainRef)
# 6.7.21 abstract must be duration
isDuration = modelConcept.periodType == "duration"
if modelConcept.isAbstract and not isDuration:
val.modelXbrl.error(
("EFM.6.07.21", "GFM.1.03.23"),
_("Element %(concept)s is declared as an abstract item with period type 'instant'. Please change it to 'duration' or "
"make the element not abstract."),
edgarCode="du-0721-Abstract-Is-Instant",
modelObject=modelConcept,
schema=modelDocument.basename,
concept=modelConcept.qname)
# 6.7.22 abstract must be stringItemType
''' removed SEC EFM v.17, Edgar release 10.4, and GFM 2011-04-08
if modelConcept.abstract == "true" and modelConcept.typeQname != XbrlConst. qnXbrliStringItemType:
val.modelXbrl.error(("EFM.6.07.22", "GFM.1.03.24"),
_("Concept %(concept)s is abstract but type is not xbrli:stringItemType"),
modelObject=modelConcept, concept=modelConcept.qname)
'''
substitutionGroupQname = modelConcept.substitutionGroupQname
# 6.7.23 Axis must be subs group dimension
if name.endswith("Axis") ^ (
substitutionGroupQname
== XbrlConst.qnXbrldtDimensionItem):
val.modelXbrl.error(
("EFM.6.07.23", "GFM.1.03.25"),
_("The substitution group 'xbrldt:dimensionItem' is only consistent with an element name that ends with 'Axis'. "
"Please change %(conceptLocalName)s or change the substitutionGroup."
),
edgarCode="du-0723-Axis-Dimension-Name-Mismatch",
modelObject=modelConcept,
concept=modelConcept.qname,
conceptLocalName=modelConcept.qname.localName)
# 6.7.24 Table must be subs group hypercube
if name.endswith("Table") ^ (
substitutionGroupQname
== XbrlConst.qnXbrldtHypercubeItem):
val.modelXbrl.error(
("EFM.6.07.24", "GFM.1.03.26"),
_("The substitution group 'xbrldt:hypercubeItem' is only allowed with an element name that ends with 'Table'. "
"Please change %(conceptLocalName)s or change the substitutionGroup."
),
edgarCode="du-0724-Table-Hypercube-Name-Mismatch",
modelObject=modelConcept,
schema=modelDocument.basename,
concept=modelConcept.qname,
conceptLocalName=modelConcept.qname.localName)
# 6.7.25 if neither hypercube or dimension, substitution group must be item
if substitutionGroupQname not in (
None, XbrlConst.qnXbrldtDimensionItem,
XbrlConst.qnXbrldtHypercubeItem,
XbrlConst.qnXbrliItem):
val.modelXbrl.error(
("EFM.6.07.25", "GFM.1.03.27"),
_("The substitution group attribute value %(substitutionGroup)s of element %(conceptLocalName)s is not allowed. "
"Please change it to one of 'xbrli:item', 'xbrldt:dimensionItem' or 'xbrldt:hypercubeItem'."
),
edgarCode="du-0725-Substitution-Group-Custom",
modelObject=modelConcept,
concept=modelConcept.qname,
conceptLocalName=modelConcept.qname.localName,
substitutionGroup=modelConcept.
substitutionGroupQname)
# 6.7.26 Table must be subs group hypercube
if name.endswith(
"LineItems") and modelConcept.abstract != "true":
val.modelXbrl.error(
("EFM.6.07.26", "GFM.1.03.28"),
_("The element %(conceptLocalName)s ends with 'LineItems' but is not abstract. Please change %(conceptLocalName)s or "
"the value of the 'abstract' attribute."),
edgarCode=
"du-0726-LineItems-Abstract-Name-Mismatch",
modelObject=modelConcept,
concept=modelConcept.qname,
conceptLocalName=modelConcept.qname.localName)
# 6.7.27 type domainMember must end with Domain or Member
conceptType = modelConcept.type
isDomainItemType = conceptType is not None and conceptType.isDomainItemType
endsWithDomainOrMember = name.endswith(
"Domain") or name.endswith("Member")
if isDomainItemType != endsWithDomainOrMember:
val.modelXbrl.error(
("EFM.6.07.27", "GFM.1.03.29"),
_("The type 'us-types:domainItemType' is only allowed with an element name that ends with 'Domain' or 'Member'. "
"Please change %(conceptLocalName)s or change the type."
),
edgarCode="du-0727-Domain-Type-Name-Mismatch",
modelObject=modelConcept,
concept=modelConcept.qname,
conceptLocalName=modelConcept.qname.localName)
# 6.7.28 domainItemType must be duration
if isDomainItemType and not isDuration:
val.modelXbrl.error(
("EFM.6.07.28", "GFM.1.03.30"),
_("Element %(conceptLocalName)s is declared as a us-types:domainItemType with period type 'instant'. "
"Please change it to 'duration' or change the item type."
),
edgarCode="du-0728-Domain-Member-Is-Instant",
modelObject=modelConcept,
concept=modelConcept.qname,
conceptLocalName=modelConcept.qname.localName)
#6.7.31 (version 27) fractions
if modelConcept.isFraction:
val.modelXbrl.error(
"EFM.6.07.31",
_("Element %(concept)s is declared as a fraction item type. Change or remove the declaration."
),
edgarCode="du-0731-Fraction-Item-Type",
modelObject=modelConcept,
concept=modelConcept.qname)
#6.7.32 (version 27) instant non numeric
if modelConcept.isItem and (not modelConcept.isNumeric
and not isDuration
and not modelConcept.isAbstract
and not isDomainItemType):
val.modelXbrl.error(
"EFM.6.07.32",
_("Declaration of element %(concept)s in %(schema)s must have xbrli:periodType of 'duration' because its base type is not numeric."
),
edgarCode="rq-0732-Nonnnumeric-Must-Be-Duration",
modelObject=modelConcept,
schema=modelDocument.basename,
concept=modelConcept.qname)
# 6.8.5 semantic check, check LC3 name
if name:
if not name[0].isupper():
val.modelXbrl.log(
"ERROR-SEMANTIC", ("EFM.6.08.05.firstLetter",
"GFM.2.03.05.firstLetter"),
_("Concept %(concept)s name must start with a capital letter"
),
modelObject=modelConcept,
concept=modelConcept.qname)
if namePattern.search(name):
val.modelXbrl.log(
"ERROR-SEMANTIC",
("EFM.6.08.05.disallowedCharacter",
"GFM.2.03.05.disallowedCharacter"),
_("Concept %(concept)s has disallowed name character"
),
modelObject=modelConcept,
concept=modelConcept.qname)
if len(name) > 200:
val.modelXbrl.log(
"ERROR-SEMANTIC",
"EFM.6.08.05.nameLength",
_("Concept %(concept)s name length %(namelength)s exceeds 200 characters"
),
modelObject=modelConcept,
concept=modelConcept.qname,
namelength=len(name))
if isEFM:
label = modelConcept.label(lang="en-US",
fallbackToQname=False)
if label:
# allow Joe's Bar, N.A. to be JoesBarNA -- remove ', allow A. as not article "a"
lc3name = ''.join(
re.sub(r"['.-]", "", (
w[0] or w[2] or w[3] or w[4])).title()
for w in re.findall(
r"((\w+')+\w+)|(A[.-])|([.-]A(?=\W|$))|(\w+)",
label
) # EFM implies this should allow - and . re.findall(r"[\w\-\.]+", label)
if w[4].lower() not in ("the", "a", "an"))
if not (name == lc3name or
(name and lc3name and lc3name[0].isdigit()
and name[1:] == lc3name and
(name[0].isalpha() or name[0] == '_'))):
val.modelXbrl.log(
"WARNING-SEMANTIC",
"EFM.6.08.05.LC3",
_("Concept %(concept)s should match expected LC3 composition %(lc3name)s"
),
modelObject=modelConcept,
concept=modelConcept.qname,
lc3name=lc3name)
if conceptType is not None:
# 6.8.6 semantic check
if not isDomainItemType and conceptType.qname != XbrlConst.qnXbrliDurationItemType:
nameProblems = nonDomainItemNameProblemPattern.findall(
name)
if any(
any(t) for t in nameProblems
): # list of tuples with possibly nonempty strings
val.modelXbrl.log(
"WARNING-SEMANTIC",
("EFM.6.08.06", "GFM.2.03.06"),
_("Concept %(concept)s should not contain company or period information, found: %(matches)s"
),
modelObject=modelConcept,
concept=modelConcept.qname,
matches=", ".join(''.join(t)
for t in nameProblems))
if conceptType.qname == XbrlConst.qnXbrliMonetaryItemType:
if not modelConcept.balance:
# 6.8.11 may not appear on a income or balance statement
if any(
linkroleDefinitionBalanceIncomeSheet.
match(roleType.definition)
for rel in val.modelXbrl.
relationshipSet(XbrlConst.parentChild).
toModelObject(modelConcept)
for roleType in val.modelXbrl.
roleTypes.get(rel.linkrole, ())):
val.modelXbrl.log(
"ERROR-SEMANTIC",
("EFM.6.08.11", "GFM.2.03.11"),
_("Concept %(concept)s must have a balance because it appears in a statement of income or balance sheet"
),
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.11.5 semantic check, must have a documentation label
stdLabel = modelConcept.label(
lang="en-US", fallbackToQname=False)
defLabel = modelConcept.label(
preferredLabel=XbrlConst.
documentationLabel,
lang="en-US",
fallbackToQname=False)
if not defLabel or ( # want different words than std label
stdLabel
and re.findall(r"\w+", stdLabel)
== re.findall(r"\w+", defLabel)):
val.modelXbrl.log(
"ERROR-SEMANTIC",
("EFM.6.11.05", "GFM.2.04.04"),
_("Concept %(concept)s is monetary without a balance and must have a documentation label that disambiguates its sign"
),
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.8.16 semantic check
if conceptType.qname == XbrlConst.qnXbrliDateItemType and modelConcept.periodType != "duration":
val.modelXbrl.log(
"ERROR-SEMANTIC",
("EFM.6.08.16", "GFM.2.03.16"),
_("Concept %(concept)s of type xbrli:dateItemType must have periodType duration"
),
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.8.17 semantic check
if conceptType.qname == XbrlConst.qnXbrliStringItemType and modelConcept.periodType != "duration":
val.modelXbrl.log(
"ERROR-SEMANTIC",
("EFM.6.08.17", "GFM.2.03.17"),
_("Concept %(concept)s of type xbrli:stringItemType must have periodType duration"
),
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.7.8 check for embedded linkbase
for e in modelDocument.xmlRootElement.iterdescendants(
tag="{http://www.xbrl.org/2003/linkbase}linkbase"):
if isinstance(e, ModelObject):
val.modelXbrl.error(
("EFM.6.07.08", "GFM.1.03.08"),
_("Your filing contained embedded linkbases in %(schema)s. Please recheck your submission and remove all embedded linkbases."
),
edgarCode="cp-0708-No-Embedded-Linkbases",
modelObject=e,
schema=modelDocument.basename)
break
requiredUsedOns = {
XbrlConst.qnLinkPresentationLink, XbrlConst.qnLinkCalculationLink,
XbrlConst.qnLinkDefinitionLink
}
standardUsedOns = {
XbrlConst.qnLinkLabel,
XbrlConst.qnLinkReference,
XbrlConst.qnLinkDefinitionArc,
XbrlConst.qnLinkCalculationArc,
XbrlConst.qnLinkPresentationArc,
XbrlConst.qnLinkLabelArc,
XbrlConst.qnLinkReferenceArc,
# per WH, private footnote arc and footnore resource roles are not allowed
XbrlConst.qnLinkFootnoteArc,
XbrlConst.qnLinkFootnote,
}
# 6.7.9 role types authority
for e in modelDocument.xmlRootElement.iterdescendants(
tag="{http://www.xbrl.org/2003/linkbase}roleType"):
if isinstance(e, ModelObject):
roleURI = e.get("roleURI")
if targetNamespaceAuthority != UrlUtil.authority(roleURI):
val.modelXbrl.error(
("EFM.6.07.09", "GFM.1.03.09"),
_("Role %(roleType)s does not begin with %(targetNamespace)s's scheme and authority. "
"Please change the role URI or target namespace URI."
),
edgarCode="du-0709-Role-Namespace-Mismatch",
modelObject=e,
roleType=roleURI,
targetNamespaceAuthority=targetNamespaceAuthority,
targetNamespace=modelDocument.targetNamespace)
# 6.7.9 end with .../role/lc3 name
if not roleTypePattern.match(roleURI):
val.modelXbrl.warning(
("EFM.6.07.09.roleEnding", "GFM.1.03.09"),
"RoleType %(roleType)s should end with /role/{LC3name}",
modelObject=e,
roleType=roleURI)
# 6.7.10 only one role type declaration in DTS
modelRoleTypes = val.modelXbrl.roleTypes.get(roleURI)
if modelRoleTypes is not None:
modelRoleType = modelRoleTypes[0]
definition = modelRoleType.definitionNotStripped
usedOns = modelRoleType.usedOns
if len(modelRoleTypes) == 1:
# 6.7.11 used on's for pre, cal, def if any has a used on
if not usedOns.isdisjoint(requiredUsedOns) and len(
requiredUsedOns - usedOns) > 0:
val.modelXbrl.error(
("EFM.6.07.11", "GFM.1.03.11"),
_("The role %(roleType)s did not provide a usedOn element for all three link types (presentation, "
"calculation and definition), missing %(usedOn)s. Change the declaration to be for all three types of link, and resubmit."
),
edgarCode=
"du-0711-Role-Type-Declaration-Incomplete",
modelObject=e,
roleType=roleURI,
usedOn=requiredUsedOns - usedOns)
# 6.7.12 definition match pattern
if (val.disclosureSystem.roleDefinitionPattern
is not None and
(definition is None or not val.disclosureSystem.
roleDefinitionPattern.match(definition))):
val.modelXbrl.error(
("EFM.6.07.12", "GFM.1.03.12-14"),
_("The definition '%(definition)s' of role %(roleType)s does not match the expected format. "
"Please check that the definition matches {number} - {type} - {text}."
),
edgarCode="rq-0712-Role-Definition-Mismatch",
modelObject=e,
roleType=roleURI,
definition=(definition or ""))
if usedOns & standardUsedOns: # semantics check
val.modelXbrl.log(
"ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"),
_("RoleType %(roleuri)s is defined using role types already defined by standard roles for: %(qnames)s"
),
modelObject=e,
roleuri=roleURI,
qnames=', '.join(
str(qn) for qn in usedOns & standardUsedOns))
# 6.7.13 arcrole types authority
for e in modelDocument.xmlRootElement.iterdescendants(
tag="{http://www.xbrl.org/2003/linkbase}arcroleType"):
if isinstance(e, ModelObject):
arcroleURI = e.get("arcroleURI")
if targetNamespaceAuthority != UrlUtil.authority(arcroleURI):
val.modelXbrl.error(
("EFM.6.07.13", "GFM.1.03.15"),
_("Relationship role %(arcroleType)s does not begin with %(targetNamespace)s's scheme and authority. "
"Please change the relationship role URI or target namespace URI."
),
edgarCode="du-0713-Arcrole-Namespace-Mismatch",
modelObject=e,
arcroleType=arcroleURI,
targetNamespaceAuthority=targetNamespaceAuthority,
targetNamespace=modelDocument.targetNamespace)
# 6.7.13 end with .../arcrole/lc3 name
if not arcroleTypePattern.match(arcroleURI):
val.modelXbrl.warning(
("EFM.6.07.13.arcroleEnding", "GFM.1.03.15"),
_("ArcroleType %(arcroleType)s should end with /arcrole/{LC3name}"
),
modelObject=e,
arcroleType=arcroleURI)
# 6.7.15 definition match pattern
modelRoleTypes = val.modelXbrl.arcroleTypes[arcroleURI]
definition = modelRoleTypes[0].definition
if definition is None or not arcroleDefinitionPattern.match(
definition):
val.modelXbrl.error(
("EFM.6.07.15", "GFM.1.03.17"),
_("Relationship role declaration %(arcroleType)s is missing a definition. Please provide a definition."
),
edgarCode="du-0715-Arcrole-Definition-Missing",
modelObject=e,
arcroleType=arcroleURI)
# semantic checks
usedOns = modelRoleTypes[0].usedOns
if usedOns & standardUsedOns: # semantics check
val.modelXbrl.log(
"ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"),
_("ArcroleType %(arcroleuri)s is defined using role types already defined by standard arcroles for: %(qnames)s"
),
modelObject=e,
arcroleuri=arcroleURI,
qnames=', '.join(
str(qn) for qn in usedOns & standardUsedOns))
#6.3.3 filename check
m = re.match(r"^\w+-([12][0-9]{3}[01][0-9][0-3][0-9]).xsd$",
modelDocument.basename)
if m:
try: # check date value
datetime.datetime.strptime(m.group(1), "%Y%m%d").date()
# date and format are ok, check "should" part of 6.3.3
if val.fileNameBasePart:
expectedFilename = "{0}-{1}.xsd".format(
val.fileNameBasePart, val.fileNameDatePart)
if modelDocument.basename != expectedFilename:
val.modelXbrl.log(
"WARNING-SEMANTIC", ("EFM.6.03.03.matchInstance",
"GFM.1.01.01.matchInstance"),
_('Schema file name warning: %(filename)s, should match %(expectedFilename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
expectedFilename=expectedFilename)
except ValueError:
val.modelXbrl.error(
(val.EFM60303, "GFM.1.01.01"),
_('Invalid schema file base name part (date) in "{base}-{yyyymmdd}.xsd": %(filename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01"))
else:
val.modelXbrl.error(
(val.EFM60303, "GFM.1.01.01"),
_('Invalid schema file name, must match "{base}-{yyyymmdd}.xsd": %(filename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01"))
elif modelDocument.type == ModelDocument.Type.LINKBASE:
# if it is part of the submission (in same directory) check name
labelRels = None
if modelDocument.filepath.startswith(
val.modelXbrl.modelDocument.filepathdir):
#6.3.3 filename check
extLinkElt = XmlUtil.descendant(
modelDocument.xmlRootElement, XbrlConst.link, "*",
"{http://www.w3.org/1999/xlink}type", "extended")
if extLinkElt is None: # no ext link element
val.modelXbrl.error(
(val.EFM60303 + ".noLinkElement",
"GFM.1.01.01.noLinkElement"),
_('Invalid linkbase file name: %(filename)s, has no extended link element, cannot determine link type.'
),
modelObject=modelDocument,
filename=modelDocument.basename,
messageCodes=("EFM.6.03.03.noLinkElement",
"EFM.6.23.01.noLinkElement",
"GFM.1.01.01.noLinkElement"))
elif extLinkElt.localName not in extLinkEltFileNameEnding:
val.modelXbrl.error(
"EFM.6.03.02",
_('Invalid linkbase link element %(linkElement)s in %(filename)s'
),
modelObject=modelDocument,
linkElement=extLinkElt.localName,
filename=modelDocument.basename)
else:
m = re.match(
r"^\w+-([12][0-9]{3}[01][0-9][0-3][0-9])(_[a-z]{3}).xml$",
modelDocument.basename)
expectedSuffix = extLinkEltFileNameEnding[extLinkElt.localName]
if m and m.group(2) == expectedSuffix:
try: # check date value
datetime.datetime.strptime(m.group(1), "%Y%m%d").date()
# date and format are ok, check "should" part of 6.3.3
if val.fileNameBasePart:
expectedFilename = "{0}-{1}{2}.xml".format(
val.fileNameBasePart, val.fileNameDatePart,
expectedSuffix)
if modelDocument.basename != expectedFilename:
val.modelXbrl.log(
"WARNING-SEMANTIC",
("EFM.6.03.03.matchInstance",
"GFM.1.01.01.matchInstance"),
_('Linkbase name warning: %(filename)s should match %(expectedFilename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
expectedFilename=expectedFilename)
except ValueError:
val.modelXbrl.error(
(val.EFM60303, "GFM.1.01.01"),
_('Invalid linkbase base file name part (date) in "{base}-{yyyymmdd}_{suffix}.xml": %(filename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
messageCodes=("EFM.6.03.03", "EFM.6.23.01",
"GFM.1.01.01"))
else:
val.modelXbrl.error(
(val.EFM60303, "GFM.1.01.01"),
_('Invalid linkbase name, must match "{base}-{yyyymmdd}%(expectedSuffix)s.xml": %(filename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
expectedSuffix=expectedSuffix,
messageCodes=("EFM.6.03.03", "EFM.6.23.01",
"GFM.1.01.01"))
if extLinkElt.localName == "labelLink":
if labelRels is None:
labelRels = val.modelXbrl.relationshipSet(
XbrlConst.conceptLabel)
for labelElt in XmlUtil.children(extLinkElt,
XbrlConst.link, "label"):
# 6.10.9
if XbrlConst.isNumericRole(labelElt.role):
for rel in labelRels.toModelObject(labelElt):
if rel.fromModelObject is not None and not rel.fromModelObject.isNumeric:
val.modelXbrl.error(
"EFM.6.10.09",
_("Non-numeric element %(concept)s has a label role for numeric elements: %(role)s. "
"Please change the role attribute."),
edgarCode="du-1009-Numeric-Label-Role",
modelObject=(labelElt,
rel.fromModelObject),
concept=rel.fromModelObject.qname,
role=labelElt.role)
def relationshipSetElement(self):
return XmlUtil.child(self, XbrlConst.verrels, "relationshipSet")
def open(self, reloadCache=False):
if not self.isOpen:
if (self.isZip or self.isTarGz or self.isEis or self.isXfd or self.isRss or self.isInstalledTaxonomyPackage) and self.cntlr:
self.basefile = self.cntlr.webCache.getfilename(self.url, reload=reloadCache)
else:
self.basefile = self.url
self.baseurl = self.url # url gets changed by selection
if not self.basefile:
return # an error should have been logged
if self.isZip:
try:
self.fs = zipfile.ZipFile(openFileStream(self.cntlr, self.basefile, 'rb'), mode="r")
self.isOpen = True
except EnvironmentError as err:
self.logError(err)
pass
elif self.isTarGz:
try:
self.fs = tarfile.open(self.basefile, "r:gz")
self.isOpen = True
except EnvironmentError as err:
self.logError(err)
pass
elif self.isEis:
# check first line of file
buf = b''
try:
file = open(self.basefile, 'rb')
more = True
while more:
l = file.read(8)
if len(l) < 8:
break
if len(buf) == 0 and l.startswith(b"<?xml "): # not compressed
buf = l + file.read() # not compressed
break
compressedBytes = file.read( struct.unpack(">L", l[0:4])[0])
if len(compressedBytes) <= 0:
break
buf += zlib.decompress(compressedBytes)
file.close()
except EnvironmentError as err:
self.logError(err)
pass
#uncomment to save for debugging
#with open("c:/temp/test.xml", "wb") as f:
# f.write(buf)
if buf.startswith(b"<?xml "):
try:
# must strip encoding
str = buf.decode(XmlUtil.encoding(buf))
endEncoding = str.index("?>", 0, 128)
if endEncoding > 0:
str = str[endEncoding+2:]
file = io.StringIO(initial_value=str)
parser = etree.XMLParser(recover=True, huge_tree=True)
self.eisDocument = etree.parse(file, parser=parser)
file.close()
self.isOpen = True
except EnvironmentError as err:
self.logError(err)
return # provide error message later
except etree.LxmlError as err:
self.logError(err)
return # provide error message later
elif self.isXfd:
# check first line of file
file = open(self.basefile, 'rb')
firstline = file.readline()
if firstline.startswith(b"application/x-xfdl;content-encoding=\"asc-gzip\""):
# file has been gzipped
base64input = file.read(-1)
file.close();
file = None;
fb = base64.b64decode(base64input)
ungzippedBytes = b""
totalLenUncompr = 0
i = 0
while i < len(fb):
lenCompr = fb[i + 0] * 256 + fb[i + 1]
lenUncomp = fb[i + 2] * 256 + fb[i + 3]
lenRead = 0
totalLenUncompr += lenUncomp
gzchunk = (bytes((31,139,8,0)) + fb[i:i+lenCompr])
try:
with gzip.GzipFile(fileobj=io.BytesIO(gzchunk)) as gf:
while True:
readSize = min(16384, lenUncomp - lenRead)
readBytes = gf.read(size=readSize)
lenRead += len(readBytes)
ungzippedBytes += readBytes
if len(readBytes) == 0 or (lenUncomp - lenRead) <= 0:
break
except IOError as err:
pass # provide error message later
i += lenCompr + 4
#for learning the content of xfd file, uncomment this:
#with open("c:\\temp\\test.xml", "wb") as fh:
# fh.write(ungzippedBytes)
file = io.StringIO(initial_value=ungzippedBytes.decode("utf-8"))
else:
# position to start of file
file.seek(0,io.SEEK_SET)
try:
self.xfdDocument = etree.parse(file)
file.close()
self.isOpen = True
except EnvironmentError as err:
self.logError(err)
return # provide error message later
except etree.LxmlError as err:
self.logError(err)
return # provide error message later
elif self.isRss:
try:
self.rssDocument = etree.parse(self.basefile)
self.isOpen = True
except EnvironmentError as err:
self.logError(err)
return # provide error message later
except etree.LxmlError as err:
self.logError(err)
return # provide error message later
elif self.isInstalledTaxonomyPackage:
self.isOpen = True
# load mappings
self.loadTaxonomyPackageMappings()
def view(self):
self.blockSelectEvent = 1
self.blockViewModelObject = 0
self.tag_has = defaultdict(list) # temporary until Tk 8.6
# relationship set based on linkrole parameter, to determine applicable linkroles
relationshipSet = self.modelXbrl.relationshipSet(
self.arcrole, self.linkrole, self.linkqname, self.arcqname)
if not relationshipSet:
self.modelXbrl.modelManager.addToLog(
_("no relationships for {0}").format(self.arcrole))
return False
# consider facts in the relationshipSet (only)
contexts = set()
self.conceptFacts = defaultdict(list)
if self.linkrole and hasattr(
self.modelXbrl.roleTypes[self.linkrole][0], "_tableFacts"):
for fact in self.modelXbrl.roleTypes[self.linkrole][0]._tableFacts:
self.conceptFacts[fact.qname].append(fact)
if fact.context is not None:
contexts.add(fact.context)
else:
for fact in self.modelXbrl.facts:
if relationshipSet.fromModelObject(
fact.concept) or relationshipSet.toModelObject(
fact.concept):
self.conceptFacts[fact.qname].append(fact)
if fact.context is not None:
contexts.add(fact.context)
# sort contexts by period
self.periodContexts = defaultdict(set)
contextStartDatetimes = {}
ignoreDims = self.ignoreDims.get()
showDimDefaults = self.showDimDefaults.get()
for context in contexts:
if ignoreDims:
if context.isForeverPeriod:
contextkey = datetime.datetime(datetime.MINYEAR, 1, 1)
else:
contextkey = context.endDatetime
else:
if context.isForeverPeriod:
contextkey = "forever"
else:
contextkey = (
context.endDatetime -
datetime.timedelta(days=1)).strftime("%Y-%m-%d")
values = []
dims = context.qnameDims
if len(dims) > 0:
for dimQname in sorted(dims.keys(), key=lambda d: str(d)):
dimvalue = dims[dimQname]
if dimvalue.isExplicit:
values.append(
dimvalue.member.
label(self.labelrole, lang=self.lang
) if dimvalue.member is not None else
str(dimvalue.memberQname))
else:
values.append(
XmlUtil.innerText(dimvalue.typedMember))
nonDimensions = context.nonDimValues(
"segment") + context.nonDimValues("scenario")
if len(nonDimensions) > 0:
for element in sorted(nonDimensions,
key=lambda e: e.localName):
values.append(XmlUtil.innerText(element))
if len(values) > 0:
contextkey += " - " + ', '.join(values)
objectId = context.objectId()
self.periodContexts[contextkey].add(objectId)
if context.isStartEndPeriod:
contextStartDatetimes[objectId] = context.startDatetime
self.periodKeys = list(self.periodContexts.keys())
self.periodKeys.sort()
# set up treeView widget and tabbed pane
self.treeView.column("#0", width=300, anchor="w")
self.treeView.heading("#0", text="Concept")
columnIds = []
columnIdHeadings = []
self.contextColId = {}
self.startdatetimeColId = {}
self.numCols = 1
for periodKey in self.periodKeys:
colId = "#{0}".format(self.numCols)
columnIds.append(colId)
columnIdHeadings.append((colId, periodKey))
for contextId in self.periodContexts[periodKey]:
self.contextColId[contextId] = colId
if contextId in contextStartDatetimes:
self.startdatetimeColId[
contextStartDatetimes[contextId]] = colId
self.numCols += 1
self.treeView["columns"] = columnIds
for colId, colHeading in columnIdHeadings:
self.treeView.column(colId, width=100, anchor="w")
if ignoreDims:
if colHeading.year == datetime.MINYEAR:
date = "forever"
else:
date = (colHeading -
datetime.timedelta(days=1)).strftime("%Y-%m-%d")
self.treeView.heading(colId, text=date)
else:
self.treeView.heading(colId, text=colHeading)
# fact rendering
self.clearTreeView()
self.rowColFactId = {}
# root node for tree view
self.id = 1
# sort URIs by definition
linkroleUris = []
relationshipSet = self.modelXbrl.relationshipSet(
self.arcrole, self.linkrole, self.linkqname, self.arcqname)
if self.linkrole:
roleType = self.modelXbrl.roleTypes[self.linkrole][0]
linkroleUris.append(
((roleType.genLabel(lang=self.lang, strip=True)
or roleType.definition
or linkroleUri), self.linkrole, roleType.objectId(self.id)))
self.id += 1
else:
for linkroleUri in relationshipSet.linkRoleUris:
modelRoleTypes = self.modelXbrl.roleTypes.get(linkroleUri)
if modelRoleTypes:
roledefinition = (modelRoleTypes[0].genLabel(
lang=self.lang, strip=True)
or modelRoleTypes[0].definition
or linkroleUri)
roleId = modelRoleTypes[0].objectId(self.id)
else:
roledefinition = linkroleUri
roleId = "node{0}".format(self.id)
self.id += 1
linkroleUris.append((roledefinition, linkroleUri, roleId))
linkroleUris.sort()
# for each URI in definition order
for linkroleUriTuple in linkroleUris:
linknode = self.treeView.insert("",
"end",
linkroleUriTuple[2],
text=linkroleUriTuple[0],
tags=("ELR", ))
linkRelationshipSet = self.modelXbrl.relationshipSet(
self.arcrole, linkroleUriTuple[1], self.linkqname,
self.arcqname)
for rootConcept in linkRelationshipSet.rootConcepts:
node = self.viewConcept(rootConcept, rootConcept, "",
self.labelrole, linknode, 1,
linkRelationshipSet, set())
if self.expandAllOnFirstDisplay:
self.expandAll()
return True
def file(self, filepath, binary=False, stripDeclaration=False, encoding=None):
'''
for text, return a tuple of (open file handle, encoding)
for binary, return a tuple of (open file handle, )
'''
archiveFileSource = self.fileSourceContainingFilepath(filepath)
if archiveFileSource is not None:
if filepath.startswith(archiveFileSource.basefile):
archiveFileName = filepath[len(archiveFileSource.basefile) + 1:]
else: # filepath.startswith(self.baseurl)
archiveFileName = filepath[len(archiveFileSource.baseurl) + 1:]
if archiveFileSource.isZip:
try:
if archiveFileSource.isZipBackslashed:
f = archiveFileName.replace("/", "\\")
else:
f = archiveFileName.replace("\\","/")
b = archiveFileSource.fs.read(f)
if binary:
return (io.BytesIO(b), )
if encoding is None:
encoding = XmlUtil.encoding(b)
if stripDeclaration:
b = stripDeclarationBytes(b)
return (FileNamedTextIOWrapper(filepath, io.BytesIO(b), encoding=encoding),
encoding)
except KeyError:
raise ArchiveFileIOError(self, errno.ENOENT, archiveFileName)
elif archiveFileSource.isTarGz:
try:
fh = archiveFileSource.fs.extractfile(archiveFileName)
b = fh.read()
fh.close() # doesn't seem to close properly using a with construct
if binary:
return (io.BytesIO(b), )
if encoding is None:
encoding = XmlUtil.encoding(b)
if stripDeclaration:
b = stripDeclarationBytes(b)
return (FileNamedTextIOWrapper(filepath, io.BytesIO(b), encoding=encoding),
encoding)
except KeyError:
raise ArchiveFileIOError(self, archiveFileName)
elif archiveFileSource.isEis:
for docElt in self.eisDocument.iter(tag="{http://www.sec.gov/edgar/common}document"):
outfn = docElt.findtext("{http://www.sec.gov/edgar/common}conformedName")
if outfn == archiveFileName:
b64data = docElt.findtext("{http://www.sec.gov/edgar/common}contents")
if b64data:
b = base64.b64decode(b64data.encode("latin-1"))
# remove BOM codes if present
if len(b) > 3 and b[0] == 239 and b[1] == 187 and b[2] == 191:
start = 3;
length = len(b) - 3;
b = b[start:start + length]
else:
start = 0;
length = len(b);
if binary:
return (io.BytesIO(b), )
if encoding is None:
encoding = XmlUtil.encoding(b, default="latin-1")
return (io.TextIOWrapper(io.BytesIO(b), encoding=encoding),
encoding)
raise ArchiveFileIOError(self, errno.ENOENT, archiveFileName)
elif archiveFileSource.isXfd:
for data in archiveFileSource.xfdDocument.iter(tag="data"):
outfn = data.findtext("filename")
if outfn == archiveFileName:
b64data = data.findtext("mimedata")
if b64data:
b = base64.b64decode(b64data.encode("latin-1"))
# remove BOM codes if present
if len(b) > 3 and b[0] == 239 and b[1] == 187 and b[2] == 191:
start = 3;
length = len(b) - 3;
b = b[start:start + length]
else:
start = 0;
length = len(b);
if binary:
return (io.BytesIO(b), )
if encoding is None:
encoding = XmlUtil.encoding(b, default="latin-1")
return (io.TextIOWrapper(io.BytesIO(b), encoding=encoding),
encoding)
raise ArchiveFileIOError(self, errno.ENOENT, archiveFileName)
elif archiveFileSource.isInstalledTaxonomyPackage:
# remove TAXONOMY_PACKAGE_FILE_NAME from file path
if filepath.startswith(archiveFileSource.basefile):
l = len(archiveFileSource.basefile)
for f in TAXONOMY_PACKAGE_FILE_NAMES:
if filepath[l - len(f):l] == f:
filepath = filepath[0:l - len(f) - 1] + filepath[l:]
break
for pluginMethod in pluginClassMethods("FileSource.File"): #custom overrides for decription, etc
fileResult = pluginMethod(self.cntlr, filepath, binary, stripDeclaration)
if fileResult is not None:
return fileResult
if binary:
return (openFileStream(self.cntlr, filepath, 'rb'), )
elif encoding:
return (openFileStream(self.cntlr, filepath, 'rt', encoding=encoding), )
else:
return openXmlFileStream(self.cntlr, filepath, stripDeclaration)
def dir(self):
self.open()
if not self.isOpen:
return None
elif self.filesDir is not None:
return self.filesDir
elif self.isZip:
files = []
for zipinfo in self.fs.infolist():
f = zipinfo.filename
if '\\' in f:
self.isZipBackslashed = True
f = f.replace("\\", "/")
files.append(f)
self.filesDir = files
elif self.isTarGz:
self.filesDir = self.fs.getnames()
elif self.isEis:
files = []
for docElt in self.eisDocument.iter(tag="{http://www.sec.gov/edgar/common}document"):
outfn = docElt.findtext("{http://www.sec.gov/edgar/common}conformedName")
if outfn:
files.append(outfn);
self.filesDir = files
elif self.isXfd:
files = []
for data in self.xfdDocument.iter(tag="data"):
outfn = data.findtext("filename")
if outfn:
if len(outfn) > 2 and outfn[0].isalpha() and \
outfn[1] == ':' and outfn[2] == '\\':
continue
files.append(outfn);
self.filesDir = files
elif self.isRss:
files = [] # return title, descr, pubdate, linst doc
edgr = "http://www.sec.gov/Archives/edgar"
try:
for dsElt in XmlUtil.descendants(self.rssDocument, None, "item"):
instDoc = None
for instDocElt in XmlUtil.descendants(dsElt, edgr, "xbrlFile"):
if instDocElt.get("(http://www.sec.gov/Archives/edgar}description").endswith("INSTANCE DOCUMENT"):
instDoc = instDocElt.get("(http://www.sec.gov/Archives/edgar}url")
break
if not instDoc:
continue
files.append((
XmlUtil.text(XmlUtil.descendant(dsElt, None, "title")),
# tooltip
"{0}\n {1}\n {2}\n {3}\n {4}".format(
XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "companyName")),
XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "formType")),
XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "filingDate")),
XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "cikNumber")),
XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "period"))),
XmlUtil.text(XmlUtil.descendant(dsElt, None, "description")),
XmlUtil.text(XmlUtil.descendant(dsElt, None, "pubDate")),
instDoc))
self.filesDir = files
except (EnvironmentError,
etree.LxmlError) as err:
pass
elif self.isInstalledTaxonomyPackage:
files = []
baseurlPathLen = len(os.path.dirname(self.baseurl)) + 1
def packageDirsFiles(dir):
for file in os.listdir(dir):
path = dir + "/" + file # must have / and not \\ even on windows
files.append(path[baseurlPathLen:])
if os.path.isdir(path):
packageDirsFiles(path)
packageDirsFiles(self.baseurl[0:baseurlPathLen - 1])
self.filesDir = files
return self.filesDir
def resultIsXbrlInstance(self):
return XmlUtil.descendant(XmlUtil.descendant(self, None, "result"), None, "instance") is not None
def checkDTS(val, modelDocument, visited):
global targetNamespaceDatePattern, efmFilenamePattern, roleTypePattern, arcroleTypePattern, \
arcroleDefinitionPattern, namePattern, linkroleDefinitionBalanceIncomeSheet, \
namespacesConflictPattern
if targetNamespaceDatePattern is None:
targetNamespaceDatePattern = re.compile(r"/([12][0-9]{3})-([01][0-9])-([0-3][0-9])|"
r"/([12][0-9]{3})([01][0-9])([0-3][0-9])|")
efmFilenamePattern = re.compile(r"^[a-z0-9][a-zA-Z0-9_\.\-]*(\.xsd|\.xml)$")
roleTypePattern = re.compile(r"^.*/role/[^/\s]+$")
arcroleTypePattern = re.compile(r"^.*/arcrole/[^/\s]+$")
arcroleDefinitionPattern = re.compile(r"^.*[^\\s]+.*$") # at least one non-whitespace character
namePattern = re.compile("[][()*+?\\\\/^{}|@#%^=~`\"';:,<>&$\u00a3\u20ac]") # u20ac=Euro, u00a3=pound sterling
linkroleDefinitionBalanceIncomeSheet = re.compile(r"[^-]+-\s+Statement\s+-\s+.*(income|balance|financial\W+position)",
re.IGNORECASE)
namespacesConflictPattern = re.compile(r"http://(xbrl\.us|fasb\.org|xbrl\.sec\.gov)/(dei|us-types|us-roles|rr)/([0-9]{4}-[0-9]{2}-[0-9]{2})$")
nonDomainItemNameProblemPattern = re.compile(
r"({0})|(FirstQuarter|SecondQuarter|ThirdQuarter|FourthQuarter|[1-4]Qtr|Qtr[1-4]|ytd|YTD|HalfYear)(?:$|[A-Z\W])"
.format(re.sub(r"\W", "", (val.entityRegistrantName or "").title())))
visited.append(modelDocument)
definesLabelLinkbase = False
for referencedDocument, modelDocumentReference in modelDocument.referencesDocument.items():
#6.07.01 no includes
if modelDocumentReference.referenceType == "include":
val.modelXbrl.error(("EFM.6.07.01", "GFM.1.03.01", "SBR.NL.2.2.0.18"),
_("Taxonomy schema %(schema)s includes %(include)s, only import is allowed"),
modelObject=modelDocumentReference.referringModelObject,
schema=os.path.basename(modelDocument.uri),
include=os.path.basename(referencedDocument.uri))
if referencedDocument not in visited:
checkDTS(val, referencedDocument, visited)
if val.disclosureSystem.standardTaxonomiesDict is None:
pass
if val.validateEFM:
if modelDocument.uri in val.disclosureSystem.standardTaxonomiesDict:
if modelDocument.targetNamespace:
# check for duplicates of us-types, dei, and rr taxonomies
match = namespacesConflictPattern.match(modelDocument.targetNamespace)
if match is not None:
val.standardNamespaceConflicts[match.group(2)].add(modelDocument)
else:
if len(modelDocument.basename) > 32:
val.modelXbrl.error("EFM.5.01.01.tooManyCharacters",
_("Document file name %(filename)s must not exceed 32 characters."),
modelObject=modelDocument, filename=modelDocument.basename)
if not efmFilenamePattern.match(modelDocument.basename):
val.modelXbrl.error("EFM.5.01.01",
_("Document file name %(filename)s must start with a-z or 0-9, contain upper or lower case letters, ., -, _, and end with .xsd or .xml."),
modelObject=modelDocument, filename=modelDocument.basename)
if (modelDocument.type == ModelDocument.Type.SCHEMA and
modelDocument.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces and
modelDocument.uri.startswith(val.modelXbrl.uriDir)):
# check schema contents types
if val.validateSBRNL:
definesLinkroles = False
definesArcroles = False
definesLinkParts = False
definesAbstractItems = False
definesNonabstractItems = False
definesConcepts = False
definesTuples = False
definesPresentationTuples = False
definesSpecificationTuples = False
definesTypes = False
definesEnumerations = False
definesDimensions = False
definesDomains = False
definesHypercubes = False
# 6.7.3 check namespace for standard authority
targetNamespaceAuthority = UrlUtil.authority(modelDocument.targetNamespace)
if targetNamespaceAuthority in val.disclosureSystem.standardAuthorities:
val.modelXbrl.error(("EFM.6.07.03", "GFM.1.03.03"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s is a disallowed authority"),
modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace,
targetNamespaceAuthority=UrlUtil.authority(modelDocument.targetNamespace, includeScheme=False))
# 6.7.4 check namespace format
if modelDocument.targetNamespace is None or not modelDocument.targetNamespace.startswith("http://"):
match = None
elif val.validateEFMorGFM:
targetNamespaceDate = modelDocument.targetNamespace[len(targetNamespaceAuthority):]
match = targetNamespaceDatePattern.match(targetNamespaceDate)
else:
match = None
if match is not None:
try:
if match.lastindex == 3:
date = datetime.date(int(match.group(1)),int(match.group(2)),int(match.group(3)))
elif match.lastindex == 6:
date = datetime.date(int(match.group(4)),int(match.group(5)),int(match.group(6)))
else:
match = None
except ValueError:
match = None
if match is None:
val.modelXbrl.error(("EFM.6.07.04", "GFM.1.03.04"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s must have format http://{authority}/{versionDate}"),
modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace)
elif val.fileNameDate and date > val.fileNameDate:
val.modelXbrl.info(("EFM.6.07.06", "GFM.1.03.06"),
_("Warning: Taxonomy schema %(schema)s namespace %(targetNamespace)s has date later than document name date %(docNameDate)s"),
modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace,
docNameDate=val.fileNameDate)
if modelDocument.targetNamespace is not None:
# 6.7.5 check prefix for _
authority = UrlUtil.authority(modelDocument.targetNamespace)
if not re.match(r"(http://|https://|ftp://|urn:)\w+",authority):
val.modelXbrl.error(("EFM.6.07.05", "GFM.1.03.05"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s must be a valid URL with a valid authority for the namespace."),
modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace)
prefix = XmlUtil.xmlnsprefix(modelDocument.xmlRootElement,modelDocument.targetNamespace)
if not prefix:
val.modelXbrl.error(("EFM.6.07.07", "GFM.1.03.07"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s missing prefix for the namespace."),
modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace)
elif "_" in prefix:
val.modelXbrl.error(("EFM.6.07.07", "GFM.1.03.07"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s prefix %(prefix)s must not have an '_'"),
modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace, prefix=prefix)
if val.validateSBRNL:
genrlSpeclRelSet = val.modelXbrl.relationshipSet(XbrlConst.generalSpecial)
for modelConcept in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/2001/XMLSchema}element"):
if isinstance(modelConcept,ModelConcept):
# 6.7.16 name not duplicated in standard taxonomies
name = modelConcept.get("name")
if name is None:
name = ""
if modelConcept.get("ref") is not None:
continue # don't validate ref's here
for c in val.modelXbrl.nameConcepts.get(name, []):
if c.modelDocument != modelDocument:
if (val.validateEFMorGFM and
not c.modelDocument.uri.startswith(val.modelXbrl.uriDir)):
val.modelXbrl.error(("EFM.6.07.16", "GFM.1.03.18"),
_("Concept %(concept)s is also defined in standard taxonomy schema schema %(standardSchema)s"),
modelObject=(modelConcept,c), concept=modelConcept.qname, standardSchema=os.path.basename(c.modelDocument.uri), standardConcept=c.qname)
elif val.validateSBRNL:
if not (genrlSpeclRelSet.isRelated(modelConcept, "child", c) or genrlSpeclRelSet.isRelated(c, "child", modelConcept)):
val.modelXbrl.error("SBR.NL.2.2.2.02",
_("Concept %(concept)s is also defined in standard taxonomy schema %(standardSchema)s without a general-special relationship"),
modelObject=c, concept=modelConcept.qname, standardSchema=os.path.basename(c.modelDocument.uri))
''' removed RH 2011-12-23 corresponding set up of table in ValidateFiling
if val.validateSBRNL and name in val.nameWordsTable:
if not any( any( genrlSpeclRelSet.isRelated(c, "child", modelConcept)
for c in val.modelXbrl.nameConcepts.get(partialWordName, []))
for partialWordName in val.nameWordsTable[name]):
val.modelXbrl.error("SBR.NL.2.3.2.01",
_("Concept %(specialName)s is appears to be missing a general-special relationship to %(generalNames)s"),
modelObject=c, specialName=modelConcept.qname, generalNames=', or to '.join(val.nameWordsTable[name]))
'''
# 6.7.17 id properly formed
id = modelConcept.id
requiredId = (prefix if prefix is not None else "") + "_" + name
if val.validateEFMorGFM and id != requiredId:
val.modelXbrl.error(("EFM.6.07.17", "GFM.1.03.19"),
_("Concept %(concept)s id %(id)s should be $(requiredId)s"),
modelObject=modelConcept, concept=modelConcept.qname, id=id, requiredId=requiredId)
# 6.7.18 nillable is true
nillable = modelConcept.get("nillable")
if nillable != "true":
val.modelXbrl.error(("EFM.6.07.18", "GFM.1.03.20"),
_("Taxonomy schema %(schema)s element %(concept)s nillable %(nillable)s should be 'true'"),
modelObject=modelConcept, schema=os.path.basename(modelDocument.uri),
concept=name, nillable=nillable)
# 6.7.19 not tuple
if modelConcept.isTuple:
if val.validateEFMorGFM:
val.modelXbrl.error(("EFM.6.07.19", "GFM.1.03.21"),
_("Concept %(concept)s is a tuple"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.7.20 no typed domain ref
if modelConcept.isTypedDimension:
val.modelXbrl.error(("EFM.6.07.20", "GFM.1.03.22"),
_("Concept %(concept)s has typedDomainRef %(typedDomainRef)s"),
modelObject=modelConcept, concept=modelConcept.qname,
typedDomainRef=modelConcept.typedDomainElement.qname if modelConcept.typedDomainElement is not None else modelConcept.typedDomainRef)
# 6.7.21 abstract must be duration
isDuration = modelConcept.periodType == "duration"
if modelConcept.abstract == "true" and not isDuration:
val.modelXbrl.error(("EFM.6.07.21", "GFM.1.03.23"),
_("Taxonomy schema %(schema)s element %(concept)s is abstract but period type is not duration"),
modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=modelConcept.qname)
# 6.7.22 abstract must be stringItemType
''' removed SEC EFM v.17, Edgar release 10.4, and GFM 2011-04-08
if modelConcept.abstract == "true" and modelConcept.typeQname != XbrlConst. qnXbrliStringItemType:
val.modelXbrl.error(("EFM.6.07.22", "GFM.1.03.24"),
_("Concept %(concept)s is abstract but type is not xbrli:stringItemType"),
modelObject=modelConcept, concept=modelConcept.qname)
'''
substititutionGroupQname = modelConcept.substitutionGroupQname
# 6.7.23 Axis must be subs group dimension
if name.endswith("Axis") ^ (substititutionGroupQname == XbrlConst.qnXbrldtDimensionItem):
val.modelXbrl.error(("EFM.6.07.23", "GFM.1.03.25"),
_("Concept %(concept)s must end in Axis to be in xbrldt:dimensionItem substitution group"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.7.24 Table must be subs group hypercube
if name.endswith("Table") ^ (substititutionGroupQname == XbrlConst.qnXbrldtHypercubeItem):
val.modelXbrl.error(("EFM.6.07.24", "GFM.1.03.26"),
_("Concept %(concept)s must end in Table to be in xbrldt:hypercubeItem substitution group"),
modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=modelConcept.qname)
# 6.7.25 if neither hypercube or dimension, substitution group must be item
if substititutionGroupQname not in (None,
XbrlConst.qnXbrldtDimensionItem,
XbrlConst.qnXbrldtHypercubeItem,
XbrlConst.qnXbrliItem):
val.modelXbrl.error(("EFM.6.07.25", "GFM.1.03.27"),
_("Concept %(concept)s has disallowed substitution group %(substitutionGroup)s"),
modelObject=modelConcept, concept=modelConcept.qname,
substitutionGroup=modelConcept.substitutionGroupQname)
# 6.7.26 Table must be subs group hypercube
if name.endswith("LineItems") and modelConcept.abstract != "true":
val.modelXbrl.error(("EFM.6.07.26", "GFM.1.03.28"),
_("Concept %(concept)s is a LineItems but not abstract"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.7.27 type domainMember must end with Domain or Member
conceptType = modelConcept.type
isDomainItemType = conceptType is not None and conceptType.isDomainItemType
endsWithDomainOrMember = name.endswith("Domain") or name.endswith("Member")
if isDomainItemType != endsWithDomainOrMember:
val.modelXbrl.error(("EFM.6.07.27", "GFM.1.03.29"),
_("Concept %(concept)s must end with Domain or Member for type of domainItemType"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.7.28 domainItemType must be duration
if isDomainItemType and not isDuration:
val.modelXbrl.error(("EFM.6.07.28", "GFM.1.03.30"),
_("Concept %(concept)s is a domainItemType and must be periodType duration"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.8.5 semantic check, check LC3 name
if not name[0].isupper():
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.05.firstLetter", "GFM.2.03.05.firstLetter"),
_("Concept %(concept)s name must start with a capital letter"),
modelObject=modelConcept, concept=modelConcept.qname)
if namePattern.search(name):
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.05.disallowedCharacter", "GFM.2.03.05.disallowedCharacter"),
_("Concept %(concept)s has disallowed name character"),
modelObject=modelConcept, concept=modelConcept.qname)
if len(name) > 200:
val.modelXbrl.log("ERROR-SEMANTIC", "EFM.6.08.05.nameLength",
_("Concept %(concept)s name length %(namelength)s exceeds 200 characters"),
modelObject=modelConcept, concept=modelConcept.qname, namelength=len(name))
if val.validateEFM:
label = modelConcept.label(lang="en-US", fallbackToQname=False)
if label:
# allow Joe's Bar, N.A. to be JoesBarNA -- remove ', allow A. as not article "a"
lc3name = ''.join(re.sub(r"['.-]", "", (w[0] or w[2] or w[3] or w[4])).title()
for w in re.findall(r"((\w+')+\w+)|(A[.-])|([.-]A(?=\W|$))|(\w+)", label) # EFM implies this should allow - and . re.findall(r"[\w\-\.]+", label)
if w[4].lower() not in ("the", "a", "an"))
if not(name == lc3name or
(name and lc3name and lc3name[0].isdigit() and name[1:] == lc3name and (name[0].isalpha() or name[0] == '_'))):
val.modelXbrl.log("WARNING-SEMANTIC", "EFM.6.08.05.LC3",
_("Concept %(concept)s should match expected LC3 composition %(lc3name)s"),
modelObject=modelConcept, concept=modelConcept.qname, lc3name=lc3name)
if conceptType is not None:
# 6.8.6 semantic check
if not isDomainItemType and conceptType.qname != XbrlConst.qnXbrliDurationItemType:
nameProblems = nonDomainItemNameProblemPattern.findall(name)
if any(any(t) for t in nameProblems): # list of tuples with possibly nonempty strings
val.modelXbrl.log("WARNING-SEMANTIC", ("EFM.6.08.06", "GFM.2.03.06"),
_("Concept %(concept)s should not contain company or period information, found: %(matches)s"),
modelObject=modelConcept, concept=modelConcept.qname,
matches=", ".join(''.join(t) for t in nameProblems))
if conceptType.qname == XbrlConst.qnXbrliMonetaryItemType:
if not modelConcept.balance:
# 6.8.11 may not appear on a income or balance statement
if any(linkroleDefinitionBalanceIncomeSheet.match(roleType.definition)
for rel in val.modelXbrl.relationshipSet(XbrlConst.parentChild).toModelObject(modelConcept)
for roleType in val.modelXbrl.roleTypes.get(rel.linkrole,())):
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.11", "GFM.2.03.11"),
_("Concept %(concept)s must have a balance because it appears in a statement of income or balance sheet"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.11.5 semantic check, must have a documentation label
stdLabel = modelConcept.label(lang="en-US", fallbackToQname=False)
defLabel = modelConcept.label(preferredLabel=XbrlConst.documentationLabel, lang="en-US", fallbackToQname=False)
if not defLabel or ( # want different words than std label
stdLabel and re.findall(r"\w+", stdLabel) == re.findall(r"\w+", defLabel)):
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.11.05", "GFM.2.04.04"),
_("Concept %(concept)s is monetary without a balance and must have a documentation label that disambiguates its sign"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.8.16 semantic check
if conceptType.qname == XbrlConst.qnXbrliDateItemType and modelConcept.periodType != "duration":
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.16", "GFM.2.03.16"),
_("Concept %(concept)s of type xbrli:dateItemType must have periodType duration"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.8.17 semantic check
if conceptType.qname == XbrlConst.qnXbrliStringItemType and modelConcept.periodType != "duration":
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.17", "GFM.2.03.17"),
_("Concept %(concept)s of type xbrli:stringItemType must have periodType duration"),
modelObject=modelConcept, concept=modelConcept.qname)
if val.validateSBRNL:
if modelConcept.isTuple:
if modelConcept.substitutionGroupQname.localName == "presentationTuple" and modelConcept.substitutionGroupQname.namespaceURI.endswith("/basis/sbr/xbrl/xbrl-syntax-extension"): # namespace may change each year
definesPresentationTuples = True
elif modelConcept.substitutionGroupQname.localName == "specificationTuple" and modelConcept.substitutionGroupQname.namespaceURI.endswith("/basis/sbr/xbrl/xbrl-syntax-extension"): # namespace may change each year
definesSpecificationTuples = True
else:
definesTuples = True
definesConcepts = True
if modelConcept.isAbstract:
val.modelXbrl.error("SBR.NL.2.2.2.03",
_("Concept %(concept)s is an abstract tuple"),
modelObject=modelConcept, concept=modelConcept.qname)
if tupleCycle(val,modelConcept):
val.modelXbrl.error("SBR.NL.2.2.2.07",
_("Tuple %(concept)s has a tuple cycle"),
modelObject=modelConcept, concept=modelConcept.qname)
if modelConcept.get("nillable") != "false" and modelConcept.isRoot:
val.modelXbrl.error("SBR.NL.2.2.2.17", #don't want default, just what was really there
_("Tuple %(concept)s must have nillable='false'"),
modelObject=modelConcept, concept=modelConcept.qname)
elif modelConcept.isItem:
definesConcepts = True
if modelConcept.abstract == "true":
if modelConcept.isRoot:
if modelConcept.get("nillable") != "false": #don't want default, just what was really there
val.modelXbrl.error("SBR.NL.2.2.2.16",
_("Abstract root concept %(concept)s must have nillable='false'"),
modelObject=modelConcept, concept=modelConcept.qname)
if modelConcept.typeQname != XbrlConst.qnXbrliStringItemType:
val.modelXbrl.error("SBR.NL.2.2.2.21",
_("Abstract root concept %(concept)s must have type='xbrli:stringItemType'"),
modelObject=modelConcept, concept=modelConcept.qname)
if modelConcept.balance:
val.modelXbrl.error("SBR.NL.2.2.2.22",
_("Abstract concept %(concept)s must not have a balance attribute"),
modelObject=modelConcept, concept=modelConcept.qname)
if modelConcept.isHypercubeItem:
definesHypercubes = True
elif modelConcept.isDimensionItem:
definesDimensions = True
elif substititutionGroupQname and substititutionGroupQname.localName in ("domainItem","domainMemberItem"):
definesDomains = True
elif modelConcept.isItem:
definesAbstractItems = True
else: # not abstract
if modelConcept.isItem:
definesNonabstractItems = True
if not (modelConcept.label(preferredLabel=XbrlConst.documentationLabel,fallbackToQname=False,lang="nl") or
val.modelXbrl.relationshipSet(XbrlConst.conceptReference).fromModelObject(c) or
modelConcept.genLabel(role=XbrlConst.genDocumentationLabel,lang="nl") or
val.modelXbrl.relationshipSet(XbrlConst.elementReference).fromModelObject(c)):
val.modelXbrl.error("SBR.NL.2.2.2.28",
_("Concept %(concept)s must have a documentation label or reference"),
modelObject=modelConcept, concept=modelConcept.qname)
if modelConcept.balance and not modelConcept.instanceOfType(XbrlConst.qnXbrliMonetaryItemType):
val.modelXbrl.error("SBR.NL.2.2.2.24",
_("Non-monetary concept %(concept)s must not have a balance attribute"),
modelObject=modelConcept, concept=modelConcept.qname)
if modelConcept.isLinkPart:
definesLinkParts = True
val.modelXbrl.error("SBR.NL.2.2.5.01",
_("Link:part concept %(concept)s is not allowed"),
modelObject=modelConcept, concept=modelConcept.qname)
if not modelConcept.genLabel(fallbackToQname=False,lang="nl"):
val.modelXbrl.error("SBR.NL.2.2.5.02",
_("Link part definition %(concept)s must have a generic label in language 'nl'"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.7.8 check for embedded linkbase
for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}linkbase"):
if isinstance(e,ModelObject):
val.modelXbrl.error(("EFM.6.07.08", "GFM.1.03.08"),
_("Taxonomy schema %(schema)s contains an embedded linkbase"),
modelObject=e, schema=modelDocument.basename)
break
requiredUsedOns = {XbrlConst.qnLinkPresentationLink,
XbrlConst.qnLinkCalculationLink,
XbrlConst.qnLinkDefinitionLink}
standardUsedOns = {XbrlConst.qnLinkLabel, XbrlConst.qnLinkReference,
XbrlConst.qnLinkDefinitionArc, XbrlConst.qnLinkCalculationArc, XbrlConst.qnLinkPresentationArc,
XbrlConst.qnLinkLabelArc, XbrlConst.qnLinkReferenceArc,
# per WH, private footnote arc and footnore resource roles are not allowed
XbrlConst.qnLinkFootnoteArc, XbrlConst.qnLinkFootnote,
}
# 6.7.9 role types authority
for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}roleType"):
if isinstance(e,ModelObject):
roleURI = e.get("roleURI")
if targetNamespaceAuthority != UrlUtil.authority(roleURI):
val.modelXbrl.error(("EFM.6.07.09", "GFM.1.03.09"),
_("RoleType %(roleType)s does not match authority %(targetNamespaceAuthority)s"),
modelObject=e, roleType=roleURI, targetNamespaceAuthority=targetNamespaceAuthority, targetNamespace=modelDocument.targetNamespace)
# 6.7.9 end with .../role/lc3 name
if not roleTypePattern.match(roleURI):
val.modelXbrl.warning(("EFM.6.07.09.roleEnding", "GFM.1.03.09"),
"RoleType %(roleType)s should end with /role/{LC3name}",
modelObject=e, roleType=roleURI)
# 6.7.10 only one role type declaration in DTS
modelRoleTypes = val.modelXbrl.roleTypes.get(roleURI)
if modelRoleTypes is not None:
modelRoleType = modelRoleTypes[0]
definition = modelRoleType.definitionNotStripped
usedOns = modelRoleType.usedOns
if len(modelRoleTypes) == 1:
# 6.7.11 used on's for pre, cal, def if any has a used on
if not usedOns.isdisjoint(requiredUsedOns) and len(requiredUsedOns - usedOns) > 0:
val.modelXbrl.error(("EFM.6.07.11", "GFM.1.03.11"),
_("RoleType %(roleType)s missing used on %(usedOn)s"),
modelObject=e, roleType=roleURI, usedOn=requiredUsedOns - usedOns)
# 6.7.12 definition match pattern
if (val.disclosureSystem.roleDefinitionPattern is not None and
(definition is None or not val.disclosureSystem.roleDefinitionPattern.match(definition))):
val.modelXbrl.error(("EFM.6.07.12", "GFM.1.03.12-14"),
_("RoleType %(roleType)s definition \"%(definition)s\" must match {Sortcode} - {Type} - {Title}"),
modelObject=e, roleType=roleURI, definition=(definition or ""))
if usedOns & standardUsedOns: # semantics check
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"),
_("RoleType %(roleuri)s is defined using role types already defined by standard roles for: %(qnames)s"),
modelObject=e, roleuri=roleURI, qnames=', '.join(str(qn) for qn in usedOns & standardUsedOns))
if val.validateSBRNL:
if usedOns & XbrlConst.standardExtLinkQnames or XbrlConst.qnGenLink in usedOns:
definesLinkroles = True
if not e.genLabel():
val.modelXbrl.error("SBR.NL.2.2.3.03",
_("Link RoleType %(roleType)s missing a generic standard label"),
modelObject=e, roleType=roleURI)
nlLabel = e.genLabel(lang="nl")
if definition != nlLabel:
val.modelXbrl.error("SBR.NL.2.2.3.04",
_("Link RoleType %(roleType)s definition does not match NL standard generic label, \ndefinition: %(definition)s \nNL label: %(label)s"),
modelObject=e, roleType=roleURI, definition=definition, label=nlLabel)
if definition and (definition[0].isspace() or definition[-1].isspace()):
val.modelXbrl.error("SBR.NL.2.2.3.07",
_('Link RoleType %(roleType)s definition has leading or trailing spaces: "%(definition)s"'),
modelObject=e, roleType=roleURI, definition=definition)
# 6.7.13 arcrole types authority
for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}arcroleType"):
if isinstance(e,ModelObject):
arcroleURI = e.get("arcroleURI")
if targetNamespaceAuthority != UrlUtil.authority(arcroleURI):
val.modelXbrl.error(("EFM.6.07.13", "GFM.1.03.15"),
_("ArcroleType %(arcroleType)s does not match authority %(targetNamespaceAuthority)s"),
modelObject=e, arcroleType=arcroleURI, targetNamespaceAuthority=targetNamespaceAuthority, targetNamespace=modelDocument.targetNamespace)
# 6.7.13 end with .../arcrole/lc3 name
if not arcroleTypePattern.match(arcroleURI):
val.modelXbrl.warning(("EFM.6.07.13.arcroleEnding", "GFM.1.03.15"),
_("ArcroleType %(arcroleType)s should end with /arcrole/{LC3name}"),
modelObject=e, arcroleType=arcroleURI)
# 6.7.15 definition match pattern
modelRoleTypes = val.modelXbrl.arcroleTypes[arcroleURI]
definition = modelRoleTypes[0].definition
if definition is None or not arcroleDefinitionPattern.match(definition):
val.modelXbrl.error(("EFM.6.07.15", "GFM.1.03.17"),
_("ArcroleType %(arcroleType)s definition must be non-empty"),
modelObject=e, arcroleType=arcroleURI)
# semantic checks
usedOns = modelRoleTypes[0].usedOns
if usedOns & standardUsedOns: # semantics check
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"),
_("ArcroleType %(arcroleuri)s is defined using role types already defined by standard arcroles for: %(qnames)s"),
modelObject=e, arcroleuri=arcroleURI, qnames=', '.join(str(qn) for qn in usedOns & standardUsedOns))
if val.validateSBRNL:
definesArcroles = True
val.modelXbrl.error("SBR.NL.2.2.4.01",
_("Arcrole type definition is not allowed: %(arcroleURI)s"),
modelObject=e, arcroleURI=arcroleURI)
if val.validateSBRNL:
for appinfoElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}appinfo"):
for nonLinkElt in appinfoElt.iterdescendants():
if isinstance(nonLinkElt, ModelObject) and nonLinkElt.namespaceURI != XbrlConst.link:
val.modelXbrl.error("SBR.NL.2.2.11.05",
_("Appinfo contains disallowed non-link element %(element)s"),
modelObject=nonLinkElt, element=nonLinkElt.qname)
for cplxTypeElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}complexType"):
choiceElt = cplxTypeElt.find("{http://www.w3.org/2001/XMLSchema}choice")
if choiceElt is not None:
val.modelXbrl.error("SBR.NL.2.2.11.09",
_("ComplexType contains disallowed xs:choice element"),
modelObject=choiceElt)
for cplxContentElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}complexContent"):
if XmlUtil.descendantAttr(cplxContentElt, "http://www.w3.org/2001/XMLSchema", ("extension","restriction"), "base") != "sbr:placeholder":
val.modelXbrl.error("SBR.NL.2.2.11.10",
_("ComplexContent is disallowed"),
modelObject=cplxContentElt)
definesTypes = (modelDocument.xmlRootElement.find("{http://www.w3.org/2001/XMLSchema}complexType") is not None or
modelDocument.xmlRootElement.find("{http://www.w3.org/2001/XMLSchema}simpleType") is not None)
for enumElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}enumeration"):
definesEnumerations = True
if any(not valueElt.genLabel(lang="nl")
for valueElt in enumElt.iter(tag="{http://www.w3.org/2001/XMLSchema}value")):
val.modelXbrl.error("SBR.NL.2.2.7.05",
_("Enumeration element has value(s) without generic label."),
modelObject=enumElt)
if (definesLinkroles + definesArcroles + definesLinkParts +
definesAbstractItems + definesNonabstractItems +
definesTuples + definesPresentationTuples + definesSpecificationTuples + definesTypes +
definesEnumerations + definesDimensions + definesDomains +
definesHypercubes) != 1:
schemaContents = []
if definesLinkroles: schemaContents.append(_("linkroles"))
if definesArcroles: schemaContents.append(_("arcroles"))
if definesLinkParts: schemaContents.append(_("link parts"))
if definesAbstractItems: schemaContents.append(_("abstract items"))
if definesNonabstractItems: schemaContents.append(_("nonabstract items"))
if definesTuples: schemaContents.append(_("tuples"))
if definesPresentationTuples: schemaContents.append(_("sbrPresentationTuples"))
if definesSpecificationTuples: schemaContents.append(_("sbrSpecificationTuples"))
if definesTypes: schemaContents.append(_("types"))
if definesEnumerations: schemaContents.append(_("enumerations"))
if definesDimensions: schemaContents.append(_("dimensions"))
if definesDomains: schemaContents.append(_("domains"))
if definesHypercubes: schemaContents.append(_("hypercubes"))
if schemaContents:
if not ((definesTuples or definesPresentationTuples or definesSpecificationTuples) and
not (definesLinkroles or definesArcroles or definesLinkParts or definesAbstractItems or
definesTypes or definesDimensions or definesDomains or definesHypercubes)):
val.modelXbrl.error("SBR.NL.2.2.1.01",
_("Taxonomy schema may only define one of these: %(contents)s"),
modelObject=modelDocument, contents=', '.join(schemaContents))
elif not any(refDoc.inDTS and refDoc.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces
for refDoc in modelDocument.referencesDocument.keys()): # no linkbase ref or includes
val.modelXbrl.error("SBR.NL.2.2.1.01",
_("Taxonomy schema must be a DTS entrypoint OR define linkroles OR arcroles OR link:parts OR context fragments OR abstract items OR tuples OR non-abstract elements OR types OR enumerations OR dimensions OR domains OR hypercubes"),
modelObject=modelDocument)
if definesConcepts ^ any( # xor so either concepts and no label LB or no concepts and has label LB
(refDoc.type == ModelDocument.Type.LINKBASE and
XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "labelLink") is not None)
for refDoc in modelDocument.referencesDocument.keys()): # no label linkbase
val.modelXbrl.error("SBR.NL.2.2.1.02",
_("A schema that defines concepts MUST have a linked 2.1 label linkbase"),
modelObject=modelDocument)
if (definesNonabstractItems or definesTuples) and not any( # was xor but changed to and not per RH 1/11/12
(refDoc.type == ModelDocument.Type.LINKBASE and
(XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "referenceLink") is not None or
XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "label", "{http://www.w3.org/1999/xlink}role", "http://www.xbrl.org/2003/role/documentation" ) is not None))
for refDoc in modelDocument.referencesDocument.keys()):
val.modelXbrl.error("SBR.NL.2.2.1.03",
_("A schema that defines non-abstract items MUST have a linked (2.1) reference linkbase AND/OR a label linkbase with @xlink:role=documentation"),
modelObject=modelDocument)
#6.3.3 filename check
m = re.match(r"^\w+-([12][0-9]{3}[01][0-9][0-3][0-9]).xsd$", modelDocument.basename)
if m:
try: # check date value
datetime.datetime.strptime(m.group(1),"%Y%m%d").date()
# date and format are ok, check "should" part of 6.3.3
if val.fileNameBasePart:
expectedFilename = "{0}-{1}.xsd".format(val.fileNameBasePart, val.fileNameDatePart)
if modelDocument.basename != expectedFilename:
val.modelXbrl.log("WARNING-SEMANTIC", ("EFM.6.03.03.matchInstance", "GFM.1.01.01.matchInstance"),
_('Schema file name warning: %(filename)s, should match %(expectedFilename)s'),
modelObject=modelDocument, filename=modelDocument.basename, expectedFilename=expectedFilename)
except ValueError:
val.modelXbrl.error(("EFM.6.03.03", "GFM.1.01.01"),
_('Invalid schema file base name part (date) in "{base}-{yyyymmdd}.xsd": %(filename)s'),
modelObject=modelDocument, filename=modelDocument.basename)
else:
val.modelXbrl.error(("EFM.6.03.03", "GFM.1.01.01"),
_('Invalid schema file name, must match "{base}-{yyyymmdd}.xsd": %(filename)s'),
modelObject=modelDocument, filename=modelDocument.basename)
elif modelDocument.type == ModelDocument.Type.LINKBASE:
# if it is part of the submission (in same directory) check name
if modelDocument.filepath.startswith(val.modelXbrl.modelDocument.filepathdir):
#6.3.3 filename check
extLinkElt = XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "*", "{http://www.w3.org/1999/xlink}type", "extended")
if extLinkElt is None:# no ext link element
val.modelXbrl.error(("EFM.6.03.03.noLinkElement", "GFM.1.01.01.noLinkElement"),
_('Invalid linkbase file name: %(filename)s, has no extended link element, cannot determine link type.'),
modelObject=modelDocument, filename=modelDocument.basename)
elif extLinkElt.localName not in extLinkEltFileNameEnding:
val.modelXbrl.error("EFM.6.03.02",
_('Invalid linkbase link element %(linkElement)s in %(filename)s'),
modelObject=modelDocument, linkElement=extLinkElt.localName, filename=modelDocument.basename)
else:
m = re.match(r"^\w+-([12][0-9]{3}[01][0-9][0-3][0-9])(_[a-z]{3}).xml$", modelDocument.basename)
expectedSuffix = extLinkEltFileNameEnding[extLinkElt.localName]
if m and m.group(2) == expectedSuffix:
try: # check date value
datetime.datetime.strptime(m.group(1),"%Y%m%d").date()
# date and format are ok, check "should" part of 6.3.3
if val.fileNameBasePart:
expectedFilename = "{0}-{1}{2}.xml".format(val.fileNameBasePart, val.fileNameDatePart, expectedSuffix)
if modelDocument.basename != expectedFilename:
val.modelXbrl.log("WARNING-SEMANTIC", ("EFM.6.03.03.matchInstance", "GFM.1.01.01.matchInstance"),
_('Linkbase name warning: %(filename)s should match %(expectedFilename)s'),
modelObject=modelDocument, filename=modelDocument.basename, expectedFilename=expectedFilename)
except ValueError:
val.modelXbrl.error(("EFM.6.03.03", "GFM.1.01.01"),
_('Invalid linkbase base file name part (date) in "{base}-{yyyymmdd}_{suffix}.xml": %(filename)s'),
modelObject=modelDocument, filename=modelDocument.basename)
else:
val.modelXbrl.error(("EFM.6.03.03", "GFM.1.01.01"),
_('Invalid linkbase name, must match "{base}-{yyyymmdd}%(expectedSuffix)s.xml": %(filename)s'),
modelObject=modelDocument, filename=modelDocument.basename, expectedSuffix=expectedSuffix)
visited.remove(modelDocument)
def description(self):
nameElement = XmlUtil.descendant(self, None,
("description", "documentation"))
if nameElement is not None:
return XmlUtil.innerText(nameElement)
return None
def assignmentRefs(self):
return XmlUtil.childrenAttrs(self, XbrlConst.ver, "assignmentRef",
"ref")
def expectedVersioningReport(self):
XmlUtil.text(
XmlUtil.text(
XmlUtil.descendant(XmlUtil.descendant(self, None, "result"),
None, "versioningReport")))
def toResourceValue(self):
return XmlUtil.childAttr(self, XbrlConst.verce, "toResource", "value")
def expected(self):
for pluginXbrlMethod in pluginClassMethods(
"ModelTestcaseVariation.ExpectedResult"):
expected = pluginXbrlMethod(self)
if expected:
return expected
# default behavior without plugins
if self.localName == "testcase":
return self.document.basename[:4] #starts with PASS or FAIL
elif self.localName == "testGroup": #w3c testcase
instanceTestElement = XmlUtil.descendant(self, None,
"instanceTest")
if instanceTestElement is not None: # take instance first
return XmlUtil.descendantAttr(instanceTestElement, None,
"expected", "validity")
else:
schemaTestElement = XmlUtil.descendant(self, None,
"schemaTest")
if schemaTestElement is not None:
return XmlUtil.descendantAttr(schemaTestElement, None,
"expected", "validity")
resultElement = XmlUtil.descendant(self, None, "result")
if resultElement is not None:
expected = resultElement.get("expected")
if expected and resultElement.get(
"nonStandardErrorCodes") == "true":
# if @expected and @nonStandardErrorCodes then use expected instead of error codes
return expected
errorElement = XmlUtil.descendant(self, None, "error")
resultElement = XmlUtil.descendant(self, None, "result")
if errorElement is not None and not errorElement.get(
"nonStandardErrorCodes"):
_errorText = XmlUtil.text(errorElement)
if ' ' in _errorText: # list of tokens
return _errorText
return ModelValue.qname(errorElement,
_errorText) # turn into a QName
if resultElement is not None:
if expected:
return expected
for assertElement in XmlUtil.children(resultElement, None,
"assert"):
num = assertElement.get("num")
if num == "99999": # inline test, use name as expected
return assertElement.get("name")
if len(num) == 5:
return "EFM.{0}.{1}.{2}".format(num[0], num[1:3], num[3:6])
asserTests = {}
for atElt in XmlUtil.children(resultElement, None,
"assertionTests"):
try:
asserTests[atElt.get("assertionID")] = (
_INT(atElt.get("countSatisfied")),
_INT(atElt.get("countNotSatisfied")))
except ValueError:
pass
if asserTests:
return asserTests
elif self.get("result"):
return self.get("result")
return None
def toConceptQname(self):
toConcept = XmlUtil.child(self, XbrlConst.vercb, "toConcept")
if toConcept is not None and toConcept.get("name"):
return qname(toConcept, toConcept.get("name"))
else:
return None
def resultIsTaxonomyPackage(self):
return any(e.localName
for e in XmlUtil.descendants(self, None, TXMY_PKG_SRC_ELTS))
def actionId(self):
return XmlUtil.parentId(self, XbrlConst.ver, "action")
def readMeFirstUris(self):
try:
return self._readMeFirstUris
except AttributeError:
self._readMeFirstUris = []
# first look if any plugin method to get readme first URIs
if not any(
pluginXbrlMethod(self)
for pluginXbrlMethod in pluginClassMethods(
"ModelTestcaseVariation.ReadMeFirstUris")):
if self.localName == "testGroup": #w3c testcase
instanceTestElement = XmlUtil.descendant(
self, None, "instanceTest")
if instanceTestElement is not None: # take instance first
self._readMeFirstUris.append(
XmlUtil.descendantAttr(
instanceTestElement, None, "instanceDocument",
"{http://www.w3.org/1999/xlink}href"))
else:
schemaTestElement = XmlUtil.descendant(
self, None, "schemaTest")
if schemaTestElement is not None:
self._readMeFirstUris.append(
XmlUtil.descendantAttr(
schemaTestElement, None, "schemaDocument",
"{http://www.w3.org/1999/xlink}href"))
elif self.localName == "test-case": #xpath testcase
inputFileElement = XmlUtil.descendant(
self, None, "input-file")
if inputFileElement is not None: # take instance first
self._readMeFirstUris.append("TestSources/" +
inputFileElement.text +
".xml")
elif self.resultIsTaxonomyPackage:
self._readMeFirstUris.append(
os.path.join(self.modelDocument.filepathdir, "tests",
self.get("name") + ".zip"))
else:
# default built-in method for readme first uris
for anElement in self.iterdescendants():
if isinstance(anElement,
ModelObject) and anElement.get(
"readMeFirst") == "true":
if anElement.get(
"{http://www.w3.org/1999/xlink}href"):
uri = anElement.get(
"{http://www.w3.org/1999/xlink}href")
else:
uri = XmlUtil.innerText(anElement)
if anElement.get("name"):
self._readMeFirstUris.append(
(ModelValue.qname(anElement,
anElement.get("name")),
uri))
elif anElement.get("dts"):
self._readMeFirstUris.append(
(anElement.get("dts"), uri))
else:
self._readMeFirstUris.append(uri)
if not self._readMeFirstUris: # provide a dummy empty instance document
self._readMeFirstUris.append(
os.path.join(self.modelXbrl.modelManager.cntlr.configDir,
"empty-instance.xml"))
return self._readMeFirstUris
def linkbaseDiscover(self, linkbaseElement, inInstance=False):
for lbElement in linkbaseElement.childNodes:
if lbElement.nodeType == 1: #element
lbLn = lbElement.localName
lbNs = lbElement.namespaceURI
if lbNs == XbrlConst.link:
if lbLn == "roleRef" or lbLn == "arcroleRef":
href = self.discoverHref(lbElement)
if href is None:
self.modelXbrl.error(
"Linkbase in {0} {1} href attribute missing or malformed"
.format(os.path.basename(self.uri),
lbLn), "err", "xbrl:hrefMissing")
else:
self.hrefObjects.append(href)
continue
if lbElement.getAttributeNS(XbrlConst.xlink,
"type") == "extended":
self.schemalocateElementNamespace(lbElement)
arcrolesFound = set()
dimensionArcFound = False
formulaArcFound = False
euRenderingArcFound = False
linkQn = qname(lbElement)
linkrole = lbElement.getAttributeNS(
XbrlConst.xlink, "role")
modelLink = ModelObject.createLink(self, lbElement)
if inInstance:
#index footnote links even if no arc children
baseSetKeys = (("XBRL-footnotes", None, None, None),
("XBRL-footnotes", linkrole, None,
None))
for baseSetKey in baseSetKeys:
self.modelXbrl.baseSets[baseSetKey].append(
modelLink)
for linkElement in lbElement.childNodes:
if linkElement.nodeType == 1: #element
self.schemalocateElementNamespace(linkElement)
xlinkType = linkElement.getAttributeNS(
XbrlConst.xlink, "type")
modelResource = None
if xlinkType == "locator":
nonDTS = linkElement.namespaceURI != XbrlConst.link or linkElement.localName != "loc"
# only link:loc elements are discovered or processed
href = self.discoverHref(linkElement,
nonDTS=nonDTS)
if href is None:
self.modelXbrl.error(
"Linkbase in {0} {1} href attribute missing or malformed"
.format(os.path.basename(self.uri),
lbLn), "err",
"xbrl:hrefMissing")
else:
modelResource = ModelObject.createLocator(
self, linkElement, href)
elif xlinkType == "arc":
arcQn = qname(linkElement)
arcrole = linkElement.getAttributeNS(
XbrlConst.xlink, "arcrole")
if arcrole not in arcrolesFound:
if linkrole == "":
linkrole = XbrlConst.defaultLinkRole
#index by both arcrole and linkrole#arcrole and dimensionsions if applicable
baseSetKeys = [(arcrole, linkrole, linkQn,
arcQn)]
baseSetKeys.append(
(arcrole, linkrole, None, None))
baseSetKeys.append(
(arcrole, None, None, None))
if XbrlConst.isDimensionArcrole(
arcrole) and not dimensionArcFound:
baseSetKeys.append(("XBRL-dimensions",
None, None, None))
baseSetKeys.append(
("XBRL-dimensions", linkrole, None,
None))
dimensionArcFound = True
if XbrlConst.isFormulaArcrole(
arcrole) and not formulaArcFound:
baseSetKeys.append(("XBRL-formulae",
None, None, None))
baseSetKeys.append(
("XBRL-formulae", linkrole, None,
None))
formulaArcFound = True
if XbrlConst.isEuRenderingArcrole(
arcrole
) and not euRenderingArcFound:
baseSetKeys.append(
("EU-rendering", None, None, None))
baseSetKeys.append(
("EU-rendering", linkrole, None,
None))
euRenderingArcFound = True
self.modelXbrl.hasEuRendering = True
for baseSetKey in baseSetKeys:
self.modelXbrl.baseSets[
baseSetKey].append(modelLink)
arcrolesFound.add(arcrole)
elif xlinkType == "resource":
# create resource and make accessible by id for document
modelResource = ModelObject.createResource(
self, linkElement)
if modelResource is not None:
if linkElement.hasAttribute("id"):
self.idObjects[linkElement.getAttribute(
"id")] = modelResource
modelLink.labeledResources[linkElement.getAttributeNS(XbrlConst.xlink, "label")] \
.append(modelResource)
else:
XmlUtil.markIdAttributes(linkElement)
def resultIsTable(self):
result = XmlUtil.descendant(self, None, "result")
if result is not None:
child = XmlUtil.child(result, None, "table")
return child is not None and XmlUtil.text(child).endswith(".xml")
return False
def stepAxis(self, op, p, sourceSequence):
targetSequence = []
for node in sourceSequence:
if not isinstance(
node, (ModelObject, etree._ElementTree, ModelAttribute)):
raise XPathException(
self.progHeader, 'err:XPTY0020',
_('Axis step {0} context item is not a node: {1}').format(
op, node))
targetNodes = []
if isinstance(p, QNameDef):
ns = p.namespaceURI
localname = p.localName
axis = p.axis
if p.isAttribute:
if isinstance(node, ModelObject):
attrTag = p.localName if p.unprefixed else p.clarkNotation
modelAttribute = None
try:
modelAttribute = node.xAttributes[attrTag]
except (AttributeError, TypeError, IndexError,
KeyError):
# may be lax or deferred validated
try:
validate(node.modelXbrl, node, p)
modelAttribute = node.xAttributes[attrTag]
except (AttributeError, TypeError, IndexError,
KeyError):
pass
if modelAttribute is None:
value = node.get(attrTag)
if value is not None:
targetNodes.append(
ModelAttribute(node, p.clarkNotation,
UNKNOWN, value, value,
value))
elif modelAttribute.xValid >= VALID:
targetNodes.append(modelAttribute)
elif op == '/' or op is None:
if axis is None or axis == "child":
if isinstance(node, (ModelObject, etree._ElementTree)):
targetNodes = XmlUtil.children(node, ns, localname)
elif axis == "parent":
if isinstance(node, ModelAttribute):
parentNode = [node.modelElement]
else:
parentNode = [XmlUtil.parent(node)]
if (isinstance(node, ModelObject)
and (not ns or ns == parentNode.namespaceURI
or ns == "*")
and (localname == parentNode.localName
or localname == "*")):
targetNodes = [parentNode]
elif axis == "self":
if (isinstance(node, ModelObject) and
(not ns or ns == node.namespaceURI or ns == "*")
and
(localname == node.localName or localname == "*")):
targetNodes = [node]
elif axis.startswith("descendant"):
if isinstance(node, (ModelObject, etree._ElementTree)):
targetNodes = XmlUtil.descendants(
node, ns, localname)
if (axis.endswith("-or-self")
and isinstance(node, ModelObject)
and (not ns or ns == node.namespaceURI
or ns == "*")
and (localname == node.localName
or localname == "*")):
targetNodes.append(node)
elif axis.startswith("ancestor"):
if isinstance(node, ModelObject):
targetNodes = [
ancestor
for ancestor in XmlUtil.ancestors(node)
if ((not ns or ns == ancestor.namespaceURI
or ns == "*") and (
localname == ancestor.localName
or localname == "*"))
]
if (axis.endswith("-or-self")
and isinstance(node, ModelObject)
and (not ns or ns == node.namespaceURI
or ns == "*")
and (localname == node.localName
or localname == "*")):
targetNodes.insert(0, node)
elif axis.endswith("-sibling"):
if isinstance(node, ModelObject):
targetNodes = [
sibling for sibling in node.itersiblings(
preceding=axis.startswith("preceding"))
if ((not ns or ns == sibling.namespaceURI
or ns == "*") and (
localname == sibling.localName
or localname == "*"))
]
elif axis == "preceding":
if isinstance(node, ModelObject):
for preceding in node.getroottree().iter():
if preceding == node:
break
elif ((not ns or ns == preceding.namespaceURI
or ns == "*")
and (localname == preceding.localName
or localname == "*")):
targetNodes.append(preceding)
elif axis == "following":
if isinstance(node, ModelObject):
foundNode = False
for following in node.getroottree().iter():
if following == node:
foundNode = True
elif (foundNode and
(not ns or ns == following.namespaceURI
or ns == "*")
and (localname == following.localName
or localname == "*")):
targetNodes.append(following)
elif op == '//':
if isinstance(node, (ModelObject, etree._ElementTree)):
targetNodes = XmlUtil.descendants(node, ns, localname)
elif op == '..':
if isinstance(node, ModelAttribute):
targetNodes = [node.modelElement]
else:
targetNodes = [XmlUtil.parent(node)]
elif isinstance(p, OperationDef) and isinstance(p.name, QNameDef):
if isinstance(node, ModelObject):
if p.name.localName == "text": # note this is not string value, just child text
targetNodes = [node.textValue]
# todo: add element, attribute, node, etc...
elif p == '*': # wildcard
if op == '/' or op is None:
if isinstance(node, (ModelObject, etree._ElementTree)):
targetNodes = XmlUtil.children(node, '*', '*')
elif op == '//':
if isinstance(node, (ModelObject, etree._ElementTree)):
targetNodes = XmlUtil.descendants(node, '*', '*')
targetSequence.extend(targetNodes)
return targetSequence
def resultXbrlInstanceUri(self):
resultInstance = XmlUtil.descendant(
XmlUtil.descendant(self, None, "result"), None, "instance")
if resultInstance is not None:
return XmlUtil.text(resultInstance)
return None
def fiscalYearEnd(self):
yrEnd = XmlUtil.text(XmlUtil.descendant(self, edgr, "fiscalYearEnd"))
if yrEnd and len(yrEnd) == 4:
return "{0}-{1}".format(yrEnd[0:2], yrEnd[2:4])
return None
def toURI(self):
return XmlUtil.childAttr(self, XbrlConst.ver, "toURI", "value")
def versioningReportUri(self):
return XmlUtil.text(XmlUtil.descendant(self, None, "versioningReport"))
def resultIsVersioningReport(self):
return XmlUtil.descendant(XmlUtil.descendant(self, None, "result"), None, "versioningReport") is not None