def createInstance(self, url=None):
"""Creates an instance document for a DTS which didn't have an instance document, such as
to create a new instance for a DTS which was loaded from a taxonomy or linkbase entry point.
:param url: File name to save the new instance document
:type url: str
"""
from arelle import (ModelDocument, FileSource)
if self.modelDocument.type == ModelDocument.Type.INSTANCE: # entry already is an instance
return self.modelDocument # use existing instance entry point
priorFileSource = self.fileSource
self.fileSource = FileSource.FileSource(url)
if self.uri.startswith("http://"):
schemaRefUri = self.uri
else: # relativize local paths
schemaRefUri = os.path.relpath(self.uri, os.path.dirname(url))
self.modelDocument = ModelDocument.create(self, ModelDocument.Type.INSTANCE, url, schemaRefs=[schemaRefUri], isEntry=True)
if priorFileSource:
priorFileSource.close()
self.closeFileSource= True
del self.entryLoadingUrl
# reload dts views
from arelle import ViewWinDTS
for view in self.views:
if isinstance(view, ViewWinDTS.ViewDTS):
self.modelManager.cntlr.uiThreadQueue.put((view.view, []))
def xfxc_element(xc, p, contextItem, args):
if not 2 <= len(args) <= 4: raise XPathContext.FunctionNumArgs()
qn = qnameArg(xc, p, args, 0, 'QName', emptyFallback=None)
attrArg = args[1] if isinstance(args[1], (list, tuple)) else (args[1], )
# attributes have to be pairs
if attrArg:
if len(attrArg) & 1 or any(
not isinstance(attrArg[i], (QName, _STR_BASE))
for i in range(0, len(attrArg), 2)):
raise XPathContext.FunctionArgType(
1,
"((xs:qname|xs:string),xs:anyAtomicValue)",
errCode="xfxce:AttributesNotNameValuePairs")
else:
attrParam = [
(attrArg[i], attrArg[i + 1]
) # need name-value pairs for XmlUtil function
for i in range(0, len(attrArg), 2)
]
else:
attrParam = None
value = atomicArg(xc, p, args, 2, "xs:anyAtomicType", emptyFallback='')
if not value: # be sure '' is None so no text node is created
value = None
if len(args) < 4:
childElements = None
else:
childElements = xc.flattenSequence(args[3])
# scratchpad instance document emulates fn:doc( ) to hold XML nodes
scratchpadXmlDocUrl = "http://www.xbrl.org/2012/function/creation/xml_scratchpad.xml"
if scratchpadXmlDocUrl in xc.modelXbrl.urlDocs:
modelDocument = xc.modelXbrl.urlDocs[scratchpadXmlDocUrl]
else:
# create scratchpad xml document
# this will get the fake instance document in the list of modelXbrl docs so that it is garbage collected
from arelle import ModelDocument
modelDocument = ModelDocument.create(
xc.modelXbrl,
ModelDocument.Type.UnknownXML,
scratchpadXmlDocUrl,
initialXml=
"<xfc:dummy xmlns:xfc='http://www.xbrl.org/2012/function/creation'/>"
)
newElement = XmlUtil.addChild(modelDocument.xmlRootElement,
qn,
attributes=attrParam,
text=value)
if childElements:
for element in childElements:
if isinstance(element, etree.ElementBase):
newElement.append(element)
# node myst be validated for use in instance creation (typed dimension references)
XmlValidate.validate(xc.modelXbrl, newElement)
return newElement
def create(modelManager, newDocumentType=None, url=None, schemaRefs=None, createModelDocument=True, isEntry=False):
from arelle import (ModelDocument, FileSource)
modelXbrl = ModelXbrl(modelManager)
modelXbrl.locale = modelManager.locale
if newDocumentType:
modelXbrl.fileSource = FileSource.FileSource(url)
if createModelDocument:
modelXbrl.modelDocument = ModelDocument.create(modelXbrl, newDocumentType, url, schemaRefs=schemaRefs, isEntry=isEntry)
return modelXbrl
def create(modelManager, newDocumentType=None, url=None, schemaRefs=None, createModelDocument=True, isEntry=False):
from arelle import (ModelDocument, FileSource)
modelXbrl = ModelXbrl(modelManager)
modelXbrl.locale = modelManager.locale
if newDocumentType:
modelXbrl.fileSource = FileSource.FileSource(url) # url may be an open file handle, use str(url) below
modelXbrl.closeFileSource= True
if createModelDocument:
modelXbrl.modelDocument = ModelDocument.create(modelXbrl, newDocumentType, str(url), schemaRefs=schemaRefs, isEntry=isEntry)
if isEntry:
del modelXbrl.entryLoadingUrl
return modelXbrl
def xfxc_element(xc, p, contextItem, args):
if not 2 <= len(args) <= 4: raise XPathContext.FunctionNumArgs()
qn = qnameArg(xc, p, args, 0, 'QName', emptyFallback=None)
attrArg = args[1] if isinstance(args[1],(list,tuple)) else (args[1],)
# attributes have to be pairs
if attrArg:
if len(attrArg) & 1 or any(not isinstance(attrArg[i], (QName, _STR_BASE))
for i in range(0, len(attrArg),2)):
raise XPathContext.FunctionArgType(1,"((xs:qname|xs:string),xs:anyAtomicValue)", errCode="xfxce:AttributesNotNameValuePairs")
else:
attrParam = [(attrArg[i],attrArg[i+1]) # need name-value pairs for XmlUtil function
for i in range(0, len(attrArg),2)]
else:
attrParam = None
value = atomicArg(xc, p, args, 2, "xs:anyAtomicType", emptyFallback='')
if not value: # be sure '' is None so no text node is created
value = None
if len(args) < 4:
childElements = None
else:
childElements = xc.flattenSequence(args[3])
# scratchpad instance document emulates fn:doc( ) to hold XML nodes
scratchpadXmlDocUrl = "http://www.xbrl.org/2012/function/creation/xml_scratchpad.xml"
if scratchpadXmlDocUrl in xc.modelXbrl.urlDocs:
modelDocument = xc.modelXbrl.urlDocs[scratchpadXmlDocUrl]
else:
# create scratchpad xml document
# this will get the fake instance document in the list of modelXbrl docs so that it is garbage collected
from arelle import ModelDocument
modelDocument = ModelDocument.create(xc.modelXbrl,
ModelDocument.Type.UnknownXML,
scratchpadXmlDocUrl,
initialXml="<xfc:dummy xmlns:xfc='http://www.xbrl.org/2012/function/creation'/>")
newElement = XmlUtil.addChild(modelDocument.xmlRootElement,
qn,
attributes=attrParam,
text=value)
if childElements:
for element in childElements:
if isinstance(element, etree.ElementBase):
newElement.append(element)
# node myst be validated for use in instance creation (typed dimension references)
XmlValidate.validate(xc.modelXbrl, newElement)
return newElement
def createInstance(self, url=None):
from arelle import (ModelDocument, FileSource)
if self.modelDocument.type == ModelDocument.Type.INSTANCE: # entry already is an instance
return self.modelDocument # use existing instance entry point
priorFileSource = self.fileSource
self.fileSource = FileSource.FileSource(url)
if self.uri.startswith("http://"):
schemaRefUri = self.uri
else: # relativize local paths
schemaRefUri = os.path.relpath(self.uri, os.path.dirname(url))
self.modelDocument = ModelDocument.create(self, ModelDocument.Type.INSTANCE, url, schemaRefs=[schemaRefUri], isEntry=True)
if priorFileSource:
priorFileSource.close()
self.closeFileSource= True
del self.entryLoadingUrl
# reload dts views
from arelle import ViewWinDTS
for view in self.views:
if isinstance(view, ViewWinDTS.ViewDTS):
self.modelManager.cntlr.uiThreadQueue.put((view.view, []))
def create(modelManager,
newDocumentType=None,
url=None,
schemaRefs=None,
createModelDocument=True,
isEntry=False):
from arelle import (ModelDocument, FileSource)
modelXbrl = ModelXbrl(modelManager)
modelXbrl.locale = modelManager.locale
if newDocumentType:
modelXbrl.fileSource = FileSource.FileSource(
url) # url may be an open file handle, use str(url) below
modelXbrl.closeFileSource = True
if createModelDocument:
modelXbrl.modelDocument = ModelDocument.create(
modelXbrl,
newDocumentType,
str(url),
schemaRefs=schemaRefs,
isEntry=isEntry)
if isEntry:
del modelXbrl.entryLoadingUrl
return modelXbrl
def createInstance(self, url=None):
from arelle import (ModelDocument, FileSource)
if self.modelDocument.type == ModelDocument.Type.INSTANCE: # entry already is an instance
return self.modelDocument # use existing instance entry point
priorFileSource = self.fileSource
self.fileSource = FileSource.FileSource(url)
if self.uri.startswith("http://"):
schemaRefUri = self.uri
else: # relativize local paths
schemaRefUri = os.path.relpath(self.uri, os.path.dirname(url))
self.modelDocument = ModelDocument.create(self,
ModelDocument.Type.INSTANCE,
url,
schemaRefs=[schemaRefUri],
isEntry=True)
if priorFileSource:
priorFileSource.close()
self.closeFileSource = True
del self.entryLoadingUrl
# reload dts views
from arelle import ViewWinDTS
for view in self.views:
if isinstance(view, ViewWinDTS.ViewDTS):
self.modelManager.cntlr.uiThreadQueue.put((view.view, []))
def loadFromExcel(cntlr, excelFile):
from arelle import xlrd
from arelle.xlrd.sheet import empty_cell
from arelle import ModelDocument, ModelXbrl, XmlUtil
from arelle.ModelDocument import ModelDocumentReference
from arelle.ModelValue import qname
startedAt = time.time()
importExcelBook = xlrd.open_workbook(excelFile)
controlSheet = importExcelBook.sheet_by_index(1)
imports = {"xbrli": ( ("namespace", XbrlConst.xbrli),
("schemaLocation", "http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd") )} # xml of imports
importXmlns = {}
linkbaseRefs = []
labelLinkbases = []
hasPreLB = hasCalLB = hasDefLB = False
# xxxLB structure [ (elr1, def1, "_ELR_", [roots]), (elr2, def2, "_ELR_", [rootw]) ...]
# roots = (rootHref, None, "_root_", [children])
# children = (childPrefix, childName, arcrole, [grandChildren])
preLB = []
defLB = []
calLB = []
def lbDepthList(lbStruct, depth, parentList=None):
if depth == 0:
return lbStruct[-1].childStruct
return lbDepthList(lbStruct[-1].childStruct, depth-1, list)
extensionElements = {}
extensionRoles = {} # key is roleURI, value is role definition
extensionLabels = {} # key = (prefix, name, lang, role), value = label text
def extensionHref(prefix, name):
if prefix == extensionSchemaPrefix:
filename = extensionSchemaFilename
elif prefix in imports:
filename = imports[prefix][1][1]
else:
return None
return "{0}#{1}_{2}".format(filename, prefix, name)
isUSGAAP = False
for iRow in range(1, controlSheet.nrows):
try:
row = controlSheet.row(iRow)
if (row[0].ctype == xlrd.XL_CELL_EMPTY): # skip if col 1 is empty
continue
action = row[0].value
filetype = row[1].value
prefix = row[2].value
filename = row[3].value
namespaceURI = row[4].value
lbType = lang = None
if action == "import":
imports[prefix] = ( ("namespace", namespaceURI), ("schemaLocation", filename) )
importXmlns[prefix] = namespaceURI
if re.match(r"http://[^/]+/us-gaap/", namespaceURI):
isUSGAAP = True
elif action == "extension":
if filetype == "schema":
extensionSchemaPrefix = prefix
extensionSchemaFilename = filename
extensionSchemaNamespaceURI = namespaceURI
elif filetype == "linkbase":
typeLang = prefix.split()
if len(typeLang) > 0:
lbType = typeLang[0]
else:
lbType = "unknown"
if len(typeLang) > 1:
lang = typeLang[1]
else:
lang = "en"
if lbType == "label":
labelLinkbases.append((lang, filename))
elif lbType == "presentation":
hasPreLB = True
elif lbType == "definition":
hasDefLB = True
elif lbType == "calculation":
hasCalLB = True
linkbaseRefs.append( (lbType, filename) )
elif filetype == "role" and namespaceURI:
extensionRoles[namespaceURI] = filename
except Exception as err:
cntlr.addToLog("Exception: {error}, Excel row: {excelRow}"
.format(error=err,
excelRow=iRow),
messageCode="importExcel:exception")
importExcelSheet = importExcelBook.sheet_by_index(0)
# find column headers row
headerCols = {}
headerRows = set()
# find out which rows are header rows
for iRow in range(0, importExcelSheet.nrows):
row = importExcelSheet.row(iRow)
for iCol, colCell in enumerate(row):
v = colCell.value
if v in importColumnHeaders:
headerCols[importColumnHeaders[v]] = iCol
if all(colName in headerCols
for colName in ("name", "type", "depth")): # must have these to be a header col
# it's a header col
headerRows.add(iRow)
headerCols.clear()
def cellValue(row, header):
if header in headerCols:
iCol = headerCols[header]
if iCol < len(row):
return row[iCol].value
return ''
def checkImport(qname):
prefix, sep, localName = qname.partition(":")
if sep:
if prefix not in imports:
if prefix == "xbrldt":
imports["xbrldt"] = ("namespace", XbrlConst.xbrldt), ("schemaLocation", "http://www.xbrl.org/2005/xbrldt-2005.xsd")
elif prefix == "nonnum":
imports["nonnum"] = ("namespace", "http://www.xbrl.org/dtr/type/non-numeric"), ("schemaLocation", "http://www.xbrl.org/dtr/type/nonNumeric-2009-12-16.xsd")
else:
cntlr.addToLog("Warning: prefix schema file is not imported for: {qname}"
.format(qname=qname),
messageCode="importExcel:warning")
# find header rows
currentELR = currentELRdefinition = None
for iRow in range(0, importExcelSheet.nrows):
useLabels = False
try:
row = importExcelSheet.row(iRow)
isHeaderRow = iRow in headerRows
isELRrow = (iRow + 1) in headerRows
if isHeaderRow:
headerCols.clear()
for iCol, colCell in enumerate(row):
v = colCell.value
if v in importColumnHeaders:
headerCols[importColumnHeaders[v]] = iCol
elif isELRrow:
currentELR = currentELRdefinition = None
for colCell in row:
v = colCell.value
if v.startswith("http://"):
currentELR = v
elif not currentELRdefinition and v.endswith("　科目一覧"):
currentELRdefinition = v[0:-5]
elif not currentELRdefinition:
currentELRdefinition = v
if currentELR or currentELRdefinition:
if hasPreLB:
preLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
if hasDefLB:
defLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
if hasCalLB:
calLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
elif headerCols:
prefix = cellValue(row, 'prefix').strip()
name = cellValue(row, 'name').strip()
if "depth" in headerCols:
try:
depth = int(cellValue(row, 'depth'))
except ValueError:
depth = None
else:
depth = None
if prefix == extensionSchemaPrefix and name not in extensionElements:
# elements row
eltType = cellValue(row, 'type')
subsGrp = cellValue(row, 'substitutionGroup')
abstract = cellValue(row, 'abstract')
nillable = cellValue(row, 'nillable')
balance = cellValue(row, 'balance')
periodType = cellValue(row, 'periodType')
newElt = [ ("name", name), ("id", prefix + "_" + name) ]
if eltType:
newElt.append( ("type", eltType) )
checkImport(eltType)
if subsGrp:
newElt.append( ("substitutionGroup", subsGrp) )
checkImport(subsGrp)
if abstract:
newElt.append( ("abstract", abstract) )
if nillable:
newElt.append( ("nillable", nillable) )
if balance:
newElt.append( ("{http://www.xbrl.org/2003/instance}balance", balance) )
if periodType:
newElt.append( ("{http://www.xbrl.org/2003/instance}periodType", periodType) )
extensionElements[name] = newElt
useLabels = True
if depth is not None:
if hasPreLB:
entryList = lbDepthList(preLB, depth)
preferredLabel = cellValue(row, 'preferredLabel')
if preferredLabel and not preferredLabel.startswith("http://"):
preferredLabel = "http://www.xbrl.org/2003/role/" + preferredLabel
if entryList is not None:
if depth == 0:
entryList.append( LBentry(prefix=prefix, name=name, isRoot=True) )
else:
entryList.append( LBentry(prefix=prefix, name=name, arcrole=XbrlConst.parentChild,
role=preferredLabel) )
if hasDefLB:
entryList = lbDepthList(defLB, depth)
if entryList is not None:
if depth == 0:
entryList.append( LBentry(prefix=prefix, name=name, isRoot=True) )
else:
entryList.append( LBentry(prefix=prefix, name=name, arcrole="_dimensions_") )
if hasCalLB:
calcParent = cellValue(row, 'calculationParent')
calcWeight = cellValue(row, 'calculationWeight')
if calcParent and calcWeight:
calcParentPrefix, sep, calcParentName = calcParent.partition(":")
entryList = lbDepthList(calLB, 0)
if entryList is not None:
entryList.append( LBentry(prefix=calcParentPrefix, name=calcParentName, isRoot=True, childStruct=
[LBentry(prefix=prefix, name=name, arcrole=XbrlConst.summationItem, weight=calcWeight )]) )
# accumulate extension labels
if useLabels:
prefix = cellValue(row, 'prefix').strip()
name = cellValue(row, 'name').strip()
preferredLabel = cellValue(row, 'preferredLabel')
if preferredLabel and not preferredLabel.startswith("http://"):
preferredLabel = "http://www.xbrl.org/2003/role/" + preferredLabel
for colItem, iCol in headerCols.items():
if isinstance(colItem, tuple):
colItemType, role, lang = colItem
cell = row[iCol]
if cell.ctype == xlrd.XL_CELL_EMPTY:
values = ()
elif colItemType == "label":
values = (cell.value,)
elif colItemType == "labels":
values = cell.value.split('\n')
else:
values = ()
if preferredLabel: # first label column sets preferredLabel if any
role = preferredLabel
preferredLabel = None
for value in values:
extensionLabels[prefix, name, lang, role] = value.strip()
except Exception as err:
cntlr.addToLog("Exception: {error}, Excel row: {excelRow}"
.format(error=err,
excelRow=iRow),
messageCode="importExcel:exception")
if isUSGAAP and hasDefLB:
# move line items above table
def fixUsggapTableDims(lvl1Struct):
foundLineItems = False
for i1, lvl1Entry in enumerate(lvl1Struct):
for i2, lvl2Entry in enumerate(lvl1Entry.childStruct):
for i3, lvl3Entry in enumerate(lvl2Entry.childStruct):
if lvl3Entry.name.endswith("LineItems") and lvl2Entry.name.endswith("Table"):
foundLineItems = True
break
if foundLineItems:
break
else:
fixUsggapTableDims(lvl1Entry.childStruct)
if foundLineItems:
lvl1Struct.insert(i1 + 1, LBentry(prefix=lvl3Entry.prefix, name=lvl3Entry.name, arcrole=lvl1Entry.arcrole, childStruct=lvl3Entry.childStruct)) # must keep lvl1Rel if it is __root__
lvl3Entry.childStruct.insert(0, lvl2Entry)
if lvl1Entry.name.endswith("Abstract"):
del lvl1Struct[i1]
del lvl2Entry.childStruct[i3]
pass
fixUsggapTableDims(defLB)
dts = cntlr.modelManager.create(newDocumentType=ModelDocument.Type.SCHEMA,
url=extensionSchemaFilename,
isEntry=True,
base='', # block pathname from becomming absolute
initialXml='''
<schema xmlns="http://www.w3.org/2001/XMLSchema"
targetNamespace="{targetNamespace}"
attributeFormDefault="unqualified"
elementFormDefault="qualified"
xmlns:xsd="http://www.w3.org/2001/XMLSchema"
xmlns:{extensionPrefix}="{targetNamespace}"
{importXmlns}
xmlns:iod="http://disclosure.edinet-fsa.go.jp/taxonomy/common/2013-03-31/iod"
xmlns:nonnum="http://www.xbrl.org/dtr/type/non-numeric"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:xbrli="http://www.xbrl.org/2003/instance"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrldt="http://xbrl.org/2005/xbrldt"/>
'''.format(targetNamespace=extensionSchemaNamespaceURI,
extensionPrefix=extensionSchemaPrefix,
importXmlns=''.join('xmlns:{0}="{1}"\n'.format(prefix, namespaceURI)
for prefix, namespaceURI in importXmlns.items())
)
)
dtsSchemaDocument = dts.modelDocument
dtsSchemaDocument.inDTS = True # entry document always in DTS
dtsSchemaDocument.targetNamespace = extensionSchemaNamespaceURI # not set until schemaDiscover too late otherwise
schemaElt = dtsSchemaDocument.xmlRootElement
#foreach linkbase
annotationElt = XmlUtil.addChild(schemaElt, XbrlConst.xsd, "annotation")
appinfoElt = XmlUtil.addChild(annotationElt, XbrlConst.xsd, "appinfo")
for iRow in range(0, importExcelSheet.nrows):
try:
row = importExcelSheet.row(iRow)
if (row[0].ctype == xlrd.XL_CELL_EMPTY): # skip if col 1 is empty
continue
testDir = row[0].value
uriFrom = row[1].value
uriTo = row[2].value
except Exception as err:
cntlr.addToLog("Exception: {error}, Excel row: {excelRow}"
.format(error=err,
excelRow=iRow),
messageCode="loadFromExcel:exception")
# add linkbaseRefs
appinfoElt = XmlUtil.descendant(schemaElt, XbrlConst.xsd, "appinfo")
# don't yet add linkbase refs, want to process imports first to get roleType definitions
# add imports
for importAttributes in sorted(imports.values()):
XmlUtil.addChild(schemaElt,
XbrlConst.xsd, "import",
attributes=importAttributes)
# add elements
for eltName, eltAttrs in sorted(extensionElements.items(), key=lambda item: item[0]):
XmlUtil.addChild(schemaElt,
XbrlConst.xsd, "element",
attributes=eltAttrs)
# add role definitions (for discovery)
for roleURI, roleDefinition in extensionRoles.items():
roleElt = XmlUtil.addChild(appinfoElt, XbrlConst.link, "roleType",
attributes=(("roleURI", roleURI),
("id", "roleType_" + roleURI.rpartition("/")[2])))
if roleDefinition:
XmlUtil.addChild(roleElt, XbrlConst.link, "definition", text=roleDefinition)
if hasPreLB:
XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:presentationLink")
if hasDefLB:
XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:definitionLink")
if hasCalLB:
XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:calculationLink")
dtsSchemaDocument.schemaDiscover(schemaElt, False, extensionSchemaNamespaceURI)
def addLinkbaseRef(lbType, lbFilename, lbDoc):
role = "http://www.xbrl.org/2003/role/{0}LinkbaseRef".format(lbType)
lbRefElt = XmlUtil.addChild(appinfoElt, XbrlConst.link, "linkbaseRef",
attributes=(("{http://www.w3.org/1999/xlink}type", "simple"),
("{http://www.w3.org/1999/xlink}href", lbFilename),
("{http://www.w3.org/1999/xlink}role", role),
("{http://www.w3.org/1999/xlink}arcrole", "http://www.w3.org/1999/xlink/properties/linkbase"),
))
dtsSchemaDocument.referencesDocument[lbDoc] = ModelDocumentReference("href", lbRefElt)
# label linkbase
for lang, filename in labelLinkbases:
lbDoc = ModelDocument.create(dts, ModelDocument.Type.LINKBASE, filename, base="", initialXml="""
<link:linkbase
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.xbrl.org/2003/linkbase
http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrli="http://www.xbrl.org/2003/instance"/>
""")
lbDoc.inDTS = True
addLinkbaseRef("label", filename, lbDoc)
lbElt = lbDoc.xmlRootElement
linkElt = XmlUtil.addChild(lbElt,
XbrlConst.link, "labelLink",
attributes=(("{http://www.w3.org/1999/xlink}type", "extended"),
("{http://www.w3.org/1999/xlink}role", "http://www.xbrl.org/2003/role/link")))
locs = set()
for labelKey, text in extensionLabels.items():
prefix, name, labelLang, role = labelKey
if lang == labelLang:
locLabel = prefix + "_" + name
if locLabel not in locs:
locs.add(locLabel)
XmlUtil.addChild(linkElt,
XbrlConst.link, "loc",
attributes=(("{http://www.w3.org/1999/xlink}type", "locator"),
("{http://www.w3.org/1999/xlink}href", extensionHref(prefix, name)),
("{http://www.w3.org/1999/xlink}label", locLabel)))
XmlUtil.addChild(linkElt,
XbrlConst.link, "labelArc",
attributes=(("{http://www.w3.org/1999/xlink}type", "arc"),
("{http://www.w3.org/1999/xlink}arcrole", "http://www.xbrl.org/2003/arcrole/concept-label"),
("{http://www.w3.org/1999/xlink}from", locLabel),
("{http://www.w3.org/1999/xlink}to", "label_" + locLabel),
("order", 1.0)))
XmlUtil.addChild(linkElt,
XbrlConst.link, "label",
attributes=(("{http://www.w3.org/1999/xlink}type", "resource"),
("{http://www.w3.org/1999/xlink}label", "label_" + locLabel),
("{http://www.w3.org/1999/xlink}role", role),
("{http://www.w3.org/XML/1998/namespace}lang", lang)),
text=text)
lbDoc.linkbaseDiscover(lbElt)
def hrefConcept(prefix, name):
qn = schemaElt.prefixedNameQname(prefix + ":" + name)
if qn in dts.qnameConcepts:
return dts.qnameConcepts[qn]
return None
def lbTreeWalk(lbType, parentElt, lbStruct, roleRefs, locs=None, fromPrefix=None, fromName=None):
order = 1.0
for lbEntry in lbStruct:
if lbEntry.isELR:
role = "unspecified"
if lbEntry.role and lbEntry.role.startswith("http://"): # have a role specified
role = lbEntry.role
elif lbEntry.name: #may be a definition
for linkroleUri, modelRoleTypes in dts.roleTypes.items():
definition = modelRoleTypes[0].definition
if lbEntry.name == definition:
role = linkroleUri
break
if role != XbrlConst.defaultLinkRole and role in dts.roleTypes: # add roleRef
roleType = modelRoleTypes[0]
roleRef = ("roleRef", role, roleType.modelDocument.uri + "#" + roleType.id)
roleRefs.add(roleRef)
linkElt = XmlUtil.addChild(parentElt,
XbrlConst.link, lbType + "Link",
attributes=(("{http://www.w3.org/1999/xlink}type", "extended"),
("{http://www.w3.org/1999/xlink}role", role)))
locs = set()
lbTreeWalk(lbType, linkElt, lbEntry.childStruct, roleRefs, locs)
else:
toPrefix = lbEntry.prefix
toName = lbEntry.name
toHref = extensionHref(toPrefix, toName)
toLabel = toPrefix + "_" + toName
if toHref not in locs:
XmlUtil.addChild(parentElt,
XbrlConst.link, "loc",
attributes=(("{http://www.w3.org/1999/xlink}type", "locator"),
("{http://www.w3.org/1999/xlink}href", toHref),
("{http://www.w3.org/1999/xlink}label", toLabel)))
locs.add(toHref)
if not lbEntry.isRoot:
fromLabel = fromPrefix + "_" + fromName
if lbType == "calculation" and lbEntry.weight is not None:
otherAttrs = ( ("weight", lbEntry.weight), )
elif lbType == "presentation" and lbEntry.role is not None:
otherAttrs = ( ("preferredLabel", lbEntry.role), )
else:
otherAttrs = ( )
if lbEntry.arcrole == "_dimensions_": # pick proper consecutive arcrole
fromConcept = hrefConcept(fromPrefix, fromName)
toConcept = hrefConcept(toPrefix, toName)
if toConcept is not None and toConcept.isHypercubeItem:
arcrole = XbrlConst.all
elif toConcept is not None and toConcept.isDimensionItem:
arcrole = XbrlConst.hypercubeDimension
elif fromConcept is not None and fromConcept.isDimensionItem:
arcrole = XbrlConst.dimensionDomain
else:
arcrole = XbrlConst.domainMember
else:
arcrole = lbEntry.arcrole
XmlUtil.addChild(parentElt,
XbrlConst.link, lbType + "Arc",
attributes=(("{http://www.w3.org/1999/xlink}type", "arc"),
("{http://www.w3.org/1999/xlink}arcrole", arcrole),
("{http://www.w3.org/1999/xlink}from", fromLabel),
("{http://www.w3.org/1999/xlink}to", toLabel),
("order", order)) + otherAttrs )
order += 1.0
if lbType != "calculation" or lbEntry.isRoot:
lbTreeWalk(lbType, parentElt, lbEntry.childStruct, roleRefs, locs, toPrefix, toName)
for hasLB, lbType, lbLB in ((hasPreLB, "presentation", preLB),
(hasDefLB, "definition", defLB),
(hasCalLB, "calculation", calLB)):
if hasLB:
for lbRefType, filename in linkbaseRefs:
if lbType == lbRefType:
# output presentation linkbase
lbDoc = ModelDocument.create(dts, ModelDocument.Type.LINKBASE, filename, base='', initialXml="""
<link:linkbase
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.xbrl.org/2003/linkbase
http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrli="http://www.xbrl.org/2003/instance"/>
""")
lbDoc.inDTS = True
addLinkbaseRef(lbRefType, filename, lbDoc)
lbElt = lbDoc.xmlRootElement
roleRefs = set()
if lbType == "definition":
roleRefs.update((("arcroleRef", XbrlConst.all, "http://www.xbrl.org/2005/xbrldt-2005.xsd#all"),
("arcroleRef", XbrlConst.dimensionDefault, "http://www.xbrl.org/2005/xbrldt-2005.xsd#dimension-default"),
("arcroleRef", XbrlConst.dimensionDomain, "http://www.xbrl.org/2005/xbrldt-2005.xsd#dimension-domain"),
("arcroleRef", XbrlConst.domainMember, "http://www.xbrl.org/2005/xbrldt-2005.xsd#domain-member"),
("arcroleRef", XbrlConst.hypercubeDimension, "http://www.xbrl.org/2005/xbrldt-2005.xsd#hypercube-dimension")))
lbTreeWalk(lbType, lbElt, lbLB, roleRefs)
firstLinkElt = None
for firstLinkElt in lbElt.iterchildren():
break
# add arcrole references
for roleref, roleURI, href in roleRefs:
XmlUtil.addChild(lbElt,
XbrlConst.link, roleref,
attributes=(("arcroleURI" if roleref == "arcroleRef" else "roleURI", roleURI),
("{http://www.w3.org/1999/xlink}type", "simple"),
("{http://www.w3.org/1999/xlink}href", href)),
beforeSibling=firstLinkElt)
lbDoc.linkbaseDiscover(lbElt)
break
#cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt),
# messageCode="loadFromExcel:info")
return dts
def secCorrespondenceLoader(modelXbrl, mappedUri, filepath, **kwargs):
if (mappedUri.startswith("http://www.sec.gov/Archives/edgar/Feed/")
and mappedUri.endswith(".nc.tar.gz")):
# daily feed loader (the rss object)
rssObject = ModelRssObject(modelXbrl, uri=mappedUri, filepath=filepath)
# location for expanded feed files
tempdir = os.path.join(modelXbrl.modelManager.cntlr.userAppDir, "tmp",
"edgarFeed")
# remove prior files
if os.path.exists(tempdir):
os.system("rm -fr {}".format(
tempdir)) # rmtree does not work with this many files!
os.makedirs(tempdir, exist_ok=True)
# untar to /temp/arelle/edgarFeed for faster operation
startedAt = time.time()
modelXbrl.fileSource.open()
modelXbrl.fileSource.fs.extractall(tempdir)
modelXbrl.info("info",
"untar edgarFeed temp files in %.2f sec" %
(time.time() - startedAt),
modelObject=modelXbrl)
# find <table> with <a>Download in it
for instanceFile in sorted(
os.listdir(tempdir)): # modelXbrl.fileSource.dir:
if instanceFile != ".":
rssObject.rssItems.append(
SECCorrespondenceItem(modelXbrl, instanceFile,
mappedUri + '/' + instanceFile))
return rssObject
elif "rssItem" in kwargs and ".nc.tar.gz/" in mappedUri:
rssItem = kwargs["rssItem"]
text = None # no instance information
# parse document
try:
startedAt = time.time()
file, encoding = modelXbrl.fileSource.file(
os.path.join(modelXbrl.modelManager.cntlr.userAppDir, "tmp",
"edgarFeed", os.path.basename(rssItem.url)))
s = file.read()
file.close()
for match in re.finditer(r"[<]([^>]+)[>]([^<\n\r]*)", s,
re.MULTILINE):
tag = match.group(1).lower()
v = match.group(2)
if tag == "accession-number":
rssItem.accessionNumber = v
elif tag == "form-type":
rssItem.formType = v
if v != "UPLOAD":
rssItem.doNotProcessRSSitem = True # skip this RSS item in validate loop, don't load DB
elif tag == "filing-date":
try:
rssItem.filingDate = datetime.date(
int(v[0:4]), int(v[4:6]), int(v[6:8]))
except (ValueError, IndexError):
pass
elif tag == "conformed-name":
rssItem.companyName = v
elif tag == "cik":
rssItem.cikNumber = v
elif tag == "assigned-sic":
rssItem.assignedSic = v
elif tag == "fiscal-year-end":
try:
rssItem.fiscalYearEnd = v[0:2] + '-' + v[2:4]
except (IndexError, TypeError):
pass
match = re.search("<PDF>(.*)</PDF>", s, re.DOTALL)
if match:
import uu, io
pageText = []
uuIn = io.BytesIO(match.group(1).encode(encoding))
uuOut = io.BytesIO()
uu.decode(uuIn, uuOut)
from pyPdf import PdfFileReader
uuOut.seek(0, 0)
try:
pdfIn = PdfFileReader(uuOut)
for pageNum in range(pdfIn.getNumPages()):
pageText.append(pdfIn.getPage(pageNum).extractText())
except:
# do we want a warning here that the PDF can't be read with this library?
pass
uuIn.close()
uuOut.close()
text = ''.join(pageText)
else:
match = re.search("<TEXT>(.*)</TEXT>", s, re.DOTALL)
if match:
text = match.group(1)
except (IOError, EnvironmentError):
pass # give up, no instance
# daily rss item loader, provide unpopulated instance document to be filled in by RssItem.Xbrl.Loaded
if not text:
rssItem.doNotProcessRSSitem = True # skip this RSS item in validate loop, don't load DB
instDoc = ModelDocument.create(
modelXbrl,
ModelDocument.Type.UnknownXML,
rssItem.url,
isEntry=True,
base='', # block pathname from becomming absolute
initialXml='<DummyXml/>')
else:
instDoc = ModelDocument.create(
modelXbrl,
ModelDocument.Type.INSTANCE,
rssItem.url,
isEntry=True,
base='', # block pathname from becomming absolute
initialXml='''
<xbrli:xbrl xmlns:doc="http://arelle.org/doc/2014-01-31"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrli="http://www.xbrl.org/2003/instance">
<link:schemaRef xlink:type="simple" xlink:href="http://arelle.org/2014/doc-2014-01-31.xsd"/>
<xbrli:context id="pubDate">
<xbrli:entity>
<xbrli:identifier scheme="http://www.sec.gov/CIK">{cik}</xbrli:identifier>
</xbrli:entity>
<xbrli:period>
<xbrli:instant>{pubDate}</xbrli:instant>
</xbrli:period>
</xbrli:context>
<doc:Correspondence contextRef="pubDate">{text}</doc:Correspondence>
</xbrli:xbrl>
'''.format(cik=rssItem.cikNumber,
pubDate=rssItem.pubDate.date(),
text=text.strip().replace("&",
"&").replace("<",
"<")))
#modelXbrl.info("info", "loaded in %.2f sec" % (time.time() - startedAt),
# modelDocument=instDoc)
return instDoc
return None
def init(modelXbrl):
# setup modelXbrl for rendering evaluation
# dimension defaults required in advance of validation
from arelle import ValidateXbrlDimensions, ValidateFormula, ModelDocument
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
hasXbrlTables = False
# validate table linkbase dimensions
for baseSetKey in modelXbrl.baseSets.keys():
arcrole, ELR, linkqname, arcqname = baseSetKey
if ELR and linkqname and arcqname and XbrlConst.isTableRenderingArcrole(
arcrole):
ValidateFormula.checkBaseSet(
modelXbrl, arcrole, ELR,
modelXbrl.relationshipSet(arcrole, ELR, linkqname, arcqname))
if arcrole in (XbrlConst.tableBreakdown,
XbrlConst.tableBreakdown201301,
XbrlConst.tableAxis2011):
hasXbrlTables = True
# provide context for view
if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE:
instance = None # use instance of the entry pont
else: # need dummy instance
instance = ModelDocument.create(
modelXbrl,
ModelDocument.Type.INSTANCE,
"dummy.xml", # fake URI and fake schemaRef
("http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd", ))
if hasXbrlTables:
# formula processor is needed for 2011 XBRL tables but not for 2010 Eurofiling tables
modelXbrl.rendrCntx = XPathContext.create(modelXbrl, instance)
modelXbrl.profileStat(None)
# setup fresh parameters from formula optoins
modelXbrl.parameters = modelXbrl.modelManager.formulaOptions.typedParameters(
)
# validate parameters and custom function signatures
ValidateFormula.validate(modelXbrl,
xpathContext=modelXbrl.rendrCntx,
parametersOnly=True,
statusMsg=_("compiling rendering tables"))
# deprecated as of 2013-05-17
# check and extract message expressions into compilable programs
for msgArcrole in (XbrlConst.tableDefinitionNodeMessage201301,
XbrlConst.tableDefinitionNodeSelectionMessage201301,
XbrlConst.tableAxisMessage2011,
XbrlConst.tableAxisSelectionMessage2011):
for msgRel in modelXbrl.relationshipSet(
msgArcrole).modelRelationships:
ValidateFormula.checkMessageExpressions(
modelXbrl, msgRel.toModelObject)
# compile and validate tables
for modelTable in modelXbrl.modelRenderingTables:
modelTable.fromInstanceQnames = None # required if referred to by variables scope chaining
modelTable.compile()
# check aspectModel
if modelTable.aspectModel not in ("non-dimensional",
"dimensional"):
modelXbrl.error(
"xbrlte:unknownAspectModel",
_("Table %(xlinkLabel)s, aspect model %(aspectModel)s not recognized"
),
modelObject=modelTable,
xlinkLabel=modelTable.xlinkLabel,
aspectModel=modelTable.aspectModel)
else:
modelTable.priorAspectAxisDisposition = {}
# check ordinate aspects against aspectModel
oppositeAspectModel = (
_DICT_SET({'dimensional', 'non-dimensional'}) -
_DICT_SET({modelTable.aspectModel})).pop()
uncoverableAspects = aspectModels[
oppositeAspectModel] - aspectModels[modelTable.aspectModel]
for tblAxisRel in modelXbrl.relationshipSet(
(XbrlConst.tableBreakdown, XbrlConst.tableBreakdown201301,
XbrlConst.tableAxis2011)).fromModelObject(modelTable):
checkDefinitionNodeAspectModel(modelXbrl, modelTable,
tblAxisRel,
uncoverableAspects)
del modelTable.priorAspectAxisDisposition
modelXbrl.profileStat(_("compileTables"))
pass
uuIn.close()
uuOut.close()
text = u''.join(pageText)
else:
match = re.search(u"<TEXT>(.*)</TEXT>", s, re.DOTALL)
if match:
text = match.group(1)
except (IOError, EnvironmentError):
pass # give up, no instance
# daily rss item loader, provide unpopulated instance document to be filled in by RssItem.Xbrl.Loaded
if not text:
rssItem.doNotProcessRSSitem = True # skip this RSS item in validate loop, don't load DB
instDoc = ModelDocument.create(modelXbrl,
ModelDocument.Type.UnknownXML,
rssItem.url,
isEntry=True,
base=u'', # block pathname from becomming absolute
initialXml=u'<DummyXml/>')
else:
instDoc = ModelDocument.create(modelXbrl,
ModelDocument.Type.INSTANCE,
rssItem.url,
isEntry=True,
base=u'', # block pathname from becomming absolute
initialXml=u'''
<xbrli:xbrl xmlns:doc="http://arelle.org/doc/2014-01-31"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrli="http://www.xbrl.org/2003/instance">
<link:schemaRef xlink:type="simple" xlink:href="http://arelle.org/2014/doc-2014-01-31.xsd"/>
<xbrli:context id="pubDate">
def secCorrespondenceLoader(modelXbrl, mappedUri, filepath, **kwargs):
if (mappedUri.startswith("http://www.sec.gov/Archives/edgar/Feed/") and
mappedUri.endswith(".nc.tar.gz")):
# daily feed loader (the rss object)
rssObject = ModelRssObject(modelXbrl, uri=mappedUri, filepath=filepath)
# location for expanded feed files
tempdir = os.path.join(modelXbrl.modelManager.cntlr.userAppDir, "tmp", "edgarFeed")
# remove prior files
if os.path.exists(tempdir):
os.system("rm -fr {}".format(tempdir)) # rmtree does not work with this many files!
os.makedirs(tempdir, exist_ok=True)
# untar to /temp/arelle/edgarFeed for faster operation
startedAt = time.time()
modelXbrl.fileSource.open()
modelXbrl.fileSource.fs.extractall(tempdir)
modelXbrl.info("info", "untar edgarFeed temp files in %.2f sec" % (time.time() - startedAt),
modelObject=modelXbrl)
# find <table> with <a>Download in it
for instanceFile in sorted(os.listdir(tempdir)): # modelXbrl.fileSource.dir:
if instanceFile != ".":
rssObject.rssItems.append(
SECCorrespondenceItem(modelXbrl, instanceFile, mappedUri + '/' + instanceFile))
return rssObject
elif "rssItem" in kwargs and ".nc.tar.gz/" in mappedUri:
rssItem = kwargs["rssItem"]
text = None # no instance information
# parse document
try:
startedAt = time.time()
file, encoding = modelXbrl.fileSource.file(
os.path.join(modelXbrl.modelManager.cntlr.userAppDir, "tmp", "edgarFeed",
os.path.basename(rssItem.url)))
s = file.read()
file.close()
for match in re.finditer(r"[<]([^>]+)[>]([^<\n\r]*)", s, re.MULTILINE):
tag = match.group(1).lower()
v = match.group(2)
if tag == "accession-number":
rssItem.accessionNumber = v
elif tag == "form-type":
rssItem.formType = v
if v != "UPLOAD":
rssItem.doNotProcessRSSitem = True # skip this RSS item in validate loop, don't load DB
elif tag == "filing-date":
try:
rssItem.filingDate = datetime.date(int(v[0:4]), int(v[4:6]), int(v[6:8]))
except (ValueError, IndexError):
pass
elif tag == "conformed-name":
rssItem.companyName = v
elif tag == "cik":
rssItem.cikNumber = v
elif tag == "assigned-sic":
rssItem.assignedSic = v
elif tag == "fiscal-year-end":
try:
rssItem.fiscalYearEnd = v[0:2] + '-' + v[2:4]
except (IndexError, TypeError):
pass
match = re.search("<PDF>(.*)</PDF>", s, re.DOTALL)
if match:
import uu, io
pageText = []
uuIn = io.BytesIO(match.group(1).encode(encoding))
uuOut = io.BytesIO()
uu.decode(uuIn, uuOut)
from pyPdf import PdfFileReader
uuOut.seek(0,0)
try:
pdfIn = PdfFileReader(uuOut)
for pageNum in range(pdfIn.getNumPages()):
pageText.append(pdfIn.getPage(pageNum).extractText())
except:
# do we want a warning here that the PDF can't be read with this library?
pass
uuIn.close()
uuOut.close()
text = ''.join(pageText)
else:
match = re.search("<TEXT>(.*)</TEXT>", s, re.DOTALL)
if match:
text = match.group(1)
except (IOError, EnvironmentError):
pass # give up, no instance
# daily rss item loader, provide unpopulated instance document to be filled in by RssItem.Xbrl.Loaded
if not text:
rssItem.doNotProcessRSSitem = True # skip this RSS item in validate loop, don't load DB
instDoc = ModelDocument.create(modelXbrl,
ModelDocument.Type.UnknownXML,
rssItem.url,
isEntry=True,
base='', # block pathname from becomming absolute
initialXml='<DummyXml/>')
else:
instDoc = ModelDocument.create(modelXbrl,
ModelDocument.Type.INSTANCE,
rssItem.url,
isEntry=True,
base='', # block pathname from becomming absolute
initialXml='''
<xbrli:xbrl xmlns:doc="http://arelle.org/doc/2014-01-31"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrli="http://www.xbrl.org/2003/instance">
<link:schemaRef xlink:type="simple" xlink:href="http://arelle.org/2014/doc-2014-01-31.xsd"/>
<xbrli:context id="pubDate">
<xbrli:entity>
<xbrli:identifier scheme="http://www.sec.gov/CIK">{cik}</xbrli:identifier>
</xbrli:entity>
<xbrli:period>
<xbrli:instant>{pubDate}</xbrli:instant>
</xbrli:period>
</xbrli:context>
<doc:Correspondence contextRef="pubDate">{text}</doc:Correspondence>
</xbrli:xbrl>
'''.format(cik=rssItem.cikNumber,
pubDate=rssItem.pubDate.date(),
text=text.strip().replace("&","&").replace("<","<")))
#modelXbrl.info("info", "loaded in %.2f sec" % (time.time() - startedAt),
# modelDocument=instDoc)
return instDoc
return None
def loadFromExcel(cntlr, excelFile):
from openpyxl import load_workbook
from arelle import ModelDocument, ModelXbrl, XmlUtil
from arelle.ModelDocument import ModelDocumentReference
from arelle.ModelValue import qname
startedAt = time.time()
if os.path.isabs(excelFile):
# allow relative filenames to loading directory
priorCWD = os.getcwd()
os.chdir(os.path.dirname(excelFile))
else:
priorCWD = None
importExcelBook = load_workbook(excelFile, read_only=True, data_only=True)
sheetNames = importExcelBook.get_sheet_names()
if "DTS" in sheetNames:
dtsWs = importExcelBook["DTS"]
elif "Sheet2" in sheetNames:
dtsWs = importExcelBook["Sheet2"]
else:
dtsWs = None
imports = {"xbrli": ( ("namespace", XbrlConst.xbrli),
("schemaLocation", "http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd") )} # xml of imports
importXmlns = {}
linkbaseRefs = []
labelLinkbases = []
hasPreLB = hasCalLB = hasDefLB = False
# xxxLB structure [ (elr1, def1, "_ELR_", [roots]), (elr2, def2, "_ELR_", [rootw]) ...]
# roots = (rootHref, None, "_root_", [children])
# children = (childPrefix, childName, arcrole, [grandChildren])
preLB = []
defLB = []
calLB = []
def lbDepthList(lbStruct, depth, parentList=None):
if depth == topDepth:
if len(lbStruct) > 0:
return lbStruct[-1].childStruct
else:
cntlr.addToLog("Depth error, Excel row: {excelRow}"
.format(excelRow=iRow),
messageCode="importExcel:depth")
return None
return lbDepthList(lbStruct[-1].childStruct, depth-1, list)
extensionElements = {}
extensionRoles = {} # key is roleURI, value is role definition
extensionLabels = {} # key = (prefix, name, lang, role), value = label text
importSheetName = None
skipRows = [] # [(from,to),(from,to)] row number starting at 1
def extensionHref(prefix, name):
if prefix == extensionSchemaPrefix:
filename = extensionSchemaFilename
elif prefix in imports:
filename = imports[prefix][1][1]
else:
return None
return "{0}#{1}_{2}".format(filename, prefix, name)
isUSGAAP = False
for iRow, row in enumerate(dtsWs.rows if dtsWs else ()):
try:
if (len(row) < 1 or row[0].value is None): # skip if col 1 is empty
continue
action = filetype = prefix = filename = namespaceURI = None
if len(row) > 0: action = row[0].value
if len(row) > 1: filetype = row[1].value
if len(row) > 2: prefix = row[2].value
if len(row) > 3: filename = row[3].value
if len(row) > 4: namespaceURI = row[4].value
lbType = lang = None
if action == "import":
imports[prefix] = ( ("namespace", namespaceURI), ("schemaLocation", filename) )
importXmlns[prefix] = namespaceURI
if re.match(r"http://[^/]+/us-gaap/", namespaceURI):
isUSGAAP = True
elif action == "extension":
if filetype == "schema":
extensionSchemaPrefix = prefix
extensionSchemaFilename = filename
extensionSchemaNamespaceURI = namespaceURI
elif filetype == "linkbase":
typeLang = prefix.split()
if len(typeLang) > 0:
lbType = typeLang[0]
else:
lbType = "unknown"
if len(typeLang) > 1:
lang = typeLang[1]
else:
lang = "en"
if lbType == "label":
labelLinkbases.append((lang, filename))
elif lbType == "presentation":
hasPreLB = True
elif lbType == "definition":
hasDefLB = True
elif lbType == "calculation":
hasCalLB = True
linkbaseRefs.append( (lbType, filename) )
elif filetype == "role" and namespaceURI:
extensionRoles[namespaceURI] = filename
elif action == "worksheet" and filename:
importSheetName = filename
elif action == "colheader" and filename and namespaceURI:
importColHeaderMap[filename].append(namespaceURI)
elif action == "skip rows":
if filename:
fromRow, _sep, toRow = filename.partition("-")
try:
skipRows.append((int(fromRow), int(toRow) if toRow else int(fromRow)))
except (ValueError, TypeError):
cntlr.addToLog("Exception: {error}, Excel row: {excelRow}"
.format(error=err,
excelRow=iRow),
messageCode="importExcel:skip rows")
except Exception as err:
cntlr.addToLog("Exception: {error}, Excel row: {excelRow}"
.format(error=err,
excelRow=iRow),
messageCode="importExcel:exception")
if not isUSGAAP: # need extra namespace declaration
importXmlns["iod"] = "http://disclosure.edinet-fsa.go.jp/taxonomy/common/2013-03-31/iod"
# find column headers row
headerCols = {}
hasLinkroleSeparateRow = True
headerRows = set()
topDepth = 999999
if importSheetName and importSheetName in sheetNames:
conceptsWs = importExcelBook[importSheetName]
elif "Concepts" in sheetNames:
conceptsWs = importExcelBook["Concepts"]
elif "Sheet1" in sheetNames:
conceptsWs = importExcelBook["Sheet1"]
else:
conceptsWs = None
def setHeaderCols(row):
headerCols.clear()
for iCol, colCell in enumerate(row):
v = colCell.value
if v in importColHeaderMap:
for hdr in importColHeaderMap[v]:
if hdr in importColumnHeaders:
headerCols[importColumnHeaders[hdr]] = iCol
elif v in importColumnHeaders:
headerCols[importColumnHeaders[v]] = iCol
# find out which rows are header rows
for iRow, row in enumerate(conceptsWs.rows if conceptsWs else ()):
if any(fromRow <= iRow+1 <= toRow for fromRow,toRow in skipRows):
continue
for iCol, colCell in enumerate(row):
setHeaderCols(row)
if all(colName in headerCols
for colName in ("name", "type", "depth")): # must have these to be a header col
# it's a header col
headerRows.add(iRow+1)
if 'linkrole' in headerCols:
hasLinkroleSeparateRow = False
headerCols.clear()
def cellHasValue(row, header, _type):
if header in headerCols:
iCol = headerCols[header]
return iCol < len(row) and isinstance(row[iCol].value, _type)
return False
def cellValue(row, header, strip=False, nameChars=False):
if header in headerCols:
iCol = headerCols[header]
if iCol < len(row):
v = row[iCol].value
if strip and isinstance(v, str):
v = v.strip()
if nameChars and isinstance(v, str):
v = ''.join(c for c in v if c.isalnum() or c in ('.', '_', '-'))
return v
return None
def checkImport(qname):
prefix, sep, localName = qname.partition(":")
if sep:
if prefix not in imports:
if prefix == "xbrldt":
imports["xbrldt"] = ("namespace", XbrlConst.xbrldt), ("schemaLocation", "http://www.xbrl.org/2005/xbrldt-2005.xsd")
elif prefix == "nonnum":
imports["nonnum"] = ("namespace", "http://www.xbrl.org/dtr/type/non-numeric"), ("schemaLocation", "http://www.xbrl.org/dtr/type/nonNumeric-2009-12-16.xsd")
else:
cntlr.addToLog("Warning: prefix schema file is not imported for: {qname}"
.format(qname=qname),
messageCode="importExcel:warning")
# find top depth
for iRow, row in enumerate(conceptsWs.rows if conceptsWs else ()):
if (iRow + 1) in headerRows:
setHeaderCols(row)
elif not (hasLinkroleSeparateRow and (iRow + 1) in headerRows) and 'depth' in headerCols:
depth = cellValue(row, 'depth')
if isinstance(depth, int) and depth < topDepth:
topDepth = depth
# find header rows
currentELR = currentELRdefinition = None
for iRow, row in enumerate(conceptsWs.rows if conceptsWs else ()):
useLabels = False
if any(fromRow <= iRow+1 <= toRow for fromRow,toRow in skipRows):
continue
if (all(col is None for col in row) or
all(isinstance(row[i].value, str) and row[i].value.strip() == "n/a"
for i in (headerCols.get("name"), headerCols.get("type"), headerCols.get("value"))
if i)):
continue # skip blank row
try:
isHeaderRow = (iRow + 1) in headerRows
isELRrow = hasLinkroleSeparateRow and (iRow + 2) in headerRows
if isHeaderRow:
setHeaderCols(row)
elif isELRrow:
currentELR = currentELRdefinition = None
for colCell in row:
v = str(colCell.value or '')
if v.startswith("http://"):
currentELR = v
elif not currentELRdefinition and v.endswith("　科目一覧"):
currentELRdefinition = v[0:-5]
elif not currentELRdefinition:
currentELRdefinition = v
if currentELR or currentELRdefinition:
if hasPreLB:
preLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
if hasDefLB:
defLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
if hasCalLB:
calLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
calRels = set() # prevent duplications when same rel in different parts of tree
elif headerCols:
if "linkrole" in headerCols and cellHasValue(row, 'linkrole', str):
v = cellValue(row, 'linkrole', strip=True)
_trialELR = _trialELRdefinition = None
if v.startswith("http://"):
_trialELR = v
elif v.endswith("　科目一覧"):
_trialELRdefinition = v[0:-5]
else:
_trialELRdefinition = v
if (_trialELR and _trialELR != currentELR) or (_trialELRdefinition and _trialELRdefinition != currentELRdefinition):
currentELR = _trialELR
currentELRdefinition = _trialELRdefinition
if currentELR or currentELRdefinition:
if hasPreLB:
preLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
if hasDefLB:
defLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
if hasCalLB:
calLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
calRels = set() # prevent duplications when same rel in different parts of tree
prefix = cellValue(row, 'prefix', nameChars=True) or extensionSchemaPrefix
if cellHasValue(row, 'name', str):
name = cellValue(row, 'name', nameChars=True)
else:
name = None
if cellHasValue(row, 'depth', int):
depth = cellValue(row, 'depth')
else:
depth = None
if (not prefix or prefix == extensionSchemaPrefix) and name not in extensionElements and name:
# elements row
eltType = cellValue(row, 'type')
eltTypePrefix = cellValue(row, 'typePrefix')
if not eltType:
eltType = 'xbrli:stringItemType'
elif eltTypePrefix and ':' not in eltType:
eltType = eltTypePrefix + ':' + eltType
elif ':' not in eltType and eltType.endswith("ItemType"):
eltType = 'xbrli:' + eltType
subsGrp = cellValue(row, 'substitutionGroup') or 'xbrli:item'
abstract = cellValue(row, 'abstract') or (cellValue(row, 'abstractMarker') is not None)
nillable = cellValue(row, 'nillable')
balance = cellValue(row, 'balance')
periodType = cellValue(row, 'periodType')
newElt = [ ("name", name), ("id", (prefix or "") + "_" + name) ]
if eltType:
newElt.append( ("type", eltType) )
checkImport(eltType)
if subsGrp:
newElt.append( ("substitutionGroup", subsGrp) )
checkImport(subsGrp)
if abstract or subsGrp in ("xbrldt:hypercubeItem", "xbrldt:dimensionItem"):
newElt.append( ("abstract", abstract or "true") )
if nillable:
newElt.append( ("nillable", nillable) )
if balance:
newElt.append( ("{http://www.xbrl.org/2003/instance}balance", balance) )
if periodType:
newElt.append( ("{http://www.xbrl.org/2003/instance}periodType", periodType) )
extensionElements[name] = newElt
useLabels = True
if depth is not None:
if hasPreLB:
entryList = lbDepthList(preLB, depth)
preferredLabel = cellValue(row, 'preferredLabel')
if preferredLabel and not preferredLabel.startswith("http://"):
preferredLabel = "http://www.xbrl.org/2003/role/" + preferredLabel
if entryList is not None:
if depth == topDepth:
entryList.append( LBentry(prefix=prefix, name=name, isRoot=True) )
else:
entryList.append( LBentry(prefix=prefix, name=name, arcrole=XbrlConst.parentChild,
role=preferredLabel) )
if hasDefLB:
entryList = lbDepthList(defLB, depth)
if entryList is not None:
if depth == topDepth:
entryList.append( LBentry(prefix=prefix, name=name, isRoot=True) )
else:
if (not preferredLabel or # prevent start/end labels from causing duplicate dim-mem relationships
not any(lbEntry.prefix == prefix and lbEntry.name == name
for lbEntry in entryList)):
entryList.append( LBentry(prefix=prefix, name=name, arcrole="_dimensions_") )
if hasCalLB:
calcParents = cellValue(row, 'calculationParent').split()
calcWeights = (str(cellValue(row, 'calculationWeight')) or '').split() # may be float or string
if calcParents and calcWeights:
# may be multiple parents split by whitespace
for i, calcParent in enumerate(calcParents):
calcWeight = calcWeights[i] if i < len(calcWeights) else calcWeights[-1]
calcParentPrefix, sep, calcParentName = calcParent.partition(":")
entryList = lbDepthList(calLB, topDepth)
if entryList is not None:
calRel = (calcParentPrefix, calcParentName, prefix, name)
if calRel not in calRels:
entryList.append( LBentry(prefix=calcParentPrefix, name=calcParentName, isRoot=True, childStruct=
[LBentry(prefix=prefix, name=name, arcrole=XbrlConst.summationItem, weight=calcWeight )]) )
calRels.add(calRel)
else:
pass
# accumulate extension labels
if useLabels:
prefix = cellValue(row, 'prefix', nameChars=True) or extensionSchemaPrefix
name = cellValue(row, 'name', nameChars=True)
if name is not None:
preferredLabel = cellValue(row, 'preferredLabel')
if preferredLabel and not preferredLabel.startswith("http://"):
preferredLabel = "http://www.xbrl.org/2003/role/" + preferredLabel
for colItem, iCol in headerCols.items():
if isinstance(colItem, tuple):
colItemType = colItem[0]
role = colItem[1]
lang = colItem[2]
cell = row[iCol]
if cell.value is None:
values = ()
elif colItemType == "label":
values = (cell.value,)
elif colItemType == "labels":
values = cell.value.split('\n')
else:
values = ()
if preferredLabel and "indented" in colItem: # indented column sets preferredLabel if any
role = preferredLabel
for value in values:
extensionLabels[prefix, name, lang, role] = value.strip()
except Exception as err:
cntlr.addToLog("Exception: {error}, Excel row: {excelRow}"
.format(error=err,
excelRow=iRow),
messageCode="importExcel:exception")
if isUSGAAP and hasDefLB:
# move line items above table
def fixUsggapTableDims(lvl1Struct):
foundLineItems = False
for i1, lvl1Entry in enumerate(lvl1Struct):
for i2, lvl2Entry in enumerate(lvl1Entry.childStruct):
for i3, lvl3Entry in enumerate(lvl2Entry.childStruct):
if lvl3Entry.name.endswith("LineItems") and lvl2Entry.name.endswith("Table"):
foundLineItems = True
break
if foundLineItems:
break
else:
fixUsggapTableDims(lvl1Entry.childStruct)
if foundLineItems:
lvl1Struct.insert(i1 + 1, LBentry(prefix=lvl3Entry.prefix, name=lvl3Entry.name, arcrole=lvl1Entry.arcrole, childStruct=lvl3Entry.childStruct)) # must keep lvl1Rel if it is __root__
lvl3Entry.childStruct.insert(0, lvl2Entry)
if lvl1Entry.name.endswith("Abstract"):
del lvl1Struct[i1]
if i3 < len(lvl2Entry.childStruct):
del lvl2Entry.childStruct[i3]
pass
fixUsggapTableDims(defLB)
dts = cntlr.modelManager.create(newDocumentType=ModelDocument.Type.SCHEMA,
url=extensionSchemaFilename,
isEntry=True,
base='', # block pathname from becomming absolute
initialXml='''
<schema xmlns="http://www.w3.org/2001/XMLSchema"
targetNamespace="{targetNamespace}"
attributeFormDefault="unqualified"
elementFormDefault="qualified"
xmlns:xsd="http://www.w3.org/2001/XMLSchema"
xmlns:{extensionPrefix}="{targetNamespace}"
{importXmlns}
xmlns:nonnum="http://www.xbrl.org/dtr/type/non-numeric"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:xbrli="http://www.xbrl.org/2003/instance"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrldt="http://xbrl.org/2005/xbrldt"/>
'''.format(targetNamespace=extensionSchemaNamespaceURI,
extensionPrefix=extensionSchemaPrefix,
importXmlns=''.join('xmlns:{0}="{1}"\n'.format(prefix, namespaceURI)
for prefix, namespaceURI in importXmlns.items())
)
)
dtsSchemaDocument = dts.modelDocument
dtsSchemaDocument.inDTS = True # entry document always in DTS
dtsSchemaDocument.targetNamespace = extensionSchemaNamespaceURI # not set until schemaDiscover too late otherwise
schemaElt = dtsSchemaDocument.xmlRootElement
#foreach linkbase
annotationElt = XmlUtil.addChild(schemaElt, XbrlConst.xsd, "annotation")
appinfoElt = XmlUtil.addChild(annotationElt, XbrlConst.xsd, "appinfo")
# add linkbaseRefs
appinfoElt = XmlUtil.descendant(schemaElt, XbrlConst.xsd, "appinfo")
# don't yet add linkbase refs, want to process imports first to get roleType definitions
# add imports
for importAttributes in sorted(imports.values()):
XmlUtil.addChild(schemaElt,
XbrlConst.xsd, "import",
attributes=importAttributes)
# add elements
for eltName, eltAttrs in sorted(extensionElements.items(), key=lambda item: item[0]):
XmlUtil.addChild(schemaElt,
XbrlConst.xsd, "element",
attributes=eltAttrs)
# add role definitions (for discovery)
for roleURI, roleDefinition in extensionRoles.items():
roleElt = XmlUtil.addChild(appinfoElt, XbrlConst.link, "roleType",
attributes=(("roleURI", roleURI),
("id", "roleType_" + roleURI.rpartition("/")[2])))
if roleDefinition:
XmlUtil.addChild(roleElt, XbrlConst.link, "definition", text=roleDefinition)
if hasPreLB:
XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:presentationLink")
if hasDefLB:
XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:definitionLink")
if hasCalLB:
XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:calculationLink")
dtsSchemaDocument.schemaDiscover(schemaElt, False, extensionSchemaNamespaceURI)
def addLinkbaseRef(lbType, lbFilename, lbDoc):
role = "http://www.xbrl.org/2003/role/{0}LinkbaseRef".format(lbType)
lbRefElt = XmlUtil.addChild(appinfoElt, XbrlConst.link, "linkbaseRef",
attributes=(("{http://www.w3.org/1999/xlink}type", "simple"),
("{http://www.w3.org/1999/xlink}href", lbFilename),
("{http://www.w3.org/1999/xlink}role", role),
("{http://www.w3.org/1999/xlink}arcrole", "http://www.w3.org/1999/xlink/properties/linkbase"),
))
dtsSchemaDocument.referencesDocument[lbDoc] = ModelDocumentReference("href", lbRefElt)
# label linkbase
for lang, filename in labelLinkbases:
lbDoc = ModelDocument.create(dts, ModelDocument.Type.LINKBASE, filename, base="", initialXml="""
<linkbase
xmlns="http://www.xbrl.org/2003/linkbase"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.xbrl.org/2003/linkbase
http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrli="http://www.xbrl.org/2003/instance"/>
""")
lbDoc.inDTS = True
addLinkbaseRef("label", filename, lbDoc)
lbElt = lbDoc.xmlRootElement
linkElt = XmlUtil.addChild(lbElt,
XbrlConst.link, "labelLink",
attributes=(("{http://www.w3.org/1999/xlink}type", "extended"),
("{http://www.w3.org/1999/xlink}role", "http://www.xbrl.org/2003/role/link")))
firstLinkElt = linkElt
locs = set()
roleRefs = set()
for labelKey, text in extensionLabels.items():
prefix, name, labelLang, role = labelKey
if lang == labelLang:
locLabel = prefix + "_" + name
if locLabel not in locs:
locs.add(locLabel)
XmlUtil.addChild(linkElt,
XbrlConst.link, "loc",
attributes=(("{http://www.w3.org/1999/xlink}type", "locator"),
("{http://www.w3.org/1999/xlink}href", extensionHref(prefix, name)),
("{http://www.w3.org/1999/xlink}label", locLabel)))
XmlUtil.addChild(linkElt,
XbrlConst.link, "labelArc",
attributes=(("{http://www.w3.org/1999/xlink}type", "arc"),
("{http://www.w3.org/1999/xlink}arcrole", "http://www.xbrl.org/2003/arcrole/concept-label"),
("{http://www.w3.org/1999/xlink}from", locLabel),
("{http://www.w3.org/1999/xlink}to", "label_" + locLabel),
("order", 1.0)))
XmlUtil.addChild(linkElt,
XbrlConst.link, "label",
attributes=(("{http://www.w3.org/1999/xlink}type", "resource"),
("{http://www.w3.org/1999/xlink}label", "label_" + locLabel),
("{http://www.w3.org/1999/xlink}role", role),
("{http://www.w3.org/XML/1998/namespace}lang", lang)),
text=text)
if role:
if role in dts.roleTypes:
roleType = dts.roleTypes[role][0]
roleRefs.add(("roleRef", role, roleType.modelDocument.uri + "#" + roleType.id))
elif role.startswith("http://www.xbrl.org/2009/role/negated"):
roleRefs.add(("roleRef", role, "http://www.xbrl.org/lrr/role/negated-2009-12-16.xsd#" + role.rpartition("/")[2]))
# add arcrole references
for roleref, roleURI, href in roleRefs:
XmlUtil.addChild(lbElt,
XbrlConst.link, roleref,
attributes=(("arcroleURI" if roleref == "arcroleRef" else "roleURI", roleURI),
("{http://www.w3.org/1999/xlink}type", "simple"),
("{http://www.w3.org/1999/xlink}href", href)),
beforeSibling=firstLinkElt)
lbDoc.linkbaseDiscover(lbElt)
def hrefConcept(prefix, name):
qn = schemaElt.prefixedNameQname(prefix + ":" + name)
if qn in dts.qnameConcepts:
return dts.qnameConcepts[qn]
return None
def lbTreeWalk(lbType, parentElt, lbStruct, roleRefs, locs=None, arcsFromTo=None, fromPrefix=None, fromName=None):
order = 1.0
for lbEntry in lbStruct:
if lbEntry.isELR:
role = "unspecified"
if lbEntry.role and lbEntry.role.startswith("http://"): # have a role specified
role = lbEntry.role
elif lbEntry.name: #may be a definition
for linkroleUri, modelRoleTypes in dts.roleTypes.items():
definition = modelRoleTypes[0].definition
if lbEntry.name == definition:
role = linkroleUri
break
if role != XbrlConst.defaultLinkRole and role in dts.roleTypes: # add roleRef
roleType = modelRoleTypes[0]
roleRefs.add(("roleRef", role, roleType.modelDocument.uri + "#" + roleType.id))
linkElt = XmlUtil.addChild(parentElt,
XbrlConst.link, lbType + "Link",
attributes=(("{http://www.w3.org/1999/xlink}type", "extended"),
("{http://www.w3.org/1999/xlink}role", role)))
locs = set()
arcsFromTo = set()
lbTreeWalk(lbType, linkElt, lbEntry.childStruct, roleRefs, locs, arcsFromTo)
else:
toPrefix = lbEntry.prefix
toName = lbEntry.name
toHref = extensionHref(toPrefix, toName)
toLabel = toPrefix + "_" + toName
toLabelAlt = None
if not lbEntry.isRoot:
fromLabel = fromPrefix + "_" + fromName
if (fromLabel, toLabel) in arcsFromTo:
# need extra loc to prevent arc from/to duplication in ELR
for i in range(1, 1000):
toLabelAlt = "{}_{}".format(toLabel, i)
if (fromLabel, toLabelAlt) not in arcsFromTo:
toLabel = toLabelAlt
break
if toHref not in locs or toLabelAlt:
XmlUtil.addChild(parentElt,
XbrlConst.link, "loc",
attributes=(("{http://www.w3.org/1999/xlink}type", "locator"),
("{http://www.w3.org/1999/xlink}href", toHref),
("{http://www.w3.org/1999/xlink}label", toLabel)))
locs.add(toHref)
if not lbEntry.isRoot:
arcsFromTo.add( (fromLabel, toLabel) )
if lbType == "calculation" and lbEntry.weight is not None:
otherAttrs = ( ("weight", lbEntry.weight), )
elif lbType == "presentation" and lbEntry.role:
otherAttrs = ( ("preferredLabel", lbEntry.role), )
if lbEntry.role and lbEntry.role in dts.roleTypes:
roleType = dts.roleTypes[lbEntry.role][0]
roleRefs.add(("roleRef", lbEntry.role, roleType.modelDocument.uri + "#" + roleType.id))
else:
otherAttrs = ( )
if lbEntry.arcrole == "_dimensions_": # pick proper consecutive arcrole
fromConcept = hrefConcept(fromPrefix, fromName)
toConcept = hrefConcept(toPrefix, toName)
if toConcept is not None and toConcept.isHypercubeItem:
arcrole = XbrlConst.all
otherAttrs += ( (XbrlConst.qnXbrldtContextElement, "segment"), )
elif toConcept is not None and toConcept.isDimensionItem:
arcrole = XbrlConst.hypercubeDimension
elif fromConcept is not None and fromConcept.isDimensionItem:
arcrole = XbrlConst.dimensionDomain
else:
arcrole = XbrlConst.domainMember
else:
arcrole = lbEntry.arcrole
XmlUtil.addChild(parentElt,
XbrlConst.link, lbType + "Arc",
attributes=(("{http://www.w3.org/1999/xlink}type", "arc"),
("{http://www.w3.org/1999/xlink}arcrole", arcrole),
("{http://www.w3.org/1999/xlink}from", fromLabel),
("{http://www.w3.org/1999/xlink}to", toLabel),
("order", order)) + otherAttrs )
order += 1.0
if lbType != "calculation" or lbEntry.isRoot:
lbTreeWalk(lbType, parentElt, lbEntry.childStruct, roleRefs, locs, arcsFromTo, toPrefix, toName)
for hasLB, lbType, lbLB in ((hasPreLB, "presentation", preLB),
(hasDefLB, "definition", defLB),
(hasCalLB, "calculation", calLB)):
if hasLB:
for lbRefType, filename in linkbaseRefs:
if lbType == lbRefType:
# output presentation linkbase
lbDoc = ModelDocument.create(dts, ModelDocument.Type.LINKBASE, filename, base='', initialXml="""
<linkbase
xmlns="http://www.xbrl.org/2003/linkbase"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.xbrl.org/2003/linkbase
http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrli="http://www.xbrl.org/2003/instance"/>
""")
lbDoc.inDTS = True
addLinkbaseRef(lbRefType, filename, lbDoc)
lbElt = lbDoc.xmlRootElement
roleRefs = set()
if lbType == "definition":
roleRefs.update((("arcroleRef", XbrlConst.all, "http://www.xbrl.org/2005/xbrldt-2005.xsd#all"),
("arcroleRef", XbrlConst.dimensionDefault, "http://www.xbrl.org/2005/xbrldt-2005.xsd#dimension-default"),
("arcroleRef", XbrlConst.dimensionDomain, "http://www.xbrl.org/2005/xbrldt-2005.xsd#dimension-domain"),
("arcroleRef", XbrlConst.domainMember, "http://www.xbrl.org/2005/xbrldt-2005.xsd#domain-member"),
("arcroleRef", XbrlConst.hypercubeDimension, "http://www.xbrl.org/2005/xbrldt-2005.xsd#hypercube-dimension")))
lbTreeWalk(lbType, lbElt, lbLB, roleRefs)
firstLinkElt = None
for firstLinkElt in lbElt.iterchildren():
break
# add arcrole references
for roleref, roleURI, href in roleRefs:
XmlUtil.addChild(lbElt,
XbrlConst.link, roleref,
attributes=(("arcroleURI" if roleref == "arcroleRef" else "roleURI", roleURI),
("{http://www.w3.org/1999/xlink}type", "simple"),
("{http://www.w3.org/1999/xlink}href", href)),
beforeSibling=firstLinkElt)
lbDoc.linkbaseDiscover(lbElt)
break
#cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt),
# messageCode="loadFromExcel:info")
if priorCWD:
os.chdir(priorCWD) # restore prior current working directory
return dts
def init(modelXbrl):
# setup modelXbrl for rendering evaluation
# dimension defaults required in advance of validation
from arelle import ValidateXbrlDimensions, ValidateFormula, ModelDocument
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
hasXbrlTables = False
# validate table linkbase dimensions
for baseSetKey in modelXbrl.baseSets.keys():
arcrole, ELR, linkqname, arcqname = baseSetKey
if ELR and linkqname and arcqname and XbrlConst.isTableRenderingArcrole(arcrole):
ValidateFormula.checkBaseSet(modelXbrl, arcrole, ELR, modelXbrl.relationshipSet(arcrole,ELR,linkqname,arcqname))
if arcrole in (XbrlConst.tableBreakdown, XbrlConst.tableBreakdownMMDD, XbrlConst.tableBreakdown201305, XbrlConst.tableBreakdown201301, XbrlConst.tableAxis2011):
hasXbrlTables = True
# provide context for view
if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE:
instance = None # use instance of the entry pont
else: # need dummy instance
instance = ModelDocument.create(modelXbrl, ModelDocument.Type.INSTANCE,
"dummy.xml", # fake URI and fake schemaRef
("http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd",))
if hasXbrlTables:
# formula processor is needed for 2011 XBRL tables but not for 2010 Eurofiling tables
modelXbrl.rendrCntx = XPathContext.create(modelXbrl, instance)
modelXbrl.profileStat(None)
# setup fresh parameters from formula options
modelXbrl.parameters = modelXbrl.modelManager.formulaOptions.typedParameters()
# validate parameters and custom function signatures
ValidateFormula.validate(modelXbrl, xpathContext=modelXbrl.rendrCntx, parametersOnly=True, statusMsg=_("compiling rendering tables"))
# deprecated as of 2013-05-17
# check and extract message expressions into compilable programs
for msgArcrole in (XbrlConst.tableDefinitionNodeMessage201301, XbrlConst.tableDefinitionNodeSelectionMessage201301,
XbrlConst.tableAxisMessage2011, XbrlConst.tableAxisSelectionMessage2011):
for msgRel in modelXbrl.relationshipSet(msgArcrole).modelRelationships:
ValidateFormula.checkMessageExpressions(modelXbrl, msgRel.toModelObject)
# compile and validate tables
for modelTable in modelXbrl.modelRenderingTables:
modelTable.fromInstanceQnames = None # required if referred to by variables scope chaining
modelTable.compile()
hasNsWithAspectModel = modelTable.namespaceURI in (XbrlConst.euRend, XbrlConst.table2011, XbrlConst.table201301, XbrlConst.table201305)
# check aspectModel (attribute removed 2013-06, now always dimensional)
if modelTable.aspectModel not in ("non-dimensional", "dimensional") and hasNsWithAspectModel:
modelXbrl.error("xbrlte:unknownAspectModel",
_("Table %(xlinkLabel)s, aspect model %(aspectModel)s not recognized"),
modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel, aspectModel=modelTable.aspectModel)
else:
modelTable.priorAspectAxisDisposition = {}
# check ordinate aspects against aspectModel
oppositeAspectModel = (_DICT_SET({'dimensional','non-dimensional'}) - _DICT_SET({modelTable.aspectModel})).pop()
if hasNsWithAspectModel:
uncoverableAspects = aspectModels[oppositeAspectModel] - aspectModels[modelTable.aspectModel]
else:
uncoverableAspects = ()
aspectsCovered = set()
for tblAxisRel in modelXbrl.relationshipSet((XbrlConst.tableBreakdown, XbrlConst.tableBreakdownMMDD, XbrlConst.tableBreakdown201305, XbrlConst.tableBreakdown201301,XbrlConst.tableAxis2011)).fromModelObject(modelTable):
breakdownAspectsCovered = set()
hasCoveredAspect = checkBreakdownDefinitionNode(modelXbrl, modelTable, tblAxisRel, tblAxisRel.axisDisposition, uncoverableAspects, breakdownAspectsCovered)
''' removed 2013-10
if not hasCoveredAspect:
definitionNode = tblAxisRel.toModelObject
modelXbrl.error("xbrlte:breakdownDefinesNoAspects",
_("Breakdown %(xlinkLabel)s has no participating aspects"),
modelObject=(modelTable,definitionNode), xlinkLabel=definitionNode.xlinkLabel, axis=definitionNode.localName)
'''
aspectsCovered |= breakdownAspectsCovered
checkBreakdownLeafNodeAspects(modelXbrl, modelTable, tblAxisRel, set(), breakdownAspectsCovered)
if Aspect.CONCEPT not in aspectsCovered and not hasNsWithAspectModel:
modelXbrl.error("xbrlte:tableMissingConceptAspect",
_("Table %(xlinkLabel)s does not include the concept aspect as one of its participating aspects"),
modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel)
del modelTable.priorAspectAxisDisposition
# check for table-parameter name clash
parameterNames = {}
for tblParamRel in modelXbrl.relationshipSet((XbrlConst.tableParameter, XbrlConst.tableParameterMMDD)).fromModelObject(modelTable):
parameterName = tblParamRel.variableQname
if parameterName in parameterNames:
modelXbrl.error("xbrlte:tableParameterNameClash ",
_("Table %(xlinkLabel)s has parameter name clash for variable %(name)s"),
modelObject=(modelTable,tblParamRel,parameterNames[parameterName]), xlinkLabel=modelTable.xlinkLabel, name=parameterName)
else:
parameterNames[parameterName] = tblParamRel
modelXbrl.profileStat(_("compileTables"))
def init(modelXbrl):
# setup modelXbrl for rendering evaluation
# dimension defaults required in advance of validation
from arelle import ValidateXbrlDimensions, ValidateFormula, ModelDocument
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
hasXbrlTables = False
# validate table linkbase dimensions
for baseSetKey in modelXbrl.baseSets.keys():
arcrole, ELR, linkqname, arcqname = baseSetKey
if ELR and linkqname and arcqname and XbrlConst.isTableRenderingArcrole(
arcrole):
ValidateFormula.checkBaseSet(
modelXbrl, arcrole, ELR,
modelXbrl.relationshipSet(arcrole, ELR, linkqname, arcqname))
if arcrole in (XbrlConst.tableBreakdown,
XbrlConst.tableBreakdownMMDD,
XbrlConst.tableBreakdown201305,
XbrlConst.tableBreakdown201301,
XbrlConst.tableAxis2011):
hasXbrlTables = True
# provide context for view
if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE:
instance = None # use instance of the entry pont
else: # need dummy instance
instance = ModelDocument.create(
modelXbrl,
ModelDocument.Type.INSTANCE,
"dummy.xml", # fake URI and fake schemaRef
("http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd", ))
if hasXbrlTables:
# formula processor is needed for 2011 XBRL tables but not for 2010 Eurofiling tables
modelXbrl.rendrCntx = XPathContext.create(modelXbrl, instance)
modelXbrl.profileStat(None)
# setup fresh parameters from formula options
modelXbrl.parameters = modelXbrl.modelManager.formulaOptions.typedParameters(
)
# validate parameters and custom function signatures
ValidateFormula.validate(modelXbrl,
xpathContext=modelXbrl.rendrCntx,
parametersOnly=True,
statusMsg=_("compiling rendering tables"))
# deprecated as of 2013-05-17
# check and extract message expressions into compilable programs
for msgArcrole in (XbrlConst.tableDefinitionNodeMessage201301,
XbrlConst.tableDefinitionNodeSelectionMessage201301,
XbrlConst.tableAxisMessage2011,
XbrlConst.tableAxisSelectionMessage2011):
for msgRel in modelXbrl.relationshipSet(
msgArcrole).modelRelationships:
ValidateFormula.checkMessageExpressions(
modelXbrl, msgRel.toModelObject)
# compile and validate tables
for modelTable in modelXbrl.modelRenderingTables:
modelTable.fromInstanceQnames = None # required if referred to by variables scope chaining
modelTable.compile()
hasNsWithAspectModel = modelTable.namespaceURI in (
XbrlConst.euRend, XbrlConst.table2011, XbrlConst.table201301,
XbrlConst.table201305)
# check aspectModel (attribute removed 2013-06, now always dimensional)
if modelTable.aspectModel not in (
"non-dimensional", "dimensional") and hasNsWithAspectModel:
modelXbrl.error(
"xbrlte:unknownAspectModel",
_("Table %(xlinkLabel)s, aspect model %(aspectModel)s not recognized"
),
modelObject=modelTable,
xlinkLabel=modelTable.xlinkLabel,
aspectModel=modelTable.aspectModel)
else:
modelTable.priorAspectAxisDisposition = {}
# check ordinate aspects against aspectModel
oppositeAspectModel = (
_DICT_SET({'dimensional', 'non-dimensional'}) -
_DICT_SET({modelTable.aspectModel})).pop()
if hasNsWithAspectModel:
uncoverableAspects = aspectModels[
oppositeAspectModel] - aspectModels[
modelTable.aspectModel]
else:
uncoverableAspects = ()
aspectsCovered = set()
for tblAxisRel in modelXbrl.relationshipSet(
(XbrlConst.tableBreakdown, XbrlConst.tableBreakdownMMDD,
XbrlConst.tableBreakdown201305,
XbrlConst.tableBreakdown201301,
XbrlConst.tableAxis2011)).fromModelObject(modelTable):
breakdownAspectsCovered = set()
hasCoveredAspect = checkBreakdownDefinitionNode(
modelXbrl, modelTable, tblAxisRel,
tblAxisRel.axisDisposition, uncoverableAspects,
breakdownAspectsCovered)
''' removed 2013-10
if not hasCoveredAspect:
definitionNode = tblAxisRel.toModelObject
modelXbrl.error("xbrlte:breakdownDefinesNoAspects",
_("Breakdown %(xlinkLabel)s has no participating aspects"),
modelObject=(modelTable,definitionNode), xlinkLabel=definitionNode.xlinkLabel, axis=definitionNode.localName)
'''
aspectsCovered |= breakdownAspectsCovered
checkBreakdownLeafNodeAspects(modelXbrl, modelTable,
tblAxisRel, set(),
breakdownAspectsCovered)
if Aspect.CONCEPT not in aspectsCovered and not hasNsWithAspectModel:
modelXbrl.error(
"xbrlte:tableMissingConceptAspect",
_("Table %(xlinkLabel)s does not include the concept aspect as one of its participating aspects"
),
modelObject=modelTable,
xlinkLabel=modelTable.xlinkLabel)
del modelTable.priorAspectAxisDisposition
# check for table-parameter name clash
parameterNames = {}
for tblParamRel in modelXbrl.relationshipSet(
(XbrlConst.tableParameter, XbrlConst.tableParameterMMDD
)).fromModelObject(modelTable):
parameterName = tblParamRel.variableQname
if parameterName in parameterNames:
modelXbrl.error(
"xbrlte:tableParameterNameClash ",
_("Table %(xlinkLabel)s has parameter name clash for variable %(name)s"
),
modelObject=(modelTable, tblParamRel,
parameterNames[parameterName]),
xlinkLabel=modelTable.xlinkLabel,
name=parameterName)
else:
parameterNames[parameterName] = tblParamRel
modelXbrl.profileStat(_("compileTables"))
def init(modelXbrl):
# setup modelXbrl for rendering evaluation
# dimension defaults required in advance of validation
from arelle import ValidateXbrlDimensions, ValidateFormula, ModelDocument
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
hasXbrlTables = False
# validate table linkbase dimensions
for baseSetKey in modelXbrl.baseSets.keys():
arcrole, ELR, linkqname, arcqname = baseSetKey
if ELR and linkqname and arcqname and XbrlConst.isTableRenderingArcrole(arcrole):
ValidateFormula.checkBaseSet(modelXbrl, arcrole, ELR, modelXbrl.relationshipSet(arcrole,ELR,linkqname,arcqname))
if arcrole in (XbrlConst.tableBreakdown, XbrlConst.tableAxis2011):
hasXbrlTables = True
# provide context for view
if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE:
instance = None # use instance of the entry pont
else: # need dummy instance
instance = ModelDocument.create(modelXbrl, ModelDocument.Type.INSTANCE,
"dummy.xml", # fake URI and fake schemaRef
("http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd",))
if hasXbrlTables:
# formula processor is needed for 2011 XBRL tables but not for 2010 Eurofiling tables
modelXbrl.rendrCntx = XPathContext.create(modelXbrl, instance)
modelXbrl.profileStat(None)
# setup fresh parameters from formula optoins
modelXbrl.parameters = modelXbrl.modelManager.formulaOptions.typedParameters()
# validate parameters and custom function signatures
ValidateFormula.validate(modelXbrl, xpathContext=modelXbrl.rendrCntx, parametersOnly=True, statusMsg=_("compiling rendering tables"))
# check and extract message expressions into compilable programs
for msgArcrole in (XbrlConst.tableDefinitionNodeMessage, XbrlConst.tableDefinitionNodeSelectionMessage,
XbrlConst.tableAxisMessage2011, XbrlConst.tableAxisSelectionMessage2011):
for msgRel in modelXbrl.relationshipSet(msgArcrole).modelRelationships:
ValidateFormula.checkMessageExpressions(modelXbrl, msgRel.toModelObject)
# compile and validate tables
for modelTable in modelXbrl.modelRenderingTables:
modelTable.fromInstanceQnames = None # required if referred to by variables scope chaining
modelTable.compile()
# check aspectModel
if modelTable.aspectModel not in ("non-dimensional", "dimensional"):
modelXbrl.error("xbrlte:unknownAspectModel",
_("Table %(xlinkLabel)s, aspect model %(aspectModel)s not recognized"),
modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel, aspectModel=modelTable.aspectModel)
else:
modelTable.priorAspectAxisDisposition = {}
# check ordinate aspects against aspectModel
oppositeAspectModel = (_DICT_SET({'dimensional','non-dimensional'}) - _DICT_SET({modelTable.aspectModel})).pop()
uncoverableAspects = aspectModels[oppositeAspectModel] - aspectModels[modelTable.aspectModel]
for tblAxisRel in modelXbrl.relationshipSet((XbrlConst.tableBreakdown,XbrlConst.tableAxis2011)).fromModelObject(modelTable):
checkDefinitionNodeAspectModel(modelXbrl, modelTable, tblAxisRel, uncoverableAspects)
del modelTable.priorAspectAxisDisposition
modelXbrl.profileStat(_("compileTables"))
