def loadFromExcel(cntlr, excelFile):
from arelle import xlrd
from arelle.xlrd.sheet import empty_cell
from arelle import ModelDocument, ModelXbrl, XmlUtil
from arelle.ModelDocument import ModelDocumentReference
from arelle.ModelValue import qname
startedAt = time.time()
importExcelBook = xlrd.open_workbook(excelFile)
controlSheet = importExcelBook.sheet_by_index(1)
imports = {"xbrli": ( ("namespace", XbrlConst.xbrli),
("schemaLocation", "http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd") )} # xml of imports
importXmlns = {}
linkbaseRefs = []
labelLinkbases = []
hasPreLB = hasCalLB = hasDefLB = False
# xxxLB structure [ (elr1, def1, "_ELR_", [roots]), (elr2, def2, "_ELR_", [rootw]) ...]
# roots = (rootHref, None, "_root_", [children])
# children = (childPrefix, childName, arcrole, [grandChildren])
preLB = []
defLB = []
calLB = []
def lbDepthList(lbStruct, depth, parentList=None):
if depth == 0:
return lbStruct[-1].childStruct
return lbDepthList(lbStruct[-1].childStruct, depth-1, list)
extensionElements = {}
extensionRoles = {} # key is roleURI, value is role definition
extensionLabels = {} # key = (prefix, name, lang, role), value = label text
def extensionHref(prefix, name):
if prefix == extensionSchemaPrefix:
filename = extensionSchemaFilename
elif prefix in imports:
filename = imports[prefix][1][1]
else:
return None
return "{0}#{1}_{2}".format(filename, prefix, name)
isUSGAAP = False
for iRow in range(1, controlSheet.nrows):
try:
row = controlSheet.row(iRow)
if (row[0].ctype == xlrd.XL_CELL_EMPTY): # skip if col 1 is empty
continue
action = row[0].value
filetype = row[1].value
prefix = row[2].value
filename = row[3].value
namespaceURI = row[4].value
lbType = lang = None
if action == "import":
imports[prefix] = ( ("namespace", namespaceURI), ("schemaLocation", filename) )
importXmlns[prefix] = namespaceURI
if re.match(r"http://[^/]+/us-gaap/", namespaceURI):
isUSGAAP = True
elif action == "extension":
if filetype == "schema":
extensionSchemaPrefix = prefix
extensionSchemaFilename = filename
extensionSchemaNamespaceURI = namespaceURI
elif filetype == "linkbase":
typeLang = prefix.split()
if len(typeLang) > 0:
lbType = typeLang[0]
else:
lbType = "unknown"
if len(typeLang) > 1:
lang = typeLang[1]
else:
lang = "en"
if lbType == "label":
labelLinkbases.append((lang, filename))
elif lbType == "presentation":
hasPreLB = True
elif lbType == "definition":
hasDefLB = True
elif lbType == "calculation":
hasCalLB = True
linkbaseRefs.append( (lbType, filename) )
elif filetype == "role" and namespaceURI:
extensionRoles[namespaceURI] = filename
except Exception as err:
cntlr.addToLog("Exception: {error}, Excel row: {excelRow}"
.format(error=err,
excelRow=iRow),
messageCode="importExcel:exception")
importExcelSheet = importExcelBook.sheet_by_index(0)
# find column headers row
headerCols = {}
headerRows = set()
# find out which rows are header rows
for iRow in range(0, importExcelSheet.nrows):
row = importExcelSheet.row(iRow)
for iCol, colCell in enumerate(row):
v = colCell.value
if v in importColumnHeaders:
headerCols[importColumnHeaders[v]] = iCol
if all(colName in headerCols
for colName in ("name", "type", "depth")): # must have these to be a header col
# it's a header col
headerRows.add(iRow)
headerCols.clear()
def cellValue(row, header):
if header in headerCols:
iCol = headerCols[header]
if iCol < len(row):
return row[iCol].value
return ''
def checkImport(qname):
prefix, sep, localName = qname.partition(":")
if sep:
if prefix not in imports:
if prefix == "xbrldt":
imports["xbrldt"] = ("namespace", XbrlConst.xbrldt), ("schemaLocation", "http://www.xbrl.org/2005/xbrldt-2005.xsd")
elif prefix == "nonnum":
imports["nonnum"] = ("namespace", "http://www.xbrl.org/dtr/type/non-numeric"), ("schemaLocation", "http://www.xbrl.org/dtr/type/nonNumeric-2009-12-16.xsd")
else:
cntlr.addToLog("Warning: prefix schema file is not imported for: {qname}"
.format(qname=qname),
messageCode="importExcel:warning")
# find header rows
currentELR = currentELRdefinition = None
for iRow in range(0, importExcelSheet.nrows):
useLabels = False
try:
row = importExcelSheet.row(iRow)
isHeaderRow = iRow in headerRows
isELRrow = (iRow + 1) in headerRows
if isHeaderRow:
headerCols.clear()
for iCol, colCell in enumerate(row):
v = colCell.value
if v in importColumnHeaders:
headerCols[importColumnHeaders[v]] = iCol
elif isELRrow:
currentELR = currentELRdefinition = None
for colCell in row:
v = colCell.value
if v.startswith("http://"):
currentELR = v
elif not currentELRdefinition and v.endswith(" 科目一覧"):
currentELRdefinition = v[0:-5]
elif not currentELRdefinition:
currentELRdefinition = v
if currentELR or currentELRdefinition:
if hasPreLB:
preLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
if hasDefLB:
defLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
if hasCalLB:
calLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
elif headerCols:
prefix = cellValue(row, 'prefix').strip()
name = cellValue(row, 'name').strip()
if "depth" in headerCols:
try:
depth = int(cellValue(row, 'depth'))
except ValueError:
depth = None
else:
depth = None
if prefix == extensionSchemaPrefix and name not in extensionElements:
# elements row
eltType = cellValue(row, 'type')
subsGrp = cellValue(row, 'substitutionGroup')
abstract = cellValue(row, 'abstract')
nillable = cellValue(row, 'nillable')
balance = cellValue(row, 'balance')
periodType = cellValue(row, 'periodType')
newElt = [ ("name", name), ("id", prefix + "_" + name) ]
if eltType:
newElt.append( ("type", eltType) )
checkImport(eltType)
if subsGrp:
newElt.append( ("substitutionGroup", subsGrp) )
checkImport(subsGrp)
if abstract:
newElt.append( ("abstract", abstract) )
if nillable:
newElt.append( ("nillable", nillable) )
if balance:
newElt.append( ("{http://www.xbrl.org/2003/instance}balance", balance) )
if periodType:
newElt.append( ("{http://www.xbrl.org/2003/instance}periodType", periodType) )
extensionElements[name] = newElt
useLabels = True
if depth is not None:
if hasPreLB:
entryList = lbDepthList(preLB, depth)
preferredLabel = cellValue(row, 'preferredLabel')
if preferredLabel and not preferredLabel.startswith("http://"):
preferredLabel = "http://www.xbrl.org/2003/role/" + preferredLabel
if entryList is not None:
if depth == 0:
entryList.append( LBentry(prefix=prefix, name=name, isRoot=True) )
else:
entryList.append( LBentry(prefix=prefix, name=name, arcrole=XbrlConst.parentChild,
role=preferredLabel) )
if hasDefLB:
entryList = lbDepthList(defLB, depth)
if entryList is not None:
if depth == 0:
entryList.append( LBentry(prefix=prefix, name=name, isRoot=True) )
else:
entryList.append( LBentry(prefix=prefix, name=name, arcrole="_dimensions_") )
if hasCalLB:
calcParent = cellValue(row, 'calculationParent')
calcWeight = cellValue(row, 'calculationWeight')
if calcParent and calcWeight:
calcParentPrefix, sep, calcParentName = calcParent.partition(":")
entryList = lbDepthList(calLB, 0)
if entryList is not None:
entryList.append( LBentry(prefix=calcParentPrefix, name=calcParentName, isRoot=True, childStruct=
[LBentry(prefix=prefix, name=name, arcrole=XbrlConst.summationItem, weight=calcWeight )]) )
# accumulate extension labels
if useLabels:
prefix = cellValue(row, 'prefix').strip()
name = cellValue(row, 'name').strip()
preferredLabel = cellValue(row, 'preferredLabel')
if preferredLabel and not preferredLabel.startswith("http://"):
preferredLabel = "http://www.xbrl.org/2003/role/" + preferredLabel
for colItem, iCol in headerCols.items():
if isinstance(colItem, tuple):
colItemType, role, lang = colItem
cell = row[iCol]
if cell.ctype == xlrd.XL_CELL_EMPTY:
values = ()
elif colItemType == "label":
values = (cell.value,)
elif colItemType == "labels":
values = cell.value.split('\n')
else:
values = ()
if preferredLabel: # first label column sets preferredLabel if any
role = preferredLabel
preferredLabel = None
for value in values:
extensionLabels[prefix, name, lang, role] = value.strip()
except Exception as err:
cntlr.addToLog("Exception: {error}, Excel row: {excelRow}"
.format(error=err,
excelRow=iRow),
messageCode="importExcel:exception")
if isUSGAAP and hasDefLB:
# move line items above table
def fixUsggapTableDims(lvl1Struct):
foundLineItems = False
for i1, lvl1Entry in enumerate(lvl1Struct):
for i2, lvl2Entry in enumerate(lvl1Entry.childStruct):
for i3, lvl3Entry in enumerate(lvl2Entry.childStruct):
if lvl3Entry.name.endswith("LineItems") and lvl2Entry.name.endswith("Table"):
foundLineItems = True
break
if foundLineItems:
break
else:
fixUsggapTableDims(lvl1Entry.childStruct)
if foundLineItems:
lvl1Struct.insert(i1 + 1, LBentry(prefix=lvl3Entry.prefix, name=lvl3Entry.name, arcrole=lvl1Entry.arcrole, childStruct=lvl3Entry.childStruct)) # must keep lvl1Rel if it is __root__
lvl3Entry.childStruct.insert(0, lvl2Entry)
if lvl1Entry.name.endswith("Abstract"):
del lvl1Struct[i1]
del lvl2Entry.childStruct[i3]
pass
fixUsggapTableDims(defLB)
dts = cntlr.modelManager.create(newDocumentType=ModelDocument.Type.SCHEMA,
url=extensionSchemaFilename,
isEntry=True,
base='', # block pathname from becomming absolute
initialXml='''
<schema xmlns="http://www.w3.org/2001/XMLSchema"
targetNamespace="{targetNamespace}"
attributeFormDefault="unqualified"
elementFormDefault="qualified"
xmlns:xsd="http://www.w3.org/2001/XMLSchema"
xmlns:{extensionPrefix}="{targetNamespace}"
{importXmlns}
xmlns:iod="http://disclosure.edinet-fsa.go.jp/taxonomy/common/2013-03-31/iod"
xmlns:nonnum="http://www.xbrl.org/dtr/type/non-numeric"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:xbrli="http://www.xbrl.org/2003/instance"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrldt="http://xbrl.org/2005/xbrldt"/>
'''.format(targetNamespace=extensionSchemaNamespaceURI,
extensionPrefix=extensionSchemaPrefix,
importXmlns=''.join('xmlns:{0}="{1}"\n'.format(prefix, namespaceURI)
for prefix, namespaceURI in importXmlns.items())
)
)
dtsSchemaDocument = dts.modelDocument
dtsSchemaDocument.inDTS = True # entry document always in DTS
dtsSchemaDocument.targetNamespace = extensionSchemaNamespaceURI # not set until schemaDiscover too late otherwise
schemaElt = dtsSchemaDocument.xmlRootElement
#foreach linkbase
annotationElt = XmlUtil.addChild(schemaElt, XbrlConst.xsd, "annotation")
appinfoElt = XmlUtil.addChild(annotationElt, XbrlConst.xsd, "appinfo")
for iRow in range(0, importExcelSheet.nrows):
try:
row = importExcelSheet.row(iRow)
if (row[0].ctype == xlrd.XL_CELL_EMPTY): # skip if col 1 is empty
continue
testDir = row[0].value
uriFrom = row[1].value
uriTo = row[2].value
except Exception as err:
cntlr.addToLog("Exception: {error}, Excel row: {excelRow}"
.format(error=err,
excelRow=iRow),
messageCode="loadFromExcel:exception")
# add linkbaseRefs
appinfoElt = XmlUtil.descendant(schemaElt, XbrlConst.xsd, "appinfo")
# don't yet add linkbase refs, want to process imports first to get roleType definitions
# add imports
for importAttributes in sorted(imports.values()):
XmlUtil.addChild(schemaElt,
XbrlConst.xsd, "import",
attributes=importAttributes)
# add elements
for eltName, eltAttrs in sorted(extensionElements.items(), key=lambda item: item[0]):
XmlUtil.addChild(schemaElt,
XbrlConst.xsd, "element",
attributes=eltAttrs)
# add role definitions (for discovery)
for roleURI, roleDefinition in extensionRoles.items():
roleElt = XmlUtil.addChild(appinfoElt, XbrlConst.link, "roleType",
attributes=(("roleURI", roleURI),
("id", "roleType_" + roleURI.rpartition("/")[2])))
if roleDefinition:
XmlUtil.addChild(roleElt, XbrlConst.link, "definition", text=roleDefinition)
if hasPreLB:
XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:presentationLink")
if hasDefLB:
XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:definitionLink")
if hasCalLB:
XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:calculationLink")
dtsSchemaDocument.schemaDiscover(schemaElt, False, extensionSchemaNamespaceURI)
def addLinkbaseRef(lbType, lbFilename, lbDoc):
role = "http://www.xbrl.org/2003/role/{0}LinkbaseRef".format(lbType)
lbRefElt = XmlUtil.addChild(appinfoElt, XbrlConst.link, "linkbaseRef",
attributes=(("{http://www.w3.org/1999/xlink}type", "simple"),
("{http://www.w3.org/1999/xlink}href", lbFilename),
("{http://www.w3.org/1999/xlink}role", role),
("{http://www.w3.org/1999/xlink}arcrole", "http://www.w3.org/1999/xlink/properties/linkbase"),
))
dtsSchemaDocument.referencesDocument[lbDoc] = ModelDocumentReference("href", lbRefElt)
# label linkbase
for lang, filename in labelLinkbases:
lbDoc = ModelDocument.create(dts, ModelDocument.Type.LINKBASE, filename, base="", initialXml="""
<link:linkbase
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.xbrl.org/2003/linkbase
http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrli="http://www.xbrl.org/2003/instance"/>
""")
lbDoc.inDTS = True
addLinkbaseRef("label", filename, lbDoc)
lbElt = lbDoc.xmlRootElement
linkElt = XmlUtil.addChild(lbElt,
XbrlConst.link, "labelLink",
attributes=(("{http://www.w3.org/1999/xlink}type", "extended"),
("{http://www.w3.org/1999/xlink}role", "http://www.xbrl.org/2003/role/link")))
locs = set()
for labelKey, text in extensionLabels.items():
prefix, name, labelLang, role = labelKey
if lang == labelLang:
locLabel = prefix + "_" + name
if locLabel not in locs:
locs.add(locLabel)
XmlUtil.addChild(linkElt,
XbrlConst.link, "loc",
attributes=(("{http://www.w3.org/1999/xlink}type", "locator"),
("{http://www.w3.org/1999/xlink}href", extensionHref(prefix, name)),
("{http://www.w3.org/1999/xlink}label", locLabel)))
XmlUtil.addChild(linkElt,
XbrlConst.link, "labelArc",
attributes=(("{http://www.w3.org/1999/xlink}type", "arc"),
("{http://www.w3.org/1999/xlink}arcrole", "http://www.xbrl.org/2003/arcrole/concept-label"),
("{http://www.w3.org/1999/xlink}from", locLabel),
("{http://www.w3.org/1999/xlink}to", "label_" + locLabel),
("order", 1.0)))
XmlUtil.addChild(linkElt,
XbrlConst.link, "label",
attributes=(("{http://www.w3.org/1999/xlink}type", "resource"),
("{http://www.w3.org/1999/xlink}label", "label_" + locLabel),
("{http://www.w3.org/1999/xlink}role", role),
("{http://www.w3.org/XML/1998/namespace}lang", lang)),
text=text)
lbDoc.linkbaseDiscover(lbElt)
def hrefConcept(prefix, name):
qn = schemaElt.prefixedNameQname(prefix + ":" + name)
if qn in dts.qnameConcepts:
return dts.qnameConcepts[qn]
return None
def lbTreeWalk(lbType, parentElt, lbStruct, roleRefs, locs=None, fromPrefix=None, fromName=None):
order = 1.0
for lbEntry in lbStruct:
if lbEntry.isELR:
role = "unspecified"
if lbEntry.role and lbEntry.role.startswith("http://"): # have a role specified
role = lbEntry.role
elif lbEntry.name: #may be a definition
for linkroleUri, modelRoleTypes in dts.roleTypes.items():
definition = modelRoleTypes[0].definition
if lbEntry.name == definition:
role = linkroleUri
break
if role != XbrlConst.defaultLinkRole and role in dts.roleTypes: # add roleRef
roleType = modelRoleTypes[0]
roleRef = ("roleRef", role, roleType.modelDocument.uri + "#" + roleType.id)
roleRefs.add(roleRef)
linkElt = XmlUtil.addChild(parentElt,
XbrlConst.link, lbType + "Link",
attributes=(("{http://www.w3.org/1999/xlink}type", "extended"),
("{http://www.w3.org/1999/xlink}role", role)))
locs = set()
lbTreeWalk(lbType, linkElt, lbEntry.childStruct, roleRefs, locs)
else:
toPrefix = lbEntry.prefix
toName = lbEntry.name
toHref = extensionHref(toPrefix, toName)
toLabel = toPrefix + "_" + toName
if toHref not in locs:
XmlUtil.addChild(parentElt,
XbrlConst.link, "loc",
attributes=(("{http://www.w3.org/1999/xlink}type", "locator"),
("{http://www.w3.org/1999/xlink}href", toHref),
("{http://www.w3.org/1999/xlink}label", toLabel)))
locs.add(toHref)
if not lbEntry.isRoot:
fromLabel = fromPrefix + "_" + fromName
if lbType == "calculation" and lbEntry.weight is not None:
otherAttrs = ( ("weight", lbEntry.weight), )
elif lbType == "presentation" and lbEntry.role is not None:
otherAttrs = ( ("preferredLabel", lbEntry.role), )
else:
otherAttrs = ( )
if lbEntry.arcrole == "_dimensions_": # pick proper consecutive arcrole
fromConcept = hrefConcept(fromPrefix, fromName)
toConcept = hrefConcept(toPrefix, toName)
if toConcept is not None and toConcept.isHypercubeItem:
arcrole = XbrlConst.all
elif toConcept is not None and toConcept.isDimensionItem:
arcrole = XbrlConst.hypercubeDimension
elif fromConcept is not None and fromConcept.isDimensionItem:
arcrole = XbrlConst.dimensionDomain
else:
arcrole = XbrlConst.domainMember
else:
arcrole = lbEntry.arcrole
XmlUtil.addChild(parentElt,
XbrlConst.link, lbType + "Arc",
attributes=(("{http://www.w3.org/1999/xlink}type", "arc"),
("{http://www.w3.org/1999/xlink}arcrole", arcrole),
("{http://www.w3.org/1999/xlink}from", fromLabel),
("{http://www.w3.org/1999/xlink}to", toLabel),
("order", order)) + otherAttrs )
order += 1.0
if lbType != "calculation" or lbEntry.isRoot:
lbTreeWalk(lbType, parentElt, lbEntry.childStruct, roleRefs, locs, toPrefix, toName)
for hasLB, lbType, lbLB in ((hasPreLB, "presentation", preLB),
(hasDefLB, "definition", defLB),
(hasCalLB, "calculation", calLB)):
if hasLB:
for lbRefType, filename in linkbaseRefs:
if lbType == lbRefType:
# output presentation linkbase
lbDoc = ModelDocument.create(dts, ModelDocument.Type.LINKBASE, filename, base='', initialXml="""
<link:linkbase
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.xbrl.org/2003/linkbase
http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrli="http://www.xbrl.org/2003/instance"/>
""")
lbDoc.inDTS = True
addLinkbaseRef(lbRefType, filename, lbDoc)
lbElt = lbDoc.xmlRootElement
roleRefs = set()
if lbType == "definition":
roleRefs.update((("arcroleRef", XbrlConst.all, "http://www.xbrl.org/2005/xbrldt-2005.xsd#all"),
("arcroleRef", XbrlConst.dimensionDefault, "http://www.xbrl.org/2005/xbrldt-2005.xsd#dimension-default"),
("arcroleRef", XbrlConst.dimensionDomain, "http://www.xbrl.org/2005/xbrldt-2005.xsd#dimension-domain"),
("arcroleRef", XbrlConst.domainMember, "http://www.xbrl.org/2005/xbrldt-2005.xsd#domain-member"),
("arcroleRef", XbrlConst.hypercubeDimension, "http://www.xbrl.org/2005/xbrldt-2005.xsd#hypercube-dimension")))
lbTreeWalk(lbType, lbElt, lbLB, roleRefs)
firstLinkElt = None
for firstLinkElt in lbElt.iterchildren():
break
# add arcrole references
for roleref, roleURI, href in roleRefs:
XmlUtil.addChild(lbElt,
XbrlConst.link, roleref,
attributes=(("arcroleURI" if roleref == "arcroleRef" else "roleURI", roleURI),
("{http://www.w3.org/1999/xlink}type", "simple"),
("{http://www.w3.org/1999/xlink}href", href)),
beforeSibling=firstLinkElt)
lbDoc.linkbaseDiscover(lbElt)
break
#cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt),
# messageCode="loadFromExcel:info")
return dts
def runFromExcel(self, options):
#testGenFileName = options.excelfilename
testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation-index.xls"
testGenDir = os.path.dirname(testGenFileName)
schemaDir = os.path.dirname(testGenDir) + os.sep + "schema"
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
if options.testfiledate:
today = options.testfiledate
else:
today = XmlUtil.dateunionValue(datetime.date.today())
startedAt = time.time()
LogHandler(self) # start logger
self.logMessages = []
logMessagesFile = testGenDir + os.sep + 'log-generation-messages.txt'
modelTestcases = ModelXbrl.create(self.modelManager, url=testGenFileName, isEntry=True)
testcaseIndexBook = xlrd.open_workbook(testGenFileName)
testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0)
self.addToLog(_("[info] xls loaded in {0:.2} secs at {1}").format(time.time() - startedAt, timeNow))
# start index file
indexFiles = [testGenDir + os.sep + 'creation-testcases-index.xml',
testGenDir + os.sep + 'consumption-testcases-index.xml']
indexDocs = []
testcasesElements = []
for purpose in ("Creation","Consumption"):
file = io.StringIO(
#'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- XBRL Versioning 1.0 {0} Tests -->'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>'
'<testcases name="XBRL Versioning 1.0 {0} Tests" '
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">'
'</testcases>'.format(purpose, today)
)
doc = etree.parse(file)
file.close()
indexDocs.append(doc)
testcasesElements.append(doc.getroot())
priorTestcasesDir = None
testcaseFiles = None
testcaseDocs = None
for iRow in range(1, testcaseIndexSheet.nrows):
try:
row = testcaseIndexSheet.row(iRow)
if (row[0].ctype == xlrd.XL_CELL_EMPTY or # must have directory
row[1].ctype == xlrd.XL_CELL_EMPTY or # from
row[2].ctype == xlrd.XL_CELL_EMPTY): # to
continue
testDir = row[0].value
uriFrom = row[1].value
uriTo = row[2].value
overrideReport = row[3].value
description = row[4].value
if description is None or len(description) == 0:
continue # test not ready to run
assignment = row[5].value
expectedEvents = row[6].value # comma space separated if multiple
note = row[7].value
useCase = row[8].value
base = os.path.join(os.path.dirname(testGenFileName),testDir) + os.sep
self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom))
if uriFrom and uriTo and assignment.lower() not in ("n.a.", "error") and expectedEvents != "N.A.":
modelDTSfrom = modelDTSto = None
for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"), True), (uriTo, _("loading to DTS"), False)):
if ',' not in URIs:
modelDTS = ModelXbrl.load(self.modelManager, URIs, msg, base=base)
else:
modelDTS = ModelXbrl.create(self.modelManager,
ModelDocument.Type.DTSENTRIES,
self.webCache.normalizeUrl(URIs.replace(", ","_") + ".dts",
base),
isEntry=True)
DTSdoc = modelDTS.modelDocument
DTSdoc.inDTS = True
for uri in URIs.split(','):
doc = ModelDocument.load(modelDTS, uri.strip(), base=base)
if doc is not None:
DTSdoc.referencesDocument[doc] = "import" #fake import
doc.inDTS = True
if isFrom: modelDTSfrom = modelDTS
else: modelDTSto = modelDTS
if modelDTSfrom is not None and modelDTSto is not None:
# generate differences report
reportUri = uriFrom.partition(',')[0] # first file
reportDir = os.path.dirname(reportUri)
if reportDir: reportDir += os.sep
reportName = os.path.basename(reportUri).replace("from.xsd","report.xml")
reportFile = reportDir + "out" + os.sep + reportName
#reportFile = reportDir + "report" + os.sep + reportName
reportFullPath = self.webCache.normalizeUrl(
reportFile,
base)
testcasesDir = os.path.dirname(os.path.dirname(reportFullPath))
if testcasesDir != priorTestcasesDir:
# close prior report
if priorTestcasesDir:
for i,testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
testcaseName = os.path.basename(testcasesDir)
testcaseFiles = [testcasesDir + os.sep + testcaseName + "-creation-testcase.xml",
testcasesDir + os.sep + testcaseName + "-consumption-testcase.xml"]
for i,testcaseFile in enumerate(testcaseFiles):
etree.SubElement(testcasesElements[i], "testcase",
attrib={"uri":
testcaseFile[len(testGenDir)+1:].replace("\\","/")} )
# start testcase file
testcaseDocs = []
testcaseElements = []
testcaseNumber = testcaseName[0:4]
if testcaseNumber.isnumeric():
testcaseNumberElement = "<number>{0}</number>".format(testcaseNumber)
testcaseName = testcaseName[5:]
else:
testcaseNumberElement = ""
testDirSegments = testDir.split('/')
if len(testDirSegments) >= 2 and '-' in testDirSegments[1]:
testedModule = testDirSegments[1][testDirSegments[1].index('-') + 1:]
else:
testedModule = ''
for purpose in ("Creation","Consumption"):
file = io.StringIO(
#'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>'
'<testcase '
' xmlns="http://xbrl.org/2008/conformance"'
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">'
'<creator>'
'<name>Roland Hommes</name>'
'<email>[email protected]</email>'
'</creator>'
'{0}'
'<name>{1}</name>'
# '<description>{0}</description>'
'<reference>'
'{2}'
'{3}'
'</reference>'
'</testcase>'.format(testcaseNumberElement,
testcaseName,
'<name>{0}</name>'.format(testedModule) if testedModule else '',
'<id>{0}</id>'.format(useCase) if useCase else '')
)
doc = etree.parse(file)
file.close()
testcaseDocs.append(doc)
testcaseElements.append(doc.getroot())
priorTestcasesDir = testcasesDir
variationSeq = 1
try:
os.makedirs(os.path.dirname(reportFullPath))
except WindowsError:
pass # dir already exists
modelVersReport = ModelVersReport.ModelVersReport(modelTestcases)
modelVersReport.diffDTSes(reportFullPath,modelDTSfrom, modelDTSto,
assignment=assignment,
schemaDir=schemaDir)
# check for expected elements
if expectedEvents:
for expectedEvent in expectedEvents.split(","):
if expectedEvent not in ("No change", "N.A."):
prefix, sep, localName = expectedEvent.partition(':')
if sep and len(modelVersReport.xmlDocument.findall(
'//{{{0}}}{1}'.format(
XbrlConst.verPrefixNS.get(prefix),
localName))) == 0:
modelTestcases.warning("warning",
"Generated test case %(reportName)s missing expected event %(event)s",
reportName=reportName,
event=expectedEvent)
modelVersReport.close()
uriFromParts = uriFrom.split('_')
if len(uriFromParts) >= 2:
variationId = uriFromParts[1]
else:
variationId = "_{0:02n}".format(variationSeq)
for i,testcaseElt in enumerate(testcaseElements):
variationElement = etree.SubElement(testcaseElt, "{http://xbrl.org/2008/conformance}variation",
attrib={"id": variationId})
nameElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}description")
nameElement.text = description
''' (removed per RH 2011/10/04
if note:
paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}description")
paramElement.text = "Note: " + note
if useCase:
paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}reference")
paramElement.set("specification", "versioning-requirements")
paramElement.set("useCase", useCase)
'''
dataElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}data")
if i == 0: # result is report
if expectedEvents:
paramElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}parameter",
attrib={"name":"expectedEvent",
"value":expectedEvents.replace(',',' ')},
nsmap={"conf":"http://xbrl.org/2008/conformance",
None:""})
if assignment:
paramElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}parameter",
attrib={"name":"assignment",
"value":assignment},
nsmap={"conf":"http://xbrl.org/2008/conformance",
None:""})
for schemaURIs, dtsAttr in ((uriFrom,"from"), (uriTo,"to")):
for schemaURI in schemaURIs.split(","):
schemaElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}schema")
schemaElement.set("dts",dtsAttr)
if i == 0:
schemaElement.set("readMeFirst","true")
schemaElement.text=os.path.basename(schemaURI.strip())
resultElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}result")
reportElement = etree.SubElement(resultElement if i == 0 else dataElement,
"{http://xbrl.org/2008/conformance}versioningReport")
if i == 1:
reportElement.set("readMeFirst","true")
reportElement.text = "report/" + reportName
variationSeq += 1
except Exception as err:
modelTestcases.error("exception",
_("Exception: %(error)s, Excel row: %(excelRow)s"),
error=err,
excelRow=iRow,
exc_info=True)
# add tests-error-code index files to consumption
for testcaseFile in self.testcaseFiles(testGenDir + os.sep + "tests-error-code"):
etree.SubElement(testcasesElements[1], "testcase",
attrib={"uri":
testcaseFile[len(testGenDir)+1:].replace("\\","/")} )
with open(logMessagesFile, "w") as fh:
fh.writelines(self.logMessages)
if priorTestcasesDir:
for i,testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
for i,indexFile in enumerate(indexFiles):
with open(indexFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def runFromExcel(self, options):
testGenFileName = options.excelfilename
#testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation\1000-2000-index.xls"
testGenDir = os.path.dirname(testGenFileName)
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
if options.testfiledate:
today = options.testfiledate
else:
today = XmlUtil.dateunionValue(datetime.date.today())
startedAt = time.time()
self.logMessages = []
logMessagesFile = testGenDir + os.sep + 'logGenerationMessages.txt'
modelTestcases = ModelXbrl.create(self.modelManager)
testcaseIndexBook = xlrd.open_workbook(testGenFileName)
testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0)
self.addToLog(
_("[info] xls loaded in {0:.2} secs at {1}").format(
time.time() - startedAt, timeNow))
# start index file
indexFiles = [
testGenDir + os.sep + 'creationTestcasesIndex.xml',
testGenDir + os.sep + 'consumptionTestcasesIndex.xml'
]
indexDocs = [
xml.dom.minidom.parseString(
'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- XBRL Versioning 1.0 {0} Tests -->'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>'
'<testcases name="XBRL Versioning 1.0 Consumption Tests" date="{1}" '
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">'
'</testcases>'.format(purpose, today))
for purpose in ("Creation", "Consumption")
]
testcasesElements = [
XmlUtil.child(indexDoc, None, "testcases")
for indexDoc in indexDocs
]
priorTestcasesDir = None
testcaseFiles = None
testcaseDocs = None
for iRow in range(1, testcaseIndexSheet.nrows):
row = testcaseIndexSheet.row(iRow)
if row[0].ctype == xlrd.XL_CELL_EMPTY or row[
1].ctype == xlrd.XL_CELL_EMPTY or row[
2].ctype == xlrd.XL_CELL_EMPTY:
continue
testDir = row[0].value
uriFrom = row[1].value
uriTo = row[2].value
intention = row[3].value
if intention is None or len(intention) == 0:
continue # test not ready to run
reason = row[4].value
expectedEvent = row[5].value
base = os.path.join(os.path.dirname(testGenFileName),
testDir) + os.sep
self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom))
if uriFrom and uriTo and reason.lower() not in (
"n.a.", "error") and expectedEvent != "N.A.":
for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"),
True), (uriTo, _("loading to DTS"),
False)):
if ',' not in URIs:
modelDTS = ModelXbrl.load(self.modelManager,
URIs,
msg,
base=base)
else:
modelDTS = ModelXbrl.create(
self.modelManager,
ModelDocument.Type.DTSENTRIES,
self.webCache.normalizeUrl(
URIs.replace(", ", "_") + ".dts", base),
isEntry=True)
DTSdoc = modelDTS.modelDocument
DTSdoc.inDTS = True
for uri in URIs.split(','):
doc = ModelDocument.load(modelDTS,
uri.strip(),
base=base)
DTSdoc.referencesDocument[
doc] = "import" #fake import
doc.inDTS = True
if isFrom: modelDTSfrom = modelDTS
else: modelDTSto = modelDTS
if modelDTSfrom and modelDTSto:
# generate differences report
reportUri = uriFrom.partition(',')[0] # first file
reportDir = os.path.dirname(reportUri)
if reportDir: reportDir += os.sep
reportName = os.path.basename(reportUri).replace(
"from.xsd", "report.xml")
reportFile = reportDir + "report" + os.sep + reportName
reportFullPath = self.webCache.normalizeUrl(
reportFile, base)
testcasesDir = os.path.dirname(
os.path.dirname(reportFullPath))
if testcasesDir != priorTestcasesDir:
# close prior report
if priorTestcasesDir:
for i, testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w",
encoding="utf-8") as fh:
XmlUtil.writexml(fh,
testcaseDocs[i],
encoding="utf-8")
testcaseName = os.path.basename(testcasesDir)
testcaseFiles = [
testcasesDir + os.sep + testcaseName +
"-creation-testcase.xml", testcasesDir + os.sep +
testcaseName + "-consumption-testcase.xml"
]
for i, testcaseFile in enumerate(testcaseFiles):
XmlUtil.addChild(
testcasesElements[i], None, "testcase",
("uri", testcaseFile[len(testGenDir) +
1:].replace("\\", "/")))
# start testcase file
testcaseDocs = [
xml.dom.minidom.parseString(
'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>'
'<testcase name="XBRL Versioning 1.0 {1} Tests" date="{2}" '
' xmlns="http://xbrl.org/2008/conformance"'
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">'
'<creator>'
'<name>Roland Hommes</name>'
'<email>[email protected]</email>'
'</creator>'
'<name>{0}</name>'
'<description>{0}</description>'
'</testcase>'.format(testcaseName, purpose,
today))
for purpose in ("Creation", "Consumption")
]
testcaseElements = [
XmlUtil.child(testcaseDoc, conformanceNS,
"testcase")
for testcaseDoc in testcaseDocs
]
priorTestcasesDir = testcasesDir
variationID = 1
try:
os.makedirs(os.path.dirname(reportFullPath))
except WindowsError:
pass # dir already exists
modelVersReport = ModelVersReport.ModelVersReport(
modelTestcases)
modelVersReport.diffDTSes(reportFullPath, modelDTSfrom,
modelDTSto)
# check for expected elements
if expectedEvent and expectedEvent not in ("No change",
"N.A."):
if len(
modelVersReport.xmlDocument.
getElementsByTagNameNS('*',
expectedEvent)) == 0:
modelTestcases.error(
"Generated test case {0} missing expected event {1}"
.format(reportName,
expectedEvent), "wrn", "missingEvent")
modelVersReport.close([])
for i, testcaseElt in enumerate(testcaseElements):
variationElement = XmlUtil.addChild(
testcaseElt,
conformanceNS,
"variation",
attributes=("id", "_{0:02n}".format(variationID)))
XmlUtil.addChild(variationElement,
conformanceNS,
"name",
text=intention)
dataElement = XmlUtil.addChild(variationElement,
conformanceNS, "data")
for schemaURIs, dtsAttr in ((uriFrom, "from"), (uriTo,
"to")):
for schemaURI in schemaURIs.split(","):
XmlUtil.addChild(
dataElement,
conformanceNS,
"schema",
attributes=((("dts", dtsAttr), ) +
((("readMeFirst",
"true"), ) if i == 0 else
())),
text=os.path.basename(schemaURI.strip()))
resultElement = XmlUtil.addChild(
variationElement, conformanceNS, "result")
XmlUtil.addChild(
resultElement if i == 0 else dataElement,
conformanceNS,
"versioningReport",
attributes=(("readMeFirst", "true") if i == 1 else
()),
text="report/" + reportName)
variationID += 1
with open(logMessagesFile, "w") as fh:
fh.writelines(self.logMessages)
if priorTestcasesDir:
for i, testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
for i, indexFile in enumerate(indexFiles):
with open(indexFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def runFromExcel(self, options):
#testGenFileName = options.excelfilename
testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation-index.xls"
testGenDir = os.path.dirname(testGenFileName)
schemaDir = os.path.dirname(testGenDir) + os.sep + "schema"
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
if options.testfiledate:
today = options.testfiledate
else:
today = XmlUtil.dateunionValue(datetime.date.today())
startedAt = time.time()
LogHandler(self) # start logger
self.logMessages = []
logMessagesFile = testGenDir + os.sep + 'log-generation-messages.txt'
modelTestcases = ModelXbrl.create(self.modelManager,
url=testGenFileName,
isEntry=True)
testcaseIndexBook = xlrd.open_workbook(testGenFileName)
testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0)
self.addToLog(
_("[info] xls loaded in {0:.2} secs at {1}").format(
time.time() - startedAt, timeNow))
# start index file
indexFiles = [
testGenDir + os.sep + 'creation-testcases-index.xml',
testGenDir + os.sep + 'consumption-testcases-index.xml'
]
indexDocs = []
testcasesElements = []
for purpose in ("Creation", "Consumption"):
file = io.StringIO(
#'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- XBRL Versioning 1.0 {0} Tests -->'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>'
'<testcases name="XBRL Versioning 1.0 {0} Tests" '
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">'
'</testcases>'.format(purpose, today))
doc = etree.parse(file)
file.close()
indexDocs.append(doc)
testcasesElements.append(doc.getroot())
priorTestcasesDir = None
testcaseFiles = None
testcaseDocs = None
for iRow in range(1, testcaseIndexSheet.nrows):
try:
row = testcaseIndexSheet.row(iRow)
if (row[0].ctype == xlrd.XL_CELL_EMPTY
or # must have directory
row[1].ctype == xlrd.XL_CELL_EMPTY or # from
row[2].ctype == xlrd.XL_CELL_EMPTY): # to
continue
testDir = row[0].value
uriFrom = row[1].value
uriTo = row[2].value
overrideReport = row[3].value
description = row[4].value
if description is None or len(description) == 0:
continue # test not ready to run
assignment = row[5].value
expectedEvents = row[
6].value # comma space separated if multiple
note = row[7].value
useCase = row[8].value
base = os.path.join(os.path.dirname(testGenFileName),
testDir) + os.sep
self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom))
if uriFrom and uriTo and assignment.lower() not in (
"n.a.", "error") and expectedEvents != "N.A.":
modelDTSfrom = modelDTSto = None
for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"),
True), (uriTo,
_("loading to DTS"),
False)):
if ',' not in URIs:
modelDTS = ModelXbrl.load(self.modelManager,
URIs,
msg,
base=base)
else:
modelDTS = ModelXbrl.create(
self.modelManager,
ModelDocument.Type.DTSENTRIES,
self.webCache.normalizeUrl(
URIs.replace(", ", "_") + ".dts", base),
isEntry=True)
DTSdoc = modelDTS.modelDocument
DTSdoc.inDTS = True
for uri in URIs.split(','):
doc = ModelDocument.load(modelDTS,
uri.strip(),
base=base)
if doc is not None:
DTSdoc.referencesDocument[
doc] = "import" #fake import
doc.inDTS = True
if isFrom: modelDTSfrom = modelDTS
else: modelDTSto = modelDTS
if modelDTSfrom is not None and modelDTSto is not None:
# generate differences report
reportUri = uriFrom.partition(',')[0] # first file
reportDir = os.path.dirname(reportUri)
if reportDir: reportDir += os.sep
reportName = os.path.basename(reportUri).replace(
"from.xsd", "report.xml")
reportFile = reportDir + "out" + os.sep + reportName
#reportFile = reportDir + "report" + os.sep + reportName
reportFullPath = self.webCache.normalizeUrl(
reportFile, base)
testcasesDir = os.path.dirname(
os.path.dirname(reportFullPath))
if testcasesDir != priorTestcasesDir:
# close prior report
if priorTestcasesDir:
for i, testcaseFile in enumerate(
testcaseFiles):
with open(testcaseFile,
"w",
encoding="utf-8") as fh:
XmlUtil.writexml(fh,
testcaseDocs[i],
encoding="utf-8")
testcaseName = os.path.basename(testcasesDir)
testcaseFiles = [
testcasesDir + os.sep + testcaseName +
"-creation-testcase.xml",
testcasesDir + os.sep + testcaseName +
"-consumption-testcase.xml"
]
for i, testcaseFile in enumerate(testcaseFiles):
etree.SubElement(
testcasesElements[i],
"testcase",
attrib={
"uri":
testcaseFile[len(testGenDir) +
1:].replace("\\", "/")
})
# start testcase file
testcaseDocs = []
testcaseElements = []
testcaseNumber = testcaseName[0:4]
if testcaseNumber.isnumeric():
testcaseNumberElement = "<number>{0}</number>".format(
testcaseNumber)
testcaseName = testcaseName[5:]
else:
testcaseNumberElement = ""
testDirSegments = testDir.split('/')
if len(testDirSegments
) >= 2 and '-' in testDirSegments[1]:
testedModule = testDirSegments[1][
testDirSegments[1].index('-') + 1:]
else:
testedModule = ''
for purpose in ("Creation", "Consumption"):
file = io.StringIO(
#'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>'
'<testcase '
' xmlns="http://xbrl.org/2008/conformance"'
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">'
'<creator>'
'<name>Roland Hommes</name>'
'<email>[email protected]</email>'
'</creator>'
'{0}'
'<name>{1}</name>'
# '<description>{0}</description>'
'<reference>'
'{2}'
'{3}'
'</reference>'
'</testcase>'.format(
testcaseNumberElement, testcaseName,
'<name>{0}</name>'.format(testedModule)
if testedModule else '',
'<id>{0}</id>'.format(useCase)
if useCase else ''))
doc = etree.parse(file)
file.close()
testcaseDocs.append(doc)
testcaseElements.append(doc.getroot())
priorTestcasesDir = testcasesDir
variationSeq = 1
try:
os.makedirs(os.path.dirname(reportFullPath))
except WindowsError:
pass # dir already exists
modelVersReport = ModelVersReport.ModelVersReport(
modelTestcases)
modelVersReport.diffDTSes(reportFullPath,
modelDTSfrom,
modelDTSto,
assignment=assignment,
schemaDir=schemaDir)
# check for expected elements
if expectedEvents:
for expectedEvent in expectedEvents.split(","):
if expectedEvent not in ("No change", "N.A."):
prefix, sep, localName = expectedEvent.partition(
':')
if sep and len(
modelVersReport.xmlDocument.
findall('//{{{0}}}{1}'.format(
XbrlConst.verPrefixNS.get(
prefix), localName))) == 0:
modelTestcases.warning(
"warning",
"Generated test case %(reportName)s missing expected event %(event)s",
reportName=reportName,
event=expectedEvent)
modelVersReport.close()
uriFromParts = uriFrom.split('_')
if len(uriFromParts) >= 2:
variationId = uriFromParts[1]
else:
variationId = "_{0:02n}".format(variationSeq)
for i, testcaseElt in enumerate(testcaseElements):
variationElement = etree.SubElement(
testcaseElt,
"{http://xbrl.org/2008/conformance}variation",
attrib={"id": variationId})
nameElement = etree.SubElement(
variationElement,
"{http://xbrl.org/2008/conformance}description"
)
nameElement.text = description
''' (removed per RH 2011/10/04
if note:
paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}description")
paramElement.text = "Note: " + note
if useCase:
paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}reference")
paramElement.set("specification", "versioning-requirements")
paramElement.set("useCase", useCase)
'''
dataElement = etree.SubElement(
variationElement,
"{http://xbrl.org/2008/conformance}data")
if i == 0: # result is report
if expectedEvents:
paramElement = etree.SubElement(
dataElement,
"{http://xbrl.org/2008/conformance}parameter",
attrib={
"name":
"expectedEvent",
"value":
expectedEvents.replace(',', ' ')
},
nsmap={
"conf":
"http://xbrl.org/2008/conformance",
None: ""
})
if assignment:
paramElement = etree.SubElement(
dataElement,
"{http://xbrl.org/2008/conformance}parameter",
attrib={
"name": "assignment",
"value": assignment
},
nsmap={
"conf":
"http://xbrl.org/2008/conformance",
None: ""
})
for schemaURIs, dtsAttr in ((uriFrom, "from"),
(uriTo, "to")):
for schemaURI in schemaURIs.split(","):
schemaElement = etree.SubElement(
dataElement,
"{http://xbrl.org/2008/conformance}schema"
)
schemaElement.set("dts", dtsAttr)
if i == 0:
schemaElement.set(
"readMeFirst", "true")
schemaElement.text = os.path.basename(
schemaURI.strip())
resultElement = etree.SubElement(
variationElement,
"{http://xbrl.org/2008/conformance}result")
reportElement = etree.SubElement(
resultElement if i == 0 else dataElement,
"{http://xbrl.org/2008/conformance}versioningReport"
)
if i == 1:
reportElement.set("readMeFirst", "true")
reportElement.text = "report/" + reportName
variationSeq += 1
except Exception as err:
modelTestcases.error(
"exception",
_("Exception: %(error)s, Excel row: %(excelRow)s"),
error=err,
excelRow=iRow,
exc_info=True)
# add tests-error-code index files to consumption
for testcaseFile in self.testcaseFiles(testGenDir + os.sep +
"tests-error-code"):
etree.SubElement(testcasesElements[1],
"testcase",
attrib={
"uri":
testcaseFile[len(testGenDir) + 1:].replace(
"\\", "/")
})
with open(logMessagesFile, "w") as fh:
fh.writelines(self.logMessages)
if priorTestcasesDir:
for i, testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
for i, indexFile in enumerate(indexFiles):
with open(indexFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def runFromExcel(self, options):
testGenFileName = options.excelfilename
#testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation\1000-2000-index.xls"
testGenDir = os.path.dirname(testGenFileName)
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
if options.testfiledate:
today = options.testfiledate
else:
today = XmlUtil.dateunionValue(datetime.date.today())
startedAt = time.time()
self.logMessages = []
logMessagesFile = testGenDir + os.sep + 'logGenerationMessages.txt'
modelTestcases = ModelXbrl.create(self.modelManager)
testcaseIndexBook = xlrd.open_workbook(testGenFileName)
testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0)
self.addToLog(_("[info] xls loaded in {0:.2} secs at {1}").format(time.time() - startedAt, timeNow))
# start index file
indexFiles = [testGenDir + os.sep + 'creationTestcasesIndex.xml',
testGenDir + os.sep + 'consumptionTestcasesIndex.xml']
indexDocs = [xml.dom.minidom.parseString(
'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- XBRL Versioning 1.0 {0} Tests -->'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>'
'<testcases name="XBRL Versioning 1.0 Consumption Tests" date="{1}" '
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">'
'</testcases>'.format(purpose, today)
) for purpose in ("Creation","Consumption")]
testcasesElements = [XmlUtil.child(indexDoc, None, "testcases") for indexDoc in indexDocs]
priorTestcasesDir = None
testcaseFiles = None
testcaseDocs = None
for iRow in range(1, testcaseIndexSheet.nrows):
row = testcaseIndexSheet.row(iRow)
if row[0].ctype == xlrd.XL_CELL_EMPTY or row[1].ctype == xlrd.XL_CELL_EMPTY or row[2].ctype == xlrd.XL_CELL_EMPTY:
continue
testDir = row[0].value
uriFrom = row[1].value
uriTo = row[2].value
intention = row[3].value
if intention is None or len(intention) == 0:
continue # test not ready to run
reason = row[4].value
expectedEvent = row[5].value
base = os.path.join(os.path.dirname(testGenFileName),testDir) + os.sep
self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom))
if uriFrom and uriTo and reason.lower() not in ("n.a.", "error") and expectedEvent != "N.A.":
for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"), True), (uriTo, _("loading to DTS"), False)):
if ',' not in URIs:
modelDTS = ModelXbrl.load(self.modelManager, URIs, msg, base=base)
else:
modelDTS = ModelXbrl.create(self.modelManager,
ModelDocument.Type.DTSENTRIES,
self.webCache.normalizeUrl(URIs.replace(", ","_") + ".dts",
base),
isEntry=True)
DTSdoc = modelDTS.modelDocument
DTSdoc.inDTS = True
for uri in URIs.split(','):
doc = ModelDocument.load(modelDTS, uri.strip(), base=base)
DTSdoc.referencesDocument[doc] = "import" #fake import
doc.inDTS = True
if isFrom: modelDTSfrom = modelDTS
else: modelDTSto = modelDTS
if modelDTSfrom and modelDTSto:
# generate differences report
reportUri = uriFrom.partition(',')[0] # first file
reportDir = os.path.dirname(reportUri)
if reportDir: reportDir += os.sep
reportName = os.path.basename(reportUri).replace("from.xsd","report.xml")
reportFile = reportDir + "report" + os.sep + reportName
reportFullPath = self.webCache.normalizeUrl(
reportFile,
base)
testcasesDir = os.path.dirname(os.path.dirname(reportFullPath))
if testcasesDir != priorTestcasesDir:
# close prior report
if priorTestcasesDir:
for i,testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
testcaseName = os.path.basename(testcasesDir)
testcaseFiles = [testcasesDir + os.sep + testcaseName + "-creation-testcase.xml",
testcasesDir + os.sep + testcaseName + "-consumption-testcase.xml"]
for i,testcaseFile in enumerate(testcaseFiles):
XmlUtil.addChild(testcasesElements[i], None, "testcase",
("uri",
testcaseFile[len(testGenDir)+1:].replace("\\","/")) )
# start testcase file
testcaseDocs = [xml.dom.minidom.parseString(
'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>'
'<testcase name="XBRL Versioning 1.0 {1} Tests" date="{2}" '
' xmlns="http://xbrl.org/2008/conformance"'
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">'
'<creator>'
'<name>Roland Hommes</name>'
'<email>[email protected]</email>'
'</creator>'
'<name>{0}</name>'
'<description>{0}</description>'
'</testcase>'.format(testcaseName,purpose,today)
) for purpose in ("Creation","Consumption")]
testcaseElements = [XmlUtil.child(testcaseDoc, conformanceNS, "testcase") for testcaseDoc in testcaseDocs]
priorTestcasesDir = testcasesDir
variationID = 1
try:
os.makedirs(os.path.dirname(reportFullPath))
except WindowsError:
pass # dir already exists
modelVersReport = ModelVersReport.ModelVersReport(modelTestcases)
modelVersReport.diffDTSes(reportFullPath,modelDTSfrom, modelDTSto)
# check for expected elements
if expectedEvent and expectedEvent not in (
"No change", "N.A."):
if len(modelVersReport.xmlDocument.getElementsByTagNameNS('*',expectedEvent)) == 0:
modelTestcases.error(
"Generated test case {0} missing expected event {1}".format(
reportName,
expectedEvent),
"wrn", "missingEvent")
modelVersReport.close([])
for i,testcaseElt in enumerate(testcaseElements):
variationElement = XmlUtil.addChild(testcaseElt, conformanceNS, "variation",
attributes=("id", "_{0:02n}".format(variationID)))
XmlUtil.addChild(variationElement, conformanceNS, "name", text=intention)
dataElement = XmlUtil.addChild(variationElement, conformanceNS, "data")
for schemaURIs, dtsAttr in ((uriFrom,"from"), (uriTo,"to")):
for schemaURI in schemaURIs.split(","):
XmlUtil.addChild(dataElement, conformanceNS, "schema",
attributes=((("dts",dtsAttr),) +
((("readMeFirst","true"),) if i == 0 else ())),
text=os.path.basename(schemaURI.strip()))
resultElement = XmlUtil.addChild(variationElement, conformanceNS, "result")
XmlUtil.addChild(resultElement if i == 0 else dataElement,
conformanceNS, "versioningReport",
attributes=(("readMeFirst","true") if i == 1 else ()),
text="report/" + reportName)
variationID += 1
with open(logMessagesFile, "w") as fh:
fh.writelines(self.logMessages)
if priorTestcasesDir:
for i,testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
for i,indexFile in enumerate(indexFiles):
with open(indexFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")