def load(modelManager, url, nextaction, base=None):
from arelle import (ModelDocument, FileSource)
modelXbrl = create(modelManager)
if isinstance(url,FileSource.FileSource):
modelXbrl.fileSource = url
url = modelXbrl.fileSource.url
else:
modelXbrl.fileSource = FileSource.FileSource(url)
modelXbrl.modelDocument = ModelDocument.load(modelXbrl, url, base, isEntry=True)
# at this point DTS is fully discovered but schemaLocated xsd's are not yet loaded
modelDocumentsSchemaLocated = set()
while True: # need this logic because each new pass may add new urlDocs
modelDocuments = set(modelXbrl.urlDocs.values()) - modelDocumentsSchemaLocated
if not modelDocuments:
break
modelDocument = modelDocuments.pop()
modelDocumentsSchemaLocated.add(modelDocument)
modelDocument.loadSchemalocatedSchemas()
#from arelle import XmlValidate
#uncomment for trial use of lxml xml schema validation of entry document
#XmlValidate.xmlValidate(modelXbrl.modelDocument)
modelManager.cntlr.webCache.saveUrlCheckTimes()
modelManager.showStatus(_("xbrl loading finished, {0}...").format(nextaction))
return modelXbrl
def create(modelManager, newDocumentType=None, url=None, schemaRefs=None, createModelDocument=True, isEntry=False):
from arelle import (ModelDocument, FileSource)
modelXbrl = ModelXbrl(modelManager)
modelXbrl.locale = modelManager.locale
if newDocumentType:
modelXbrl.fileSource = FileSource.FileSource(url)
if createModelDocument:
modelXbrl.modelDocument = ModelDocument.create(modelXbrl, newDocumentType, url, schemaRefs=schemaRefs, isEntry=isEntry)
return modelXbrl
def runFromXml(self):
testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\Arelle\roland test cases\1000-Concepts\index.xml"
filesource = FileSource.FileSource(testGenFileName)
startedAt = time.time()
LogHandler(self) # start logger
modelTestcases = self.modelManager.load(filesource, _("views loading"))
self.addToLog(
_("[info] loaded in {0:.2} secs").format(time.time() - startedAt))
if modelTestcases.modelDocument.type == ModelDocument.Type.TESTCASESINDEX:
for testcasesElement in modelTestcases.modelDocument.iter(
tag="testcases"):
rootAttr = testcasesElement.get("root")
title = testcasesElement.get("title")
self.addToLog(_("[info] testcases {0}").format(title))
if rootAttr is not None:
base = os.path.join(
os.path.dirname(modelTestcases.modelDocument.filepath),
rootAttr) + os.sep
else:
base = self.filepath
for testcaseElement in testcasesElement.iterchildren(
tag="testcase"):
uriFrom = testcaseElement.get("uriFrom")
uriTo = testcaseElement.get("uriTo")
modelDTSfrom = modelDTSto = None
self.addToLog(
_("[info] testcase uriFrom {0}").format(uriFrom))
if uriFrom is not None and uriTo is not None:
modelDTSfrom = ModelXbrl.load(
modelTestcases.modelManager,
uriFrom,
_("loading from DTS"),
base=base)
modelDTSto = ModelXbrl.load(
modelTestcases.modelManager,
uriTo,
_("loading to DTS"),
base=base)
if modelDTSfrom is not None and modelDTSto is not None:
# generate differences report
reportName = os.path.basename(uriFrom).replace(
"from.xsd", "report.xml")
reportFile = os.path.dirname(
uriFrom) + "\\report\\" + reportName
reportFullPath = self.webCache.normalizeUrl(
reportFile, base)
try:
os.makedirs(os.path.dirname(reportFullPath))
except WindowsError:
pass # dir already exists
ModelVersReport.ModelVersReport(
modelTestcases).diffDTSes(
reportFullPath, modelDTSfrom, modelDTSto)
def run(self):
self.messages = []
modelManager = ModelManager.initialize(self)
# filesource = FileSource.FileSource("./aapl-20180929.xml")
filesource = FileSource.FileSource(
"https://www.sec.gov/Archives/edgar/data/320187/000032018719000051/nke-20190531.xml"
)
modelXbrl = modelManager.load(
"https://www.sec.gov/Archives/edgar/data/320187/000032018719000051/nke-20190531.xml",
_("views loading"))
for fact in modelXbrl.facts:
print(fact)
def create(modelManager,
newDocumentType=None,
url=None,
schemaRefs=None,
createModelDocument=True,
isEntry=False):
from arelle import (ModelDocument, FileSource)
modelXbrl = ModelXbrl(modelManager)
modelXbrl.locale = modelManager.locale
if newDocumentType:
modelXbrl.fileSource = FileSource.FileSource(
url) # url may be an open file handle, use str(url) below
modelXbrl.closeFileSource = True
if createModelDocument:
modelXbrl.modelDocument = ModelDocument.create(
modelXbrl,
newDocumentType,
str(url),
schemaRefs=schemaRefs,
isEntry=isEntry)
if isEntry:
del modelXbrl.entryLoadingUrl
return modelXbrl
def createInstance(self, url=None):
from arelle import (ModelDocument, FileSource)
if self.modelDocument.type == ModelDocument.Type.INSTANCE: # entry already is an instance
return self.modelDocument # use existing instance entry point
priorFileSource = self.fileSource
self.fileSource = FileSource.FileSource(url)
if self.uri.startswith("http://"):
schemaRefUri = self.uri
else: # relativize local paths
schemaRefUri = os.path.relpath(self.uri, os.path.dirname(url))
self.modelDocument = ModelDocument.create(self,
ModelDocument.Type.INSTANCE,
url,
schemaRefs=[schemaRefUri],
isEntry=True)
if priorFileSource:
priorFileSource.close()
self.closeFileSource = True
del self.entryLoadingUrl
# reload dts views
from arelle import ViewWinDTS
for view in self.views:
if isinstance(view, ViewWinDTS.ViewDTS):
self.modelManager.cntlr.uiThreadQueue.put((view.view, []))
def validateTestcase(self, testcase):
self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
self.modelXbrl.viewModelObject(testcase.objectId())
if hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcase.testcaseVariations:
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(modelTestcaseVariation)
# try to load instance document
self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"),
modelObject=modelTestcaseVariation,
id=modelTestcaseVariation.id,
name=modelTestcaseVariation.name,
expected=modelTestcaseVariation.expected,
description=modelTestcaseVariation.description)
errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY
parameters = modelTestcaseVariation.parameters.copy()
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri,tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager,
Type.DTSENTRIES,
self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
isEntry=True,
errorCaptureLevel=errorCaptureLevel)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement)
if doc is not None:
DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import
doc.inDTS = True
elif resultIsTaxonomyPackage:
from arelle import PackageManager, PrototypeInstanceObject
dtsName = readMeFirstUri
modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri)
PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors)
else: # not a multi-schemaRef versioning report
if self.useFileSource.isArchive:
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
else: # need own file source, may need instance discovery
filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr)
if filesource and not filesource.selection and filesource.isArchive:
for _archiveFile in filesource.dir: # find instance document in archive
filesource.select(_archiveFile)
if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL):
break # use this selection
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
filesource,
_("validating"),
base=baseForElement,
errorCaptureLevel=errorCaptureLevel)
modelXbrl.isTestcaseVariation = True
if modelXbrl.modelDocument is None:
modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s document not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif resultIsVersioningReport or resultIsTaxonomyPackage:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif testcase.type == Type.REGISTRYTESTCASE:
self.instValidator.validate(modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
self.instValidator.close()
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
# validate except for formulas
_hasFormulae = modelXbrl.hasFormulae
modelXbrl.hasFormulae = False
try:
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation)
self.instValidator.validate(modelXbrl, parameters)
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
modelXbrl.hasFormulae = _hasFormulae
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri, baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
versReportFile, inputDTSes["from"], inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes.values():
inputDTS.close()
del inputDTSes # dereference
elif resultIsTaxonomyPackage:
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters
if dtsName: # named instance
parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes)
elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents
parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable:
RenderingEvaluator.init(modelXbrl)
if modelXbrl.hasFormulae:
try:
# validate only formulae
self.instValidator.parameters = parameters
ValidateFormula.validate(self.instValidator)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"):
pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri)
infoset = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultInfosetUri,
_("loading result infoset"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if infoset.modelDocument is None:
modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result infoset not loadable"
else: # check infoset
ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
infoset.close()
if modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
# diff (or generate) table infoset
resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement)
if not any(alternativeValidation(modelXbrl, resultTableUri)
for alternativeValidation in pluginClassMethods("Validate.TableInfoset")):
ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files
self.instValidator.close()
extraErrors = []
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors)
self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status
if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual):
# if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost)
self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors)
if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
self.instValidator.close()
compareIxResultInstance = getattr(modelXbrl, "extractedInlineInstance", False) and modelTestcaseVariation.resultXbrlInstanceUri
if compareIxResultInstance:
formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance
errMsgPrefix = "ix"
else: # delete input instances before formula output comparision
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
errMsgPrefix = "formula"
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if expectedInstance.modelDocument is None:
self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix),
_("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri),
messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded"))
modelTestcaseVariation.status = "result not loadable"
else: # compare facts
if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix),
_("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
expectedFacts=len(expectedInstance.facts),
messageCodes=("formula:resultFactCounts","ix:resultFactCounts"))
else:
formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes")
expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes")
def factFootnotes(fact, footnotesRelSet):
footnotes = []
footnoteRels = footnotesRelSet.fromModelObject(fact)
if footnoteRels:
# most process rels in same order between two instances, use labels to sort
for i, footnoteRel in enumerate(sorted(footnoteRels,
key=lambda r: (r.fromLabel,r.toLabel))):
modelObject = footnoteRel.toModelObject
if isinstance(modelObject, ModelResource):
xml = modelObject.viewText().strip()
footnotes.append("Footnote {}: {}".format(
i+1, # compare footnote with HTML serialized
xml,
#re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue))
))
elif isinstance(modelObject, ModelFact):
footnotes.append("Footnoted fact {}: {} context: {} value: {}".format(
i+1,
modelObject.qname,
modelObject.contextID,
collapseWhitespace(modelObject.value)))
return footnotes
for expectedInstanceFact in expectedInstance.facts:
unmatchedFactsStack = []
formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True)
if formulaOutputFact is None:
if unmatchedFactsStack: # get missing nested tuple fact, if possible
missingFact = unmatchedFactsStack[-1]
else:
missingFact = expectedInstanceFact
formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix),
_("Output missing expected fact %(fact)s"),
modelXbrl=missingFact, fact=missingFact.qname,
messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing"))
else: # compare footnotes
expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet)
formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet)
if expectedInstanceFactFootnotes != formulaOutputFactFootnotes:
formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix),
_("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"),
modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=expectedInstanceFactFootnotes, footnotes2=formulaOutputFactFootnotes,
messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference"))
# for debugging uncomment next line to save generated instance document
# formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
expectedInstance.close()
del expectedInstance # dereference
self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors)
formulaOutputInstance.close()
del formulaOutputInstance
if compareIxResultInstance:
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def tdNetLoader(modelXbrl, mappedUri, filepath, **kwargs):
if not (mappedUri.startswith("https://www.release.tdnet.info/inbs/I_")
and mappedUri.endswith(".html")):
return None # not a td net info file
rssObject = ModelRssObject(modelXbrl, uri=mappedUri, filepath=filepath)
hasMoreSections = True
while hasMoreSections:
# treat tdnet as an RSS feed object
try:
tdInfoDoc = html.parse(filepath)
except (IOError, EnvironmentError):
return None # give up, use ordinary loader
# find date
date = None
for elt in tdInfoDoc.iter():
if elt.tag == "table":
break # no date portion, probably wrong document
if elt.text and datePattern.match(elt.text):
g = datePattern.match(elt.text).groups()
date = datetime.date(int(g[0]), int(g[1]), int(g[2]))
break
if not date:
return None # give up, not a TDnet index document
urlDir = os.path.dirname(mappedUri)
# find <table> with <a>Download in it
for tableElt in tdInfoDoc.iter(tag="table"):
useThisTableElt = False
for aElt in tableElt.iterdescendants(tag="a"):
if "download" in aElt.text.lower():
useThisTableElt = True
break
if useThisTableElt:
cols = {}
for trElt in tableElt.iter(tag="tr"):
col = 0
rowData = {}
for tdElt in trElt.iter(tag="td"):
text = ''.join(t.strip() for t in tdElt.itertext())
if tdElt.get("class") == "tableh": #header
type = {
"時刻": "time",
"コード": "code",
"会社名": "companyName",
"表題": "title",
"XBRL": "zipUrl",
"上場取引所": "stockExchange",
"更新履歴": "changeLog"
}.get(text, None)
if type:
cols[col] = type
cols[type] = col
elif col == cols["title"]:
rowData["title"] = text
rowData["pdfUrl"] = descendantAttr(
tdElt, "a", "href")
elif col == cols["zipUrl"]:
rowData["zipUrl"] = descendantAttr(
tdElt, "a", "href")
elif col in cols: # body
rowData[cols[col]] = text
col += int(tdElt.get("colspan", 1))
if rowData:
time = rowData.get("time", "")
if timePattern.match(time):
g = timePattern.match(time).groups()
dateTime = datetime.datetime(
date.year, date.month, date.day, int(g[0]),
int(g[1]))
else:
dateTime = datetime.datetime.now()
filingCode = rowData.get("code")
companyName = rowData.get("companyName")
stockExchange = rowData.get("stockExchange")
title = rowData.get("title")
pdfUrl = rowData.get("pdfUrl")
if pdfUrl:
pdfUrl = urlDir + "/" + pdfUrl
zipUrl = rowData.get("zipUrl")
if zipUrl:
zipUrl = urlDir + "/" + zipUrl
changeLog = rowData.get("changeLog")
# find instance doc in file
instanceUrls = []
if zipUrl:
try:
normalizedUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
zipUrl)
filepath = modelXbrl.modelManager.cntlr.webCache.getfilename(
normalizedUri)
filesource = FileSource.FileSource(filepath)
dir = filesource.dir
filesource.close()
if dir:
for file in dir:
if "ixbrl" in file or file.endswith(
".xbrl") or "instance" in file:
instanceUrls.append(zipUrl + "/" +
file)
except:
continue # forget this filing
for instanceUrl in instanceUrls:
rssObject.rssItems.append(
TDnetItem(modelXbrl, date, dateTime,
filingCode, companyName, title,
pdfUrl, instanceUrl, stockExchange))
# next screen if continuation
hasMoreSections = False
for elt in tdInfoDoc.iter(tag="input"):
if elt.value == "次画面": # next screen button
nextLocation = elt.get("onclick")
if nextLocation and nextLocationPattern.match(nextLocation):
hasMoreSections = True
nextUrl = urlDir + "/" + nextLocationPattern.match(
nextLocation).groups()[0]
mappedUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
nextUrl)
filepath = modelXbrl.modelManager.cntlr.webCache.getfilename(
mappedUri)
return rssObject
def run(self, options):
if options.logFile:
self.messages = []
else:
self.messages = None
self.filename = options.filename
filesource = FileSource.FileSource(self.filename, self)
if options.validateEFM:
if options.gfmName:
self.addToLog(
_("[info] both --efm and --gfm validation are requested, proceeding with --efm only"
))
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select("efm")
elif options.gfmName:
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select(options.gfmName)
else:
self.modelManager.disclosureSystem.select(
None) # just load ordinary mappings
if options.calcDecimals:
if options.calcPrecision:
self.addToLog(
_("[info] both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"
))
self.modelManager.validateInferDecimals = True
self.modelManager.validateCalcLB = True
elif options.calcPrecision:
self.modelManager.validateInferDecimals = False
self.modelManager.validateCalcLB = True
if options.utrValidate:
self.modelManager.validateUtr = True
fo = FormulaOptions()
if options.formulaParamExprResult:
fo.traceParameterExpressionResult = True
if options.formulaParamInputValue:
fo.traceParameterInputValue = True
if options.formulaCallExprSource:
fo.traceCallExpressionSource = True
if options.formulaCallExprCode:
fo.traceCallExpressionCode = True
if options.formulaCallExprEval:
fo.traceCallExpressionEvaluation = True
if options.formulaCallExprResult:
fo.traceCallExpressionResult = True
if options.formulaVarSetExprEval:
fo.traceVariableSetExpressionEvaluation = True
if options.formulaVarSetExprResult:
fo.traceVariableSetExpressionResult = True
if options.formulaAsserResultCounts:
fo.traceAssertionResultCounts = True
if options.formulaFormulaRules:
fo.traceFormulaRules = True
if options.formulaVarsOrder:
fo.traceVariablesOrder = True
if options.formulaVarExpressionSource:
fo.traceVariableExpressionSource = True
if options.formulaVarExpressionCode:
fo.traceVariableExpressionCode = True
if options.formulaVarExpressionEvaluation:
fo.traceVariableExpressionEvaluation = True
if options.formulaVarExpressionResult:
fo.traceVariableExpressionResult = True
if options.formulaVarFiltersResult:
fo.traceVariableFiltersResult = True
self.modelManager.formulaOptions = fo
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
startedAt = time.time()
modelXbrl = self.modelManager.load(filesource, _("views loading"))
self.addToLog(
format_string(self.modelManager.locale,
_("[info] loaded in %.2f secs at %s"),
(time.time() - startedAt, timeNow)))
if options.diffFilename and options.versReportFilename:
diffFilesource = FileSource.FileSource(self.diffFilename, self)
startedAt = time.time()
modelXbrl = self.modelManager.load(diffFilesource,
_("views loading"))
self.addToLog(
format_string(
self.modelManager.locale,
_("[info] diff comparison DTS loaded in %.2f secs"),
time.time() - startedAt))
startedAt = time.time()
self.modelManager.compareDTSes(options.versReportFilename)
self.addToLog(
format_string(self.modelManager.locale,
_("[info] compared in %.2f secs"),
time.time() - startedAt))
try:
if options.validate:
startedAt = time.time()
self.modelManager.validate()
self.addToLog(
format_string(self.modelManager.locale,
_("[info] validated in %.2f secs"),
time.time() - startedAt))
if (options.csvTestReport
and self.modelManager.modelXbrl.modelDocument.type
in (ModelDocument.Type.TESTCASESINDEX,
ModelDocument.Type.REGISTRY)):
ViewCsvTests.viewTests(self.modelManager.modelXbrl,
options.csvTestReport)
if options.csvDTS:
ViewCsvDTS.viewDTS(modelXbrl, options.csvDTS)
if options.csvFactList:
ViewCsvFactList.viewFacts(modelXbrl, options.csvFactList)
if options.csvConcepts:
ViewCsvConcepts.viewConcepts(modelXbrl, options.csvConcepts)
if options.csvPre:
ViewCsvRelationshipSet.viewRelationshipSet(
modelXbrl, options.csvPre, "Presentation",
"http://www.xbrl.org/2003/arcrole/parent-child")
if options.csvCal:
ViewCsvRelationshipSet.viewRelationshipSet(
modelXbrl, options.csvCal, "Calculation",
"http://www.xbrl.org/2003/arcrole/summation-item")
if options.csvDim:
ViewCsvRelationshipSet.viewRelationshipSet(
modelXbrl, options.csvDim, "Dimension", "XBRL-dimensions")
except (IOError, EnvironmentError) as err:
self.addToLog(
_("[IOError] Failed to save output:\n {0}").format(err))
if self.messages:
try:
with open(options.logFile, "w", encoding="utf-8") as fh:
fh.writelines(self.messages)
except (IOError, EnvironmentError) as err:
print("Unable to save log to file: " + err)
def run(self, options, sourceZipStream=None):
self.entrypointFile = options.entrypointFile
filesource = FileSource.openFileSource(self.entrypointFile, self, sourceZipStream)
if options.validateEFM:
if options.gfmName:
self.addToLog(_("both --efm and --gfm validation are requested, proceeding with --efm only"),
messageCode="info", file=self.entrypointFile)
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select("efm")
elif options.gfmName:
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select(options.gfmName)
elif options.validateHMRC:
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select("hmrc")
else:
self.modelManager.disclosureSystem.select(None) # just load ordinary mappings
if options.calcDecimals:
if options.calcPrecision:
self.addToLog(_("both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"),
messageCode="info", file=self.entrypointFile)
self.modelManager.validateInferDecimals = True
self.modelManager.validateCalcLB = True
elif options.calcPrecision:
self.modelManager.validateInferDecimals = False
self.modelManager.validateCalcLB = True
if options.utrValidate:
self.modelManager.validateUtr = True
fo = FormulaOptions()
if options.parameters:
fo.parameterValues = dict(((qname(key, noPrefixIsNoNamespace=True),(None,value))
for param in options.parameters.split(',')
for key,sep,value in (param.partition('='),) ) )
if options.formulaParamExprResult:
fo.traceParameterExpressionResult = True
if options.formulaParamInputValue:
fo.traceParameterInputValue = True
if options.formulaCallExprSource:
fo.traceCallExpressionSource = True
if options.formulaCallExprCode:
fo.traceCallExpressionCode = True
if options.formulaCallExprEval:
fo.traceCallExpressionEvaluation = True
if options.formulaCallExprResult:
fo.traceCallExpressionResult = True
if options.formulaVarSetExprEval:
fo.traceVariableSetExpressionEvaluation = True
if options.formulaVarSetExprResult:
fo.traceVariableSetExpressionResult = True
if options.formulaAsserResultCounts:
fo.traceAssertionResultCounts = True
if options.formulaFormulaRules:
fo.traceFormulaRules = True
if options.formulaVarsOrder:
fo.traceVariablesOrder = True
if options.formulaVarExpressionSource:
fo.traceVariableExpressionSource = True
if options.formulaVarExpressionCode:
fo.traceVariableExpressionCode = True
if options.formulaVarExpressionEvaluation:
fo.traceVariableExpressionEvaluation = True
if options.formulaVarExpressionResult:
fo.traceVariableExpressionResult = True
if options.formulaVarFilterWinnowing:
fo.traceVariableFilterWinnowing = True
if options.formulaVarFiltersResult:
fo.traceVariableFiltersResult = True
self.modelManager.formulaOptions = fo
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
startedAt = time.time()
modelDiffReport = None
success = True
modelXbrl = None
try:
modelXbrl = self.modelManager.load(filesource, _("views loading"))
except Exception as err:
self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format(
err,
traceback.format_tb(sys.exc_info()[2])))
success = False # loading errors, don't attempt to utilize loaded DTS
if modelXbrl and modelXbrl.modelDocument:
self.addToLog(format_string(self.modelManager.locale,
_("loaded in %.2f secs at %s"),
(time.time() - startedAt, timeNow)),
messageCode="info", file=self.entrypointFile)
if options.importFiles:
for importFile in options.importFiles.split("|"):
ModelDocument.load(modelXbrl, importFile.strip())
self.addToLog(format_string(self.modelManager.locale,
_("imported in %.2f secs at %s"),
(time.time() - startedAt, timeNow)),
messageCode="info", file=importFile)
if modelXbrl.errors:
success = False # loading errors, don't attempt to utilize loaded DTS
else:
success = False
if success and options.diffFile and options.versReportFile:
diffFilesource = FileSource.FileSource(options.diffFile,self)
startedAt = time.time()
modelXbrl2 = self.modelManager.load(diffFilesource, _("views loading"))
if modelXbrl2.errors:
if not options.keepOpen:
modelXbrl2.close()
success = False
else:
self.addToLog(format_string(self.modelManager.locale,
_("diff comparison DTS loaded in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.entrypointFile)
startedAt = time.time()
modelDiffReport = self.modelManager.compareDTSes(options.versReportFile)
self.addToLog(format_string(self.modelManager.locale,
_("compared in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.entrypointFile)
if success:
try:
if options.validate:
startedAt = time.time()
self.modelManager.validate()
self.addToLog(format_string(self.modelManager.locale,
_("validated in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.entrypointFile)
if (options.testReport and
self.modelManager.modelXbrl.modelDocument.type in
(ModelDocument.Type.TESTCASESINDEX,
ModelDocument.Type.TESTCASE,
ModelDocument.Type.REGISTRY)):
ViewFileTests.viewTests(self.modelManager.modelXbrl, options.testReport)
if options.DTSFile:
ViewFileDTS.viewDTS(modelXbrl, options.DTSFile)
if options.factsFile:
ViewFileFactList.viewFacts(modelXbrl, options.factsFile, labelrole=options.labelRole, lang=options.labelLang, cols=options.factListCols)
if options.factTableFile:
ViewFileFactTable.viewFacts(modelXbrl, options.factTableFile, labelrole=options.labelRole, lang=options.labelLang)
if options.conceptsFile:
ViewFileConcepts.viewConcepts(modelXbrl, options.conceptsFile, labelrole=options.labelRole, lang=options.labelLang)
if options.preFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.preFile, "Presentation Linkbase", "http://www.xbrl.org/2003/arcrole/parent-child", labelrole=options.labelRole, lang=options.labelLang)
if options.calFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.calFile, "Calculation Linkbase", "http://www.xbrl.org/2003/arcrole/summation-item", labelrole=options.labelRole, lang=options.labelLang)
if options.dimFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.dimFile, "Dimensions", "XBRL-dimensions", labelrole=options.labelRole, lang=options.labelLang)
if options.formulaeFile:
ViewFileFormulae.viewFormulae(modelXbrl, options.formulaeFile, "Formulae", lang=options.labelLang)
for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Run"):
pluginXbrlMethod(self, options, modelXbrl)
except (IOError, EnvironmentError) as err:
self.addToLog(_("[IOError] Failed to save output:\n {0}").format(err))
success = False
except Exception as err:
self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format(
err,
traceback.format_tb(sys.exc_info()[2])))
success = False
if not options.keepOpen:
if modelDiffReport:
modelDiffReport.close()
elif modelXbrl:
modelXbrl.close()
return success
