def loadAuthorityValidations(modelXbrl): _file = openFileStream(modelXbrl.modelManager.cntlr, resourcesFilePath(modelXbrl.modelManager, "authority-validations.json"), 'rt', encoding='utf-8') validations = json.load(_file) # {localName: date, ...} _file.close() return validations
def loadOtherStandardTaxonomies(modelXbrl, val): _file = openFileStream(modelXbrl.modelManager.cntlr, resourcesFilePath(modelXbrl.modelManager, "other-standard-taxonomies.json"), 'rt', encoding='utf-8') otherStandardTaxonomies = json.load(_file) # {localName: date, ...} _file.close() otherStandardNsPrefixes = otherStandardTaxonomies.get("taxonomyPrefixes",{}) return set(doc.targetNamespace for doc in modelXbrl.urlDocs.values() if doc.targetNamespace and doc.targetNamespace not in val.disclosureSystem.standardTaxonomiesDict and any(doc.targetNamespace.startswith(nsPrefix) for nsPrefix in otherStandardNsPrefixes))
def loadDeprecatedConceptDates(val, deprecatedConceptDates): for modelDocument in val.modelXbrl.urlDocs.values(): ns = modelDocument.targetNamespace abbrNs = abbreviatedWildNamespace(ns) latestTaxonomyDoc = latestTaxonomyDocs.get(abbrNs) _fileName = deprecatedConceptDatesFile(val.modelXbrl.modelManager, abbrNs, latestTaxonomyDoc) if _fileName: _file = openFileStream(val.modelXbrl.modelManager.cntlr, _fileName, 'rt', encoding='utf-8') _deprecatedConceptDates = json.load(_file) # {localName: date, ...} _file.close() for localName, date in _deprecatedConceptDates.items(): deprecatedConceptDates[qname(ns, localName)] = date
def loadTaxonomyCompatibility(modelXbrl): _file = openFileStream(modelXbrl.modelManager.cntlr, resourcesFilePath(modelXbrl.modelManager, "taxonomy-compatibility.json"), 'rt', encoding='utf-8') compat = json.load(_file, object_pairs_hook=OrderedDict) # preserve order of keys _file.close() tc = compat["taxonomy-classes"] cc = compat["compatible-classes"] def refTx(txAbbrs): return [refTx(tc[txAbbr[1:]]) if txAbbr.startswith("@") else txAbbr for txAbbr in txAbbrs] for k in cc.keys(): cc[k] = set(flattenSequence(refTx(cc[k]))) compat["checked-taxonomies"] = set(flattenSequence([t for t in cc.items()])) return compat
def loadIxTransformRegistries(modelXbrl): _file = openFileStream(modelXbrl.modelManager.cntlr, resourcesFilePath( modelXbrl.modelManager, "ixbrl-transform-registries.json"), 'rt', encoding='utf-8') ixTrRegistries = json.load( _file, object_pairs_hook=OrderedDict) # preserve order of keys _file.close() ixTrRegistries.pop("copyright", None) ixTrRegistries.pop("description", None) return ixTrRegistries
def loadCustomAxesReplacements(modelXbrl): # returns match expression, standard patterns _file = openFileStream(modelXbrl.modelManager.cntlr, resourcesFilePath(modelXbrl.modelManager, "axiswarnings.json"), 'rt', encoding='utf-8') axiswarnings = json.load(_file) # {localName: date, ...} _file.close() standardAxes = {} matchPattern = [] for i, (standardAxis, customAxisPattern) in enumerate(axiswarnings.items()): if standardAxis not in ("#", "copyright", "description"): patternName = "_{}".format(i) standardAxes[patternName] = standardAxis matchPattern.append("(?P<{}>^{}$)".format(patternName, customAxisPattern)) return attrdict(standardAxes=standardAxes, customNamePatterns=re.compile("|".join(matchPattern)))
def loadNonNegativeFacts(modelXbrl, dqcRules, ugtRels): # for us-gaap newer than 2020 use DQCRT non-negative facts. if dqcRules and ugtRels: # not used before 2020 if usgaapYear( modelXbrl) == "2020" and "dqcrt-2021-usgaap-2020" not in ( modelXbrl.modelManager.disclosureSystem.options or ""): dqcRules.clear() # remove dqc rules return ugtRels[ "DQC.US.0015"] # use 20.1 2020 nonNegFacts test and warning return None # use all available DQCRT tests # for us-gaap < dqcyear use EFM non-negative warning insead of DQC rule _file = openFileStream(modelXbrl.modelManager.cntlr, resourcesFilePath(modelXbrl.modelManager, "signwarnings.json"), 'rt', encoding='utf-8') signwarnings = json.load(_file) # {localName: date, ...} _file.close() concepts = set() excludedMembers = set() excludedMemberStrings = set() excludedAxesMembers = defaultdict(set) for modelDocument in modelXbrl.urlDocs.values(): ns = modelDocument.targetNamespace # set up non neg lookup by full NS for abbrNs in (abbreviatedNamespace(ns), abbreviatedNamespace(ns, WILD)): nsMatch = False for exName, exSet, isQName in (("conceptNames", concepts, True), ("excludedMemberNames", excludedMembers, True), ("excludedMemberStrings", excludedMemberStrings, False)): for localName in signwarnings[exName].get(abbrNs, ()): exSet.add(qname(ns, localName) if isQName else localName) nsMatch = True for localDimName, localMemNames in signwarnings[ "excludedAxesMembers"].get(abbrNs, EMPTY_DICT).items(): for localMemName in localMemNames: excludedAxesMembers[qname(ns, localDimName)].add( qname(ns, localMemName) if localMemName != "*" else "*" ) nsMatch = True if nsMatch: break # use explicit year rules if available, else generic year rules return attrdict(concepts=concepts, excludedAxesMembers=excludedAxesMembers, excludedMembers=excludedMembers, excludedMemberNamesPattern=re.compile( "|".join(excludedMemberStrings), re.IGNORECASE) if excludedMemberStrings else None)
def loadNonNegativeFacts(modelXbrl): _file = openFileStream(modelXbrl.modelManager.cntlr, resourcesFilePath(modelXbrl.modelManager, "signwarnings.json"), 'rt', encoding='utf-8') signwarnings = json.load(_file) # {localName: date, ...} _file.close() concepts = set() excludedAxesMembers = defaultdict(set) for modelDocument in modelXbrl.urlDocs.values(): ns = modelDocument.targetNamespace # set up non neg lookup by full NS for abbrNs in (abbreviatedNamespace(ns), abbreviatedWildNamespace(ns)): for localName in signwarnings["conceptNames"].get(abbrNs, ()): concepts.add(qname(ns, localName)) for localDimName, localMemNames in signwarnings["excludedAxesMembers"].get(abbrNs, EMPTY_DICT).items(): for localMemName in localMemNames: excludedAxesMembers[qname(ns, localDimName)].add(qname(ns, localMemName) if localMemName != "*" else None) return attrdict(concepts=concepts, excludedAxesMembers=excludedAxesMembers)
def loadNonNegativeFacts(modelXbrl): _file = openFileStream(modelXbrl.modelManager.cntlr, resourcesFilePath(modelXbrl.modelManager, "signwarnings.json"), 'rt', encoding='utf-8') signwarnings = json.load(_file) # {localName: date, ...} _file.close() concepts = set() excludedAxesMembers = defaultdict(set) for modelDocument in modelXbrl.urlDocs.values(): ns = modelDocument.targetNamespace # set up non neg lookup by full NS for abbrNs in (abbreviatedNamespace(ns), abbreviatedWildNamespace(ns)): for localName in signwarnings["conceptNames"].get(abbrNs, ()): concepts.add(qname(ns, localName)) for localDimName, localMemNames in signwarnings["excludedAxesMembers"].get(abbrNs, EMPTY_DICT).items(): for localMemName in localMemNames: excludedAxesMembers[qname(ns, localDimName)].add(qname(ns, localMemName) if localMemName != "*" else "*") return attrdict(concepts=concepts, excludedAxesMembers=excludedAxesMembers)
def _load_cache(val): """ Loads the cached taxonomy default demensions. If the file isn't cached yet it will create a new cache :param val: ValidateXbrl to be validated :type val: :class: '~arelle.ValidateXbrl.ValidateXbrl' :return: no explicit return, but loads caches for dqc_us_0041 :rtype: None """ val.linroleDefinitionIsDisclosure = ( re.compile(r"-\s+Disclosure\s+-\s", re.IGNORECASE) ) val.linkroleDefinitionStatementSheet = ( re.compile(r"[^-]+-\s+Statement\s+-\s+.*", re.IGNORECASE) ) # no restriction to type of statement val.ugtNamespace = None cntlr = val.modelXbrl.modelManager.cntlr year = _EARLIEST_GAAP_YEAR for ugt in ugtDocs: ugt_namespace = ugt["namespace"] if _is_in_namespace(val, ugt_namespace): ugt_default_dimensions_json_file = os.path.join( os.path.dirname(__file__), 'resources', 'DQC_US_0041', '{}_ugt-default-dimensions.json'.format(str(year)) ) file = None try: file = openFileStream( cntlr, ugt_default_dimensions_json_file, 'rt', encoding='utf-8' ) val.usgaapDefaultDimensions = json.load(file) file.close() except FileNotFoundError: # noqa if file: file.close() year += 1
def _load_cache(val): """ Loads the needed deprecated concepts cache into memory :param val: ValidateXbrl to load the concepts into :type val: :class:'~arelle.ValidateXbrl.ValidateXbrl' :return: Return True if cache exists, False otherwise. :rtype: bool """ cntlr = val.modelXbrl.modelManager.cntlr year = _EARLIEST_US_GAAP_YEAR for ugt in ugtDocs: ugt_namespace = ugt["namespace"] if ((ugt_namespace in val.modelXbrl.namespaceDocs and len(val.modelXbrl.namespaceDocs[ugt_namespace]) > 0)): val.ugtNamespace = ugt_namespace deprecations_json_file = os.path.join( os.path.dirname(__file__), 'resources', 'DQC_US_0018', '{}_deprecated-concepts.json'.format(str(year)) ) file = None try: file = openFileStream( cntlr, deprecations_json_file, 'rt', encoding='utf-8' ) val.usgaapDeprecations = json.load(file) file.close() except FileNotFoundError: # noqa if file: file.close() # year should be cached. It is not, so return False return False # year should be cached, and is. Return True return True year += 1 # checked all years. No cache found. return False
def loadDqcRules(modelXbrl): # returns match expression, standard patterns # determine taxonomy usage by facts, must have more us-gaap facts than ifrs facts # (some ifrs filings may have a few us-gaap facts or us-gaap concepts loaded but are not us-gaap filings) namespaceUsage = {} for f in modelXbrl.facts: ns = f.qname.namespaceURI namespaceUsage[ns] = namespaceUsage.get(ns, 0) + 1 numUsGaapFacts = sum(n for ns,n in namespaceUsage.items() if "us-gaap" in ns) numIfrsFacts = sum(n for ns,n in namespaceUsage.items() if "ifrs" in ns) if (usgaapYear(modelXbrl) >= "2020" and ((numUsGaapFacts == 0 and numIfrsFacts == 0) or (numUsGaapFacts > numIfrsFacts))): # found us-gaap facts present (more than ifrs facts present), load us-gaap DQC.US rules _file = openFileStream(modelXbrl.modelManager.cntlr, resourcesFilePath(modelXbrl.modelManager, "dqc-us-rules.json"), 'rt', encoding='utf-8') dqcRules = json.load(_file, object_pairs_hook=OrderedDict) # preserve order of keys _file.close() return dqcRules return {}
def _load_cache(val): """ Loads the needed deprecated concepts cache into memory :param val: ValidateXbrl to load the concepts into :type val: :class:'~arelle.ValidateXbrl.ValidateXbrl' :return: Return True if cache exists, False otherwise. :rtype: bool """ cntlr = val.modelXbrl.modelManager.cntlr year = _EARLIEST_US_GAAP_YEAR for ugt in ugtDocs: ugt_namespace = ugt["namespace"] if ((ugt_namespace in val.modelXbrl.namespaceDocs and len(val.modelXbrl.namespaceDocs[ugt_namespace]) > 0)): val.ugtNamespace = ugt_namespace deprecations_json_file = os.path.join( os.path.dirname(__file__), 'resources', 'DQC_US_0018', '{}_deprecated-concepts.json'.format(str(year)) ) file = None try: file = openFileStream( cntlr, deprecations_json_file, 'rt', encoding='utf-8' ) val.usgaapDeprecations = json.load(file) file.close() except FileNotFoundError: if file: file.close() # year should be cached. It is not, so return False return False # year should be cached, and is. Return True return True year += 1 # checked all years. No cache found. return False
def _load_cache(val): """ Loads the cached taxonomy default demensions. If the file isn't cached yet it will create a new cache :param val: ValidateXbrl to be validated :type val: :class: '~arelle.ValidateXbrl.ValidateXbrl' :return: no explicit return, but loads caches for dqc_us_0041 :rtype: None """ val.linroleDefinitionIsDisclosure = (re.compile(r"-\s+Disclosure\s+-\s", re.IGNORECASE)) val.linkroleDefinitionStatementSheet = (re.compile( r"[^-]+-\s+Statement\s+-\s+.*", re.IGNORECASE)) # no restriction to type of statement val.ugtNamespace = None cntlr = val.modelXbrl.modelManager.cntlr year = _EARLIEST_GAAP_YEAR for ugt in ugtDocs: ugt_namespace = ugt["namespace"] if _is_in_namespace(val, ugt_namespace): ugt_default_dimensions_json_file = os.path.join( os.path.dirname(__file__), 'resources', 'DQC_US_0041', '{}_ugt-default-dimensions.json'.format(str(year))) file = None try: file = openFileStream(cntlr, ugt_default_dimensions_json_file, 'rt', encoding='utf-8') val.usgaapDefaultDimensions = json.load(file) file.close() except FileNotFoundError: # noqa if file: file.close() year += 1
def moduleModuleInfo(moduleURL, reload=False): #TODO several directories, eg User Application Data moduleFilename = _cntlr.webCache.getfilename(moduleURL, reload=reload, normalize=True, base=_pluginBase) if moduleFilename: f = None try: # if moduleFilename is a directory containing an __ini__.py file, open that instead if os.path.isdir(moduleFilename) and os.path.isfile( os.path.join(moduleFilename, "__init__.py")): moduleFilename = os.path.join(moduleFilename, "__init__.py") f = openFileStream(_cntlr, moduleFilename) tree = ast.parse(f.read(), filename=moduleFilename) for item in tree.body: if isinstance(item, ast.Assign): attr = item.targets[0].id if attr == "__pluginInfo__": f.close() moduleInfo = {} classMethods = [] for i, key in enumerate(item.value.keys): _key = key.s _value = item.value.values[i] _valueType = _value.__class__.__name__ if _valueType == 'Str': moduleInfo[_key] = _value.s elif _valueType == 'Name': classMethods.append(_key) moduleInfo['classMethods'] = classMethods moduleInfo["moduleURL"] = moduleURL moduleInfo["status"] = 'enabled' moduleInfo["fileDate"] = time.strftime( '%Y-%m-%dT%H:%M:%S UTC', time.gmtime(os.path.getmtime(moduleFilename))) return moduleInfo except EnvironmentError: pass if f: f.close() return None
def loadNonNegativeFacts(modelXbrl): _file = openFileStream(modelXbrl.modelManager.cntlr, resourcesFilePath(modelXbrl.modelManager, "signwarnings.json"), 'rt', encoding='utf-8') signwarnings = json.load(_file) # {localName: date, ...} _file.close() concepts = set() excludedMembers = set() excludedMemberStrings = set() excludedAxesMembers = defaultdict(set) for modelDocument in modelXbrl.urlDocs.values(): ns = modelDocument.targetNamespace # set up non neg lookup by full NS for abbrNs in (abbreviatedNamespace(ns), abbreviatedNamespace(ns, WILD)): nsMatch = False for exName, exSet, isQName in (("conceptNames", concepts, True), ("excludedMemberNames", excludedMembers, True), ("excludedMemberStrings", excludedMemberStrings, False)): for localName in signwarnings[exName].get(abbrNs, ()): exSet.add(qname(ns, localName) if isQName else localName) nsMatch = True for localDimName, localMemNames in signwarnings[ "excludedAxesMembers"].get(abbrNs, EMPTY_DICT).items(): for localMemName in localMemNames: excludedAxesMembers[qname(ns, localDimName)].add( qname(ns, localMemName) if localMemName != "*" else "*" ) nsMatch = True if nsMatch: break # use explicit year rules if available, else generic year rules return attrdict(concepts=concepts, excludedAxesMembers=excludedAxesMembers, excludedMembers=excludedMembers, excludedMemberNamesPattern=re.compile( "|".join(excludedMemberStrings), re.IGNORECASE) if excludedMemberStrings else None)
def loadDqcRules(modelXbrl): # returns match expression, standard patterns # determine taxonomy usage by facts, must have more us-gaap facts than ifrs facts # (some ifrs filings may have a few us-gaap facts or us-gaap concepts loaded but are not us-gaap filings) namespaceUsage = {} for f in modelXbrl.facts: ns = f.qname.namespaceURI namespaceUsage[ns] = namespaceUsage.get(ns, 0) + 1 if (modelXbrl.modelManager.disclosureSystem.version[0] >= 56 and # EDGAR release >= 21.1 sum(n for ns, n in namespaceUsage.items() if "us-gaap" in ns) > # mostly us-gaap elements, not ifrs elements sum(n for ns, n in namespaceUsage.items() if "ifrs" in ns)): # found us-gaap facts present (more than ifrs facts present), load us-gaap DQC.US rules _file = openFileStream(modelXbrl.modelManager.cntlr, resourcesFilePath(modelXbrl.modelManager, "dqc-us-rules.json"), 'rt', encoding='utf-8') dqcRules = json.load( _file, object_pairs_hook=OrderedDict) # preserve order of keys _file.close() return dqcRules return {}
def moduleModuleInfo(moduleURL, reload=False): #TODO several directories, eg User Application Data moduleFilename = _cntlr.webCache.getfilename(moduleURL, reload=reload, normalize=True, base=_pluginBase) if moduleFilename: f = None try: # if moduleFilename is a directory containing an __ini__.py file, open that instead if os.path.isdir(moduleFilename) and os.path.isfile(os.path.join(moduleFilename, "__init__.py")): moduleFilename = os.path.join(moduleFilename, "__init__.py") f = openFileStream(_cntlr, moduleFilename) tree = ast.parse(f.read(), filename=moduleFilename) for item in tree.body: if isinstance(item, ast.Assign): attr = item.targets[0].id if attr == "__pluginInfo__": f.close() moduleInfo = {} classMethods = [] for i, key in enumerate(item.value.keys): _key = key.s _value = item.value.values[i] _valueType = _value.__class__.__name__ if _valueType == 'Str': moduleInfo[_key] = _value.s elif _valueType == 'Name': classMethods.append(_key) moduleInfo['classMethods'] = classMethods moduleInfo["moduleURL"] = moduleURL moduleInfo["status"] = 'enabled' moduleInfo["fileDate"] = time.strftime('%Y-%m-%dT%H:%M:%S UTC', time.gmtime(os.path.getmtime(moduleFilename))) return moduleInfo except EnvironmentError: pass if f: f.close() return None
def loadDeiValidations(modelXbrl, isInlineXbrl): _file = openFileStream(modelXbrl.modelManager.cntlr, resourcesFilePath(modelXbrl.modelManager, "dei-validations.json"), 'rt', encoding='utf-8') validations = json.load(_file) # {localName: date, ...} _file.close() #print ("original validations size {}".format(pyObjectSize(validations))) # get dei namespaceURI deiNamespaceURI = None for doc in modelXbrl.urlDocs.values(): if doc.targetNamespace and doc.targetNamespace.startswith("http://xbrl.sec.gov/dei/"): deiNamespaceURI = doc.targetNamespace break # compile form-classes fc = validations["form-classes"] def compileFormSet(forms, formSet=None, visitedClasses=None): if visitedClasses is None: visitedClasses = set() if formSet is None: formSet = set() for form in flattenSequence(forms): if form.startswith("@"): referencedClass = form[1:] if referencedClass not in fc: modelXbrl.error("arelle:loadDeiValidations", _("Missing declaration for %(referencedClass)s."), referencedClass=form) elif form in visitedClasses: modelXbrl.error("arelle:loadDeiValidations", _("Circular reference to %(formClass)s in %(formClasses)s."), formClass=referencedClass, formClasses=sorted(visitedClasses)) else: visitedClasses.add(form) compileFormSet(fc[referencedClass], formSet, visitedClasses) else: formSet.add(form) return formSet for fev in validations["form-element-validations"]: for field in ("xbrl-names", "validation", "efm", "source") : if field not in fev: modelXbrl.error("arelle:loadDeiValidations", _("Missing form-element-validation[\"%(field)s\"] from %(validation)s."), field=field, validation=fev) if "severity" in fev and not any(field.startswith("message") for field in fev): modelXbrl.error("arelle:loadDeiValidations", _("Missing form-element-validation[\"%(field)s\"] from %(validation)s."), field="message*", validation=fev) validationCode = fev.get("validation") if validationCode in ("f2", "og", "ol1", "ol2", "oph", "ar", "sr", "oth", "t", "tb", "t1") and "references" not in fev: modelXbrl.error("arelle:loadDeiValidations", _("Missing form-element-validation[\"references\"] from %(validation)s."), field=field, validation=fev) if validationCode in ("ru", "ou"): if isinstance(fev.get("value"), list): fev["value"] = set(fev["value"]) # change options list into set else: modelXbrl.error("arelle:loadDeiValidations", _("Missing form-element-validation[\"value\"] from %(validation)s, must be a list."), field=field, validation=fev) if validationCode in (): if isinstance(fev.get("reference-value"), list): fev["reference-value"] = set(fev["reference-value"]) # change options list into set else: modelXbrl.error("arelle:loadDeiValidations", _("Missing form-element-validation[\"value\"] from %(validation)s, must be a list."), field=field, validation=fev) if validationCode not in validations["validations"]: modelXbrl.error("arelle:loadDeiValidations", _("Missing validation[\"%(validationCode)s\"]."), validationCode=validationCode) axisCode = fev.get("axis") if axisCode and axisCode not in validations["axis-validations"]: modelXbrl.error("arelle:loadDeiValidations", _("Missing axis[\"%(axisCode)s\"]."), axisCode=axisCode) if "lang" in fev: fev["langPattern"] = re.compile(fev["lang"]) s = fev.get("source") if s not in ("inline", "non-inline", "both"): modelXbrl.error("arelle:loadDeiValidations", _("Invalid source [\"%(source)s\"]."), source=s) elif (isInlineXbrl and s in ("inline", "both")) or (not isInlineXbrl and s in ("non-inline", "both")): messageKey = fev.get("message") if messageKey and messageKey not in validations["messages"]: modelXbrl.error("arelle:loadDeiValidations", _("Missing message[\"%(messageKey)s\"]."), messageKey=messageKey) # only include dei names in current dei taxonomy fev["xbrl-names"] = [name for name in flattenSequence(fev.get("xbrl-names", ())) if qname(deiNamespaceURI, name) in modelXbrl.qnameConcepts] formSet = compileFormSet(fev.get("forms", (fev.get("form",()),))) if "*" in formSet: formSet = "all" # change to string for faster testing in Filing.py fev["formSet"] = formSet for axisKey, axisValidation in validations["axis-validations"].items(): messageKey = axisValidation.get("message") if messageKey and messageKey not in validations["messages"]: modelXbrl.error("arelle:loadDeiValidations", _("Missing axis \"%(axisKey)s\" message[\"%(messageKey)s\"]."), axisKey=axisKey, messageKey=messageKey) for valKey, validation in validations["validations"].items(): messageKey = validation.get("message") if messageKey and messageKey not in validations["messages"]: modelXbrl.error("arelle:loadDeiValidations", _("Missing validation \"%(valKey)s\" message[\"%(messageKey)s\"]."), valKey=valKey, messageKey=messageKey) #print ("compiled validations size {}".format(pyObjectSize(validations))) return validations
def moduleModuleInfo(moduleURL, reload=False, parentImportsSubtree=False): #TODO several directories, eg User Application Data moduleFilename = _cntlr.webCache.getfilename(moduleURL, reload=reload, normalize=True, base=_pluginBase) if moduleFilename: f = None try: # if moduleFilename is a directory containing an __ini__.py file, open that instead if os.path.isdir(moduleFilename) and os.path.isfile(os.path.join(moduleFilename, "__init__.py")): moduleFilename = os.path.join(moduleFilename, "__init__.py") moduleDir = os.path.dirname(moduleFilename) f = openFileStream(_cntlr, moduleFilename) tree = ast.parse(f.read(), filename=moduleFilename) moduleImports = [] for item in tree.body: if isinstance(item, ast.Assign): attr = item.targets[0].id if attr == "__pluginInfo__": f.close() moduleInfo = {"name":None} classMethods = [] importURLs = [] for i, key in enumerate(item.value.keys): _key = key.s _value = item.value.values[i] _valueType = _value.__class__.__name__ if _key == "import": if _valueType == 'Str': importURLs.append(_value.s) elif _valueType in ("List", "Tuple"): for elt in _value.elts: importURLs.append(elt.s) elif _valueType == 'Str': moduleInfo[_key] = _value.s elif _valueType == 'Name': classMethods.append(_key) elif _key == "imports" and _valueType in ("List", "Tuple"): importURLs = [elt.s for elt in _value.elts] moduleInfo['classMethods'] = classMethods moduleInfo["moduleURL"] = moduleURL moduleInfo["status"] = 'enabled' moduleInfo["fileDate"] = time.strftime('%Y-%m-%dT%H:%M:%S UTC', time.gmtime(os.path.getmtime(moduleFilename))) mergedImportURLs = [] _moduleImportsSubtree = False for _url in importURLs: if _url.startswith("module_import"): for moduleImport in moduleImports: mergedImportURLs.append(moduleImport + ".py") if _url == "module_import_subtree": _moduleImportsSubtree = True else: mergedImportURLs.append(_url) if parentImportsSubtree and not _moduleImportsSubtree: _moduleImportsSubtree = True for moduleImport in moduleImports: mergedImportURLs.append(moduleImport + ".py") imports = [] for _url in mergedImportURLs: _importURL = (_url if isAbsolute(_url) or os.path.isabs(_url) else os.path.join(os.path.dirname(moduleURL), _url)) _importModuleInfo = moduleModuleInfo(_importURL, reload, _moduleImportsSubtree) if _importModuleInfo: _importModuleInfo["isImported"] = True imports.append(_importModuleInfo) moduleInfo["imports"] = imports return moduleInfo elif isinstance(item, ast.ImportFrom): if item.level == 1: # starts with . if item.module is None: # from . import module1, module2, ... for importee in item.names: if (os.path.isfile(os.path.join(moduleDir, importee.name + ".py")) and importee.name not in moduleImports): moduleImports.append(importee.name) else: modulePkgs = item.module.split('.') modulePath = os.path.join(*modulePkgs) if (os.path.isfile(os.path.join(moduleDir, modulePath) + ".py") and modulePath not in moduleImports): moduleImports.append(modulePath) for importee in item.names: _importeePfxName = os.path.join(modulePath, importee.name) if (os.path.isfile(os.path.join(moduleDir, _importeePfxName) + ".py") and _importeePfxName not in moduleImports): moduleImports.append(_importeePfxName) except EnvironmentError: pass if f: f.close() return None
def moduleModuleInfo(moduleURL, reload=False, parentImportsSubtree=False): #TODO several directories, eg User Application Data moduleFilename = _cntlr.webCache.getfilename(moduleURL, reload=reload, normalize=True, base=_pluginBase) if moduleFilename: f = None try: # if moduleFilename is a directory containing an __ini__.py file, open that instead if os.path.isdir(moduleFilename) and os.path.isfile( os.path.join(moduleFilename, "__init__.py")): moduleFilename = os.path.join(moduleFilename, "__init__.py") moduleDir = os.path.dirname(moduleFilename) f = openFileStream(_cntlr, moduleFilename) tree = ast.parse(f.read(), filename=moduleFilename) moduleImports = [] for item in tree.body: if isinstance(item, ast.Assign): attr = item.targets[0].id if attr == "__pluginInfo__": f.close() moduleInfo = {"name": None} classMethods = [] importURLs = [] for i, key in enumerate(item.value.keys): _key = key.s _value = item.value.values[i] _valueType = _value.__class__.__name__ if _key == "import": if _valueType == 'Str': importURLs.append(_value.s) elif _valueType in ("List", "Tuple"): for elt in _value.elts: importURLs.append(elt.s) elif _valueType == 'Str': moduleInfo[_key] = _value.s elif _valueType == 'Name': classMethods.append(_key) elif _key == "imports" and _valueType in ("List", "Tuple"): importURLs = [elt.s for elt in _value.elts] moduleInfo['classMethods'] = classMethods moduleInfo["moduleURL"] = moduleURL moduleInfo["status"] = 'enabled' moduleInfo["fileDate"] = time.strftime( '%Y-%m-%dT%H:%M:%S UTC', time.gmtime(os.path.getmtime(moduleFilename))) mergedImportURLs = [] _moduleImportsSubtree = False for _url in importURLs: if _url.startswith("module_import"): for moduleImport in moduleImports: mergedImportURLs.append(moduleImport + ".py") if _url == "module_import_subtree": _moduleImportsSubtree = True else: mergedImportURLs.append(_url) if parentImportsSubtree and not _moduleImportsSubtree: _moduleImportsSubtree = True for moduleImport in moduleImports: mergedImportURLs.append(moduleImport + ".py") imports = [] for _url in mergedImportURLs: _importURL = ( _url if isAbsolute(_url) or os.path.isabs(_url) else os.path.join( os.path.dirname(moduleURL), _url)) _importModuleInfo = moduleModuleInfo( _importURL, reload, _moduleImportsSubtree) if _importModuleInfo: _importModuleInfo["isImported"] = True imports.append(_importModuleInfo) moduleInfo["imports"] = imports return moduleInfo elif isinstance(item, ast.ImportFrom): if item.level == 1: # starts with . if item.module is None: # from . import module1, module2, ... for importee in item.names: if (os.path.isfile( os.path.join(moduleDir, importee.name + ".py")) and importee.name not in moduleImports): moduleImports.append(importee.name) else: modulePkgs = item.module.split('.') modulePath = os.path.join(*modulePkgs) if (os.path.isfile( os.path.join(moduleDir, modulePath) + ".py") and modulePath not in moduleImports): moduleImports.append(modulePath) for importee in item.names: _importeePfxName = os.path.join( modulePath, importee.name) if (os.path.isfile( os.path.join(moduleDir, _importeePfxName) + ".py") and _importeePfxName not in moduleImports): moduleImports.append(_importeePfxName) except EnvironmentError: pass if f: f.close() return None
def setup(val): if not val.validateLoggingSemantic: # all checks herein are SEMANTIC return val.linroleDefinitionIsDisclosure = re.compile(r"-\s+Disclosure\s+-\s", re.IGNORECASE) val.linkroleDefinitionStatementSheet = re.compile(r"[^-]+-\s+Statement\s+-\s+.*", # no restriction to type of statement re.IGNORECASE) val.ugtNamespace = None cntlr = val.modelXbrl.modelManager.cntlr # load deprecated concepts for filed year of us-gaap for ugt in ugtDocs: ugtNamespace = ugt["namespace"] if ugtNamespace in val.modelXbrl.namespaceDocs and len(val.modelXbrl.namespaceDocs[ugtNamespace]) > 0: val.ugtNamespace = ugtNamespace usgaapDoc = val.modelXbrl.namespaceDocs[ugtNamespace][0] deprecationsJsonFile = usgaapDoc.filepathdir + os.sep + "deprecated-concepts.json" file = None try: file = openFileStream(cntlr, deprecationsJsonFile, 'rt', encoding='utf-8') val.usgaapDeprecations = json.load(file) file.close() except Exception: if file: file.close() val.modelXbrl.modelManager.addToLog(_("loading us-gaap {0} deprecated concepts into cache").format(ugt["year"])) startedAt = time.time() ugtDocLB = ugt["docLB"] val.usgaapDeprecations = {} # load without SEC/EFM validation (doc file would not be acceptable) priorValidateDisclosureSystem = val.modelXbrl.modelManager.validateDisclosureSystem val.modelXbrl.modelManager.validateDisclosureSystem = False deprecationsInstance = ModelXbrl.load(val.modelXbrl.modelManager, # "http://xbrl.fasb.org/us-gaap/2012/elts/us-gaap-doc-2012-01-31.xml", # load from zip (especially after caching) is incredibly faster openFileSource(ugtDocLB, cntlr), _("built deprecations table in cache")) val.modelXbrl.modelManager.validateDisclosureSystem = priorValidateDisclosureSystem if deprecationsInstance is None: val.modelXbrl.error("arelle:notLoaded", _("US-GAAP documentation not loaded: %(file)s"), modelXbrl=val, file=os.path.basename(ugtDocLB)) else: # load deprecations for labelRel in deprecationsInstance.relationshipSet(XbrlConst.conceptLabel).modelRelationships: modelDocumentation = labelRel.toModelObject conceptName = labelRel.fromModelObject.name if modelDocumentation.role == 'http://www.xbrl.org/2009/role/deprecatedLabel': val.usgaapDeprecations[conceptName] = (val.usgaapDeprecations.get(conceptName, ('',''))[0], modelDocumentation.text) elif modelDocumentation.role == 'http://www.xbrl.org/2009/role/deprecatedDateLabel': val.usgaapDeprecations[conceptName] = (modelDocumentation.text, val.usgaapDeprecations.get(conceptName, ('',''))[1]) jsonStr = _STR_UNICODE(json.dumps(val.usgaapDeprecations, ensure_ascii=False, indent=0)) # might not be unicode in 2.7 saveFile(cntlr, deprecationsJsonFile, jsonStr) # 2.7 gets unicode this way deprecationsInstance.close() del deprecationsInstance # dereference closed modelXbrl val.modelXbrl.profileStat(_("build us-gaap deprecated concepts cache"), time.time() - startedAt) ugtCalcsJsonFile = usgaapDoc.filepathdir + os.sep + "ugt-calculations.json" ugtDefaultDimensionsJsonFile = usgaapDoc.filepathdir + os.sep + "ugt-default-dimensions.json" file = None try: file = openFileStream(cntlr, ugtCalcsJsonFile, 'rt', encoding='utf-8') val.usgaapCalculations = json.load(file) file.close() file = openFileStream(cntlr, ugtDefaultDimensionsJsonFile, 'rt', encoding='utf-8') val.usgaapDefaultDimensions = json.load(file) file.close() except Exception: if file: file.close() val.modelXbrl.modelManager.addToLog(_("loading us-gaap {0} calculations and default dimensions into cache").format(ugt["year"])) startedAt = time.time() ugtEntryXsd = ugt["entryXsd"] val.usgaapCalculations = {} val.usgaapDefaultDimensions = {} # load without SEC/EFM validation (doc file would not be acceptable) priorValidateDisclosureSystem = val.modelXbrl.modelManager.validateDisclosureSystem val.modelXbrl.modelManager.validateDisclosureSystem = False calculationsInstance = ModelXbrl.load(val.modelXbrl.modelManager, # "http://xbrl.fasb.org/us-gaap/2012/entire/us-gaap-entryPoint-std-2012-01-31.xsd", # load from zip (especially after caching) is incredibly faster openFileSource(ugtEntryXsd, cntlr), _("built us-gaap calculations cache")) val.modelXbrl.modelManager.validateDisclosureSystem = priorValidateDisclosureSystem if calculationsInstance is None: val.modelXbrl.error("arelle:notLoaded", _("US-GAAP calculations not loaded: %(file)s"), modelXbrl=val, file=os.path.basename(ugtEntryXsd)) else: # load calculations for ELR in calculationsInstance.relationshipSet(XbrlConst.summationItem).linkRoleUris: elrRelSet = calculationsInstance.relationshipSet(XbrlConst.summationItem, ELR) definition = "" for roleType in calculationsInstance.roleTypes.get(ELR,()): definition = roleType.definition break isStatementSheet = bool(val.linkroleDefinitionStatementSheet.match(definition)) elrUgtCalcs = {"#roots": [c.name for c in elrRelSet.rootConcepts], "#definition": definition, "#isStatementSheet": isStatementSheet} for relFrom, rels in elrRelSet.fromModelObjects().items(): elrUgtCalcs[relFrom.name] = [rel.toModelObject.name for rel in rels] val.usgaapCalculations[ELR] = elrUgtCalcs jsonStr = _STR_UNICODE(json.dumps(val.usgaapCalculations, ensure_ascii=False, indent=0)) # might not be unicode in 2.7 saveFile(cntlr, ugtCalcsJsonFile, jsonStr) # 2.7 gets unicode this way # load default dimensions for defaultDimRel in calculationsInstance.relationshipSet(XbrlConst.dimensionDefault).modelRelationships: if defaultDimRel.fromModelObject is not None and defaultDimRel.toModelObject is not None: val.usgaapDefaultDimensions[defaultDimRel.fromModelObject.name] = defaultDimRel.toModelObject.name jsonStr = _STR_UNICODE(json.dumps(val.usgaapDefaultDimensions, ensure_ascii=False, indent=0)) # might not be unicode in 2.7 saveFile(cntlr, ugtDefaultDimensionsJsonFile, jsonStr) # 2.7 gets unicode this way calculationsInstance.close() del calculationsInstance # dereference closed modelXbrl val.modelXbrl.profileStat(_("build us-gaap calculations and default dimensions cache"), time.time() - startedAt) break val.deprecatedFactConcepts = defaultdict(list) val.deprecatedDimensions = defaultdict(list) val.deprecatedMembers = defaultdict(list)
def loadDqc0015signwarningRules(modelXbrl): conceptRule = "http://fasb.org/dqcrules/arcrole/concept-rule" # FASB arcrule rule0015 = "http://fasb.org/us-gaap/role/dqc/0015" modelManager = modelXbrl.modelManager cntlr = modelXbrl.modelManager.cntlr # check for cached completed signwarnings _signwarningsFileName = resourcesFilePath(modelManager, "signwarnings.json") if os.path.exists(_signwarningsFileName): _file = openFileStream(modelManager.cntlr, _signwarningsFileName, 'rt', encoding='utf-8') signwarnings = json.load(_file) # {localName: date, ...} _file.close() return signwarnings # load template rules _fileName = resourcesFilePath(modelManager, "signwarnings-template.json") if _fileName: _file = openFileStream(modelXbrl.modelManager.cntlr, _fileName, 'rt', encoding='utf-8') signwarnings = json.load(_file, object_pairs_hook=OrderedDict) # {localName: date, ...} _file.close() # load rules and add to signwarnings template for dqcAbbr, dqcrtUrl in latestDqcrtDocs.items(): modelManager.addToLog(_("loading {} DQC Rules {}").format(dqcAbbr, dqcrtUrl), messageCode="info") # load without SEC/EFM validation (doc file would not be acceptable) priorValidateDisclosureSystem = modelManager.validateDisclosureSystem modelManager.validateDisclosureSystem = False from arelle import ModelXbrl dqcrtInstance = ModelXbrl.load(modelManager, # "http://xbrl.fasb.org/us-gaap/2012/elts/us-gaap-doc-2012-01-31.xml", # load from zip (especially after caching) is incredibly faster openFileSource(dqcrtUrl, cntlr), _("built dqcrt table in cache")) modelManager.validateDisclosureSystem = priorValidateDisclosureSystem if dqcrtInstance is None: modelManager.addToLog( _("%(name)s documentation not loaded"), messageCode="arelle:notLoaded", messageArgs={"modelXbrl": val, "name":dqcAbbr}) else: # load signwarnings from DQC 0015 dqcRelSet = dqcrtInstance.relationshipSet(conceptRule, rule0015) for signWrnObj, headEltName in (("conceptNames", "Dqc_0015_ListOfElements"), ("excludedMemberNames", "Dqc_0015_ExcludeNonNegMembersAbstract"), ("excludedAxesMembers", "Dqc_0015_ExcludeNonNegAxisAbstract"), ("excludedAxesMembers", "Dqc_0015_ExcludeNonNegAxisMembersAbstract"), ("excludedMemberStrings", "Dqc_0015_ExcludeNonNegMemberStringsAbstract")): headElts = dqcrtInstance.nameConcepts.get(headEltName,()) for headElt in headElts: if signWrnObj == "excludedMemberStrings": for refRel in dqcrtInstance.relationshipSet(XbrlConst.conceptReference).fromModelObject(headElt): for refPart in refRel.toModelObject.iterchildren("{*}allowableSubString"): for subStr in refPart.text.split(): signwarnings[signWrnObj].setdefault(nsAbbr, []).append(subStr) else: for ruleRel in dqcRelSet.fromModelObject(headElt): elt = ruleRel.toModelObject nsAbbr = abbreviatedNamespace(elt.qname.namespaceURI) if signWrnObj in ("conceptNames", "excludedMemberNames"): signwarnings[signWrnObj].setdefault(nsAbbr, []).append(elt.name) else: l = signwarnings[signWrnObj].setdefault(nsAbbr, {}).setdefault(elt.name, []) if headEltName == "Dqc_0015_ExcludeNonNegAxisAbstract": l.append("*") else: for memRel in dqcRelSet.fromModelObject(elt): l.append(memRel.toModelObject.name) jsonStr = _STR_UNICODE(json.dumps(signwarnings, ensure_ascii=False, indent=2)) # might not be unicode in 2.7 saveFile(cntlr, _signwarningsFileName, jsonStr) # 2.7 gets unicode this way dqcrtInstance.close() del dqcrtInstance # dereference closed modelXbrl return signwarnings
def setup(val): val.linroleDefinitionIsDisclosure = re.compile(r"-\s+Disclosure\s+-\s", re.IGNORECASE) val.linkroleDefinitionStatementSheet = re.compile(r"[^-]+-\s+Statement\s+-\s+.*", # no restriction to type of statement re.IGNORECASE) val.ugtNamespace = None cntlr = val.modelXbrl.modelManager.cntlr # load deprecated concepts for filed year of us-gaap for ugt in ugtDocs: ugtNamespace = ugt["namespace"] if ugtNamespace in val.modelXbrl.namespaceDocs and len(val.modelXbrl.namespaceDocs[ugtNamespace]) > 0: val.ugtNamespace = ugtNamespace usgaapDoc = val.modelXbrl.namespaceDocs[ugtNamespace][0] deprecationsJsonFile = usgaapDoc.filepathdir + os.sep + "deprecated-concepts.json" file = None try: file = openFileStream(cntlr, deprecationsJsonFile, 'rt', encoding='utf-8') val.usgaapDeprecations = json.load(file) file.close() except Exception: if file: file.close() val.modelXbrl.modelManager.addToLog(_("loading us-gaap {0} deprecated concepts into cache").format(ugt["year"])) startedAt = time.time() ugtDocLB = ugt["docLB"] val.usgaapDeprecations = {} # load without SEC/EFM validation (doc file would not be acceptable) priorValidateDisclosureSystem = val.modelXbrl.modelManager.validateDisclosureSystem val.modelXbrl.modelManager.validateDisclosureSystem = False deprecationsInstance = ModelXbrl.load(val.modelXbrl.modelManager, # "http://xbrl.fasb.org/us-gaap/2012/elts/us-gaap-doc-2012-01-31.xml", # load from zip (especially after caching) is incredibly faster openFileSource(ugtDocLB, cntlr), _("built deprecations table in cache")) val.modelXbrl.modelManager.validateDisclosureSystem = priorValidateDisclosureSystem if deprecationsInstance is None: val.modelXbrl.error("arelle:notLoaded", _("US-GAAP documentation not loaded: %(file)s"), modelXbrl=val, file=os.path.basename(ugtDocLB)) else: # load deprecations for labelRel in deprecationsInstance.relationshipSet(XbrlConst.conceptLabel).modelRelationships: modelDocumentation = labelRel.toModelObject conceptName = labelRel.fromModelObject.name if modelDocumentation.role == 'http://www.xbrl.org/2009/role/deprecatedLabel': val.usgaapDeprecations[conceptName] = (val.usgaapDeprecations.get(conceptName, ('',''))[0], modelDocumentation.text) elif modelDocumentation.role == 'http://www.xbrl.org/2009/role/deprecatedDateLabel': val.usgaapDeprecations[conceptName] = (modelDocumentation.text, val.usgaapDeprecations.get(conceptName, ('',''))[1]) jsonStr = _STR_UNICODE(json.dumps(val.usgaapDeprecations, ensure_ascii=False, indent=0)) # might not be unicode in 2.7 saveFile(cntlr, deprecationsJsonFile, jsonStr) # 2.7 gets unicode this way deprecationsInstance.close() del deprecationsInstance # dereference closed modelXbrl val.modelXbrl.profileStat(_("build us-gaap deprecated concepts cache"), time.time() - startedAt) ugtCalcsJsonFile = usgaapDoc.filepathdir + os.sep + "ugt-calculations.json" ugtDefaultDimensionsJsonFile = usgaapDoc.filepathdir + os.sep + "ugt-default-dimensions.json" file = None try: file = openFileStream(cntlr, ugtCalcsJsonFile, 'rt', encoding='utf-8') val.usgaapCalculations = json.load(file) file.close() file = openFileStream(cntlr, ugtDefaultDimensionsJsonFile, 'rt', encoding='utf-8') val.usgaapDefaultDimensions = json.load(file) file.close() except Exception: if file: file.close() val.modelXbrl.modelManager.addToLog(_("loading us-gaap {0} calculations and default dimensions into cache").format(ugt["year"])) startedAt = time.time() ugtEntryXsd = ugt["entryXsd"] val.usgaapCalculations = {} val.usgaapDefaultDimensions = {} # load without SEC/EFM validation (doc file would not be acceptable) priorValidateDisclosureSystem = val.modelXbrl.modelManager.validateDisclosureSystem val.modelXbrl.modelManager.validateDisclosureSystem = False calculationsInstance = ModelXbrl.load(val.modelXbrl.modelManager, # "http://xbrl.fasb.org/us-gaap/2012/entire/us-gaap-entryPoint-std-2012-01-31.xsd", # load from zip (especially after caching) is incredibly faster openFileSource(ugtEntryXsd, cntlr), _("built us-gaap calculations cache")) val.modelXbrl.modelManager.validateDisclosureSystem = priorValidateDisclosureSystem if calculationsInstance is None: val.modelXbrl.error("arelle:notLoaded", _("US-GAAP calculations not loaded: %(file)s"), modelXbrl=val, file=os.path.basename(ugtEntryXsd)) else: # load calculations for ELR in calculationsInstance.relationshipSet(XbrlConst.summationItem).linkRoleUris: elrRelSet = calculationsInstance.relationshipSet(XbrlConst.summationItem, ELR) definition = "" for roleType in calculationsInstance.roleTypes.get(ELR,()): definition = roleType.definition break isStatementSheet = bool(val.linkroleDefinitionStatementSheet.match(definition)) elrUgtCalcs = {"#roots": [c.name for c in elrRelSet.rootConcepts], "#definition": definition, "#isStatementSheet": isStatementSheet} for relFrom, rels in elrRelSet.fromModelObjects().items(): elrUgtCalcs[relFrom.name] = [rel.toModelObject.name for rel in rels] val.usgaapCalculations[ELR] = elrUgtCalcs jsonStr = _STR_UNICODE(json.dumps(val.usgaapCalculations, ensure_ascii=False, indent=0)) # might not be unicode in 2.7 saveFile(cntlr, ugtCalcsJsonFile, jsonStr) # 2.7 gets unicode this way # load default dimensions for defaultDimRel in calculationsInstance.relationshipSet(XbrlConst.dimensionDefault).modelRelationships: if defaultDimRel.fromModelObject is not None and defaultDimRel.toModelObject is not None: val.usgaapDefaultDimensions[defaultDimRel.fromModelObject.name] = defaultDimRel.toModelObject.name jsonStr = _STR_UNICODE(json.dumps(val.usgaapDefaultDimensions, ensure_ascii=False, indent=0)) # might not be unicode in 2.7 saveFile(cntlr, ugtDefaultDimensionsJsonFile, jsonStr) # 2.7 gets unicode this way calculationsInstance.close() del calculationsInstance # dereference closed modelXbrl val.modelXbrl.profileStat(_("build us-gaap calculations and default dimensions cache"), time.time() - startedAt) break val.deprecatedFactConcepts = defaultdict(list) val.deprecatedDimensions = defaultdict(list) val.deprecatedMembers = defaultdict(list)
def moduleModuleInfo(moduleURL, reload=False, parentImportsSubtree=False): #TODO several directories, eg User Application Data moduleFilename = _cntlr.webCache.getfilename(moduleURL, reload=reload, normalize=True, base=_pluginBase) if moduleFilename: f = None try: # if moduleFilename is a directory containing an __ini__.py file, open that instead if os.path.isdir(moduleFilename) and os.path.isfile(os.path.join(moduleFilename, "__init__.py")): moduleFilename = os.path.join(moduleFilename, "__init__.py") moduleDir = os.path.dirname(moduleFilename) if PLUGIN_TRACE_FILE: with open(PLUGIN_TRACE_FILE, "at", encoding='utf-8') as fh: fh.write("Scanning module for plug-in info: {}\n".format(moduleFilename)) f = openFileStream(_cntlr, moduleFilename) tree = ast.parse(f.read(), filename=moduleFilename) moduleImports = [] for item in tree.body: if isinstance(item, ast.Assign): attr = item.targets[0].id if attr == "__pluginInfo__": f.close() moduleInfo = {"name":None} classMethods = [] importURLs = [] for i, key in enumerate(item.value.keys): _key = key.s _value = item.value.values[i] _valueType = _value.__class__.__name__ if _key == "import": if _valueType == 'Str': importURLs.append(_value.s) elif _valueType in ("List", "Tuple"): for elt in _value.elts: importURLs.append(elt.s) elif _valueType == 'Str': moduleInfo[_key] = _value.s elif _valueType == 'Name': classMethods.append(_key) elif _key == "imports" and _valueType in ("List", "Tuple"): importURLs = [elt.s for elt in _value.elts] moduleInfo['classMethods'] = classMethods moduleInfo["moduleURL"] = moduleURL moduleInfo["status"] = 'enabled' moduleInfo["fileDate"] = time.strftime('%Y-%m-%dT%H:%M:%S UTC', time.gmtime(os.path.getmtime(moduleFilename))) mergedImportURLs = [] _moduleImportsSubtree = False for _url in importURLs: if _url.startswith("module_import"): for moduleImport in moduleImports: mergedImportURLs.append(moduleImport + ".py") if _url == "module_import_subtree": _moduleImportsSubtree = True else: mergedImportURLs.append(_url) if parentImportsSubtree and not _moduleImportsSubtree: _moduleImportsSubtree = True for moduleImport in moduleImports: mergedImportURLs.append(moduleImport + ".py") imports = [] for _url in mergedImportURLs: if isAbsolute(_url) or os.path.isabs(_url): _importURL = _url # URL is absolute http or local file system else: # check if exists relative to this module's directory _importURL = os.path.join(os.path.dirname(moduleURL), _url) if not os.path.exists(_importURL): # not relative to this plugin, assume standard plugin base _importURL = os.path.join(_pluginBase, _url) _importModuleInfo = moduleModuleInfo(_importURL, reload, _moduleImportsSubtree) if _importModuleInfo: _importModuleInfo["isImported"] = True imports.append(_importModuleInfo) moduleInfo["imports"] = imports return moduleInfo elif isinstance(item, ast.ImportFrom): if item.level == 1: # starts with . if item.module is None: # from . import module1, module2, ... for importee in item.names: if (os.path.isfile(os.path.join(moduleDir, importee.name + ".py")) and importee.name not in moduleImports): moduleImports.append(importee.name) else: modulePkgs = item.module.split('.') modulePath = os.path.join(*modulePkgs) if (os.path.isfile(os.path.join(moduleDir, modulePath) + ".py") and modulePath not in moduleImports): moduleImports.append(modulePath) for importee in item.names: _importeePfxName = os.path.join(modulePath, importee.name) if (os.path.isfile(os.path.join(moduleDir, _importeePfxName) + ".py") and _importeePfxName not in moduleImports): moduleImports.append(_importeePfxName) if PLUGIN_TRACE_FILE: with open(PLUGIN_TRACE_FILE, "at", encoding='utf-8') as fh: fh.write("Successful module plug-in info: " + moduleFilename + '\n') except Exception as err: _msg = _("Exception obtaining plug-in module info: {error}\n{traceback}").format( error=err, traceback=traceback.format_tb(sys.exc_info()[2])) if PLUGIN_TRACE_FILE: with open(PLUGIN_TRACE_FILE, "at", encoding='utf-8') as fh: fh.write(_msg + '\n') else: print(_msg, file=sys.stderr) if f: f.close() return None
def moduleModuleInfo(moduleURL, reload=False, parentImportsSubtree=False): #TODO several directories, eg User Application Data moduleFilename = _cntlr.webCache.getfilename(moduleURL, reload=reload, normalize=True, base=_pluginBase) if moduleFilename: f = None try: # if moduleFilename is a directory containing an __ini__.py file, open that instead if os.path.isdir(moduleFilename): if os.path.isfile(os.path.join(moduleFilename, "__init__.py")): moduleFilename = os.path.join(moduleFilename, "__init__.py") else: # impossible to get a moduleinfo from a directory without an __init__.py return None elif not moduleFilename.endswith(".py") and not os.path.exists( moduleFilename) and os.path.exists(moduleFilename + ".py"): moduleFilename += ".py" # extension module without .py suffix moduleDir, moduleName = os.path.split(moduleFilename) if PLUGIN_TRACE_FILE: with open(PLUGIN_TRACE_FILE, "at", encoding='utf-8') as fh: fh.write("Scanning module for plug-in info: {}\n".format( moduleFilename)) f = openFileStream(_cntlr, moduleFilename) tree = ast.parse(f.read(), filename=moduleFilename) constantStrings = {} functionDefNames = set() moduleImports = [] for item in tree.body: if isinstance(item, ast.Assign): attr = item.targets[0].id if attr == "__pluginInfo__": f.close() moduleInfo = {"name": None} classMethods = [] importURLs = [] for i, key in enumerate(item.value.keys): _key = key.s _value = item.value.values[i] _valueType = _value.__class__.__name__ if _key == "import": if _valueType == 'Str': importURLs.append(_value.s) elif _valueType in ("List", "Tuple"): for elt in _value.elts: importURLs.append(elt.s) elif _valueType == 'Str': moduleInfo[_key] = _value.s elif _valueType == 'Name': if _value.id in constantStrings: moduleInfo[_key] = constantStrings[ _value.id] elif _value.id in functionDefNames: classMethods.append(_key) elif _key == "imports" and _valueType in ("List", "Tuple"): importURLs = [elt.s for elt in _value.elts] moduleInfo['classMethods'] = classMethods moduleInfo["moduleURL"] = moduleURL moduleInfo["status"] = 'enabled' moduleInfo["fileDate"] = time.strftime( '%Y-%m-%dT%H:%M:%S UTC', time.gmtime(os.path.getmtime(moduleFilename))) mergedImportURLs = [] _moduleImportsSubtree = False for _url in importURLs: if _url.startswith("module_import"): for moduleImport in moduleImports: mergedImportURLs.append(moduleImport + ".py") if _url == "module_import_subtree": _moduleImportsSubtree = True elif _url == "module_subtree": for _dir in os.listdir(moduleDir): _subtreeModule = os.path.join( moduleDir, _dir) if os.path.isdir( _subtreeModule ) and _dir != "__pycache__": mergedImportURLs.append(_subtreeModule) else: mergedImportURLs.append(_url) if parentImportsSubtree and not _moduleImportsSubtree: _moduleImportsSubtree = True for moduleImport in moduleImports: mergedImportURLs.append(moduleImport + ".py") imports = [] for _url in mergedImportURLs: if isAbsolute(_url) or os.path.isabs(_url): _importURL = _url # URL is absolute http or local file system else: # check if exists relative to this module's directory _importURL = os.path.join( os.path.dirname(moduleURL), os.path.normpath(_url)) if not os.path.exists( _importURL ): # not relative to this plugin, assume standard plugin base _importURL = _url # moduleModuleInfo adjusts relative URL to plugin base _importModuleInfo = moduleModuleInfo( _importURL, reload, _moduleImportsSubtree) if _importModuleInfo: _importModuleInfo["isImported"] = True imports.append(_importModuleInfo) moduleInfo["imports"] = imports return moduleInfo elif isinstance( item.value, ast.Str ): # possible constant used in plugininfo, such as VERSION for assignmentName in item.targets: constantStrings[assignmentName.id] = item.value.s elif isinstance(item, ast.ImportFrom): if item.level == 1: # starts with . if item.module is None: # from . import module1, module2, ... for importee in item.names: if importee.name == '*': #import all submodules for _file in os.listdir(moduleDir): if _file != moduleFile and os.path.isfile( _file) and _file.endswith( ".py"): moduleImports.append(_file) elif (os.path.isfile( os.path.join(moduleDir, importee.name + ".py")) and importee.name not in moduleImports): moduleImports.append(importee.name) else: modulePkgs = item.module.split('.') modulePath = os.path.join(*modulePkgs) if (os.path.isfile( os.path.join(moduleDir, modulePath) + ".py") and modulePath not in moduleImports): moduleImports.append(modulePath) for importee in item.names: _importeePfxName = os.path.join( modulePath, importee.name) if (os.path.isfile( os.path.join(moduleDir, _importeePfxName) + ".py") and _importeePfxName not in moduleImports): moduleImports.append(_importeePfxName) elif isinstance(item, ast.FunctionDef ): # possible functionDef used in plugininfo functionDefNames.add(item.name) if PLUGIN_TRACE_FILE: with open(PLUGIN_TRACE_FILE, "at", encoding='utf-8') as fh: fh.write("Successful module plug-in info: " + moduleFilename + '\n') except Exception as err: _msg = _( "Exception obtaining plug-in module info: {moduleFilename}\n{error}\n{traceback}" ).format(error=err, moduleFilename=moduleFilename, traceback=traceback.format_tb(sys.exc_info()[2])) if PLUGIN_TRACE_FILE: with open(PLUGIN_TRACE_FILE, "at", encoding='utf-8') as fh: fh.write(_msg + '\n') else: print(_msg, file=sys.stderr) if f: f.close() return None
def loadUgtRelQnames(modelXbrl, dqcRules): if not dqcRules: return {} # not a us-gaap filing abbrNs = "" for modelDocument in modelXbrl.urlDocs.values(): abbrNs = abbreviatedNamespace(modelDocument.targetNamespace) if abbrNs and abbrNs.startswith("us-gaap/"): break if not abbrNs: # no gaap/ifrs taxonomy for this filing return {} _ugtRelsFileName = resourcesFilePath( modelXbrl.modelManager, "us-gaap-rels-{}.json".format(abbrNs.rpartition("/")[2])) if not os.path.exists(_ugtRelsFileName): buildUgtFullRelsFiles(modelXbrl, dqcRules) if not os.path.exists(_ugtRelsFileName): return {} _file = openFileStream(modelXbrl.modelManager.cntlr, _ugtRelsFileName, 'rt', encoding='utf-8') ugtRels = json.load(_file) # {localName: date, ...} _file.close() def qn(nsPrefix, localName): return qname(nsPrefix + ":" + localName, modelXbrl.prefixedNamespaces) ugtCalcsByQnames = defaultdict( dict ) # store as concept indices to avoid using memory for repetitive strings for wgt, fromNSes in ugtRels["calcs"].items(): calcWgtObj = ugtCalcsByQnames.setdefault( float(wgt), {}) # json weight object needs to be float for fromNs, fromObjs in fromNSes.items(): for fromName, toNSes in fromObjs.items(): fromConcept = modelXbrl.qnameConcepts.get(qn(fromNs, fromName)) if fromConcept is not None: calcFromObj = calcWgtObj.setdefault( fromConcept.qname, set()) for toNs, toNames in toNSes.items(): for toName in toNames: toConcept = modelXbrl.qnameConcepts.get( qn(toNs, toName)) if toConcept is not None: calcFromObj.add(toConcept.qname) ugtAxesByQnames = defaultdict( set ) # store as concept indices to avoid using memory for repetitive strings for axisName, memNames in ugtRels["axes"].items(): for axisConcept in modelXbrl.nameConcepts.get(axisName, ()): axisObj = ugtAxesByQnames[axisConcept.name] for memName in memNames: for memConcept in modelXbrl.nameConcepts.get(memName, ()): axisObj.add(memConcept.qname) ugt = {"calcs": ugtCalcsByQnames, "axes": ugtAxesByQnames} # dqc0015 if "DQC.US.0015" in ugtRels: dqc0015 = ugtRels["DQC.US.0015"] concepts = set() excludedMembers = set() excludedMemberStrings = set() excludedAxesMembers = defaultdict(set) conceptRuleIDs = {} for exName, exSet, isQName in (("conceptNames", concepts, True), ("excludedMemberNames", excludedMembers, True), ("excludedMemberStrings", excludedMemberStrings, False)): for ns, names in dqc0015[exName].items(): for localName in names: exSet.add(qn(ns, localName) if isQName else localName) for localDimNs, localDimMems in dqc0015["excludedAxesMembers"].items(): for localDimName, localMemObjs in localDimMems.items(): for localMemNs, localMemNames in localMemObjs.items(): if localMemNs == "*": excludedAxesMembers[qn(localDimNs, localDimName)].add("*") else: for localMemName in localMemNames: excludedAxesMembers[qn( localDimNs, localDimName)].add( qn(localMemNs, localMemName ) if localMemName != "*" else "*") #if abbrNs < "us-gaap/2021": # no rel ids in us-gaap/2020 # _ugtRelsFileName = resourcesFilePath(modelXbrl.modelManager, "us-gaap-rels-2021.json") # _file = openFileStream(modelXbrl.modelManager.cntlr, _ugtRelsFileName, 'rt', encoding='utf-8') # ugtRels = json.load(_file) # {localName: date, ...} # _file.close() for conceptNs, conceptNameIDs in ugtRels["DQC.US.0015"][ "conceptRuleIDs"].items(): for conceptName, conceptID in conceptNameIDs.items(): conceptRuleIDs[qn(conceptNs, conceptName)] = conceptID ugt["DQC.US.0015"] = attrdict( concepts=concepts, excludedAxesMembers=excludedAxesMembers, excludedMembers=excludedMembers, excludedMemberNamesPattern=re.compile( "|".join(excludedMemberStrings), re.IGNORECASE) if excludedMemberStrings else None, conceptRuleIDs=conceptRuleIDs) return ugt