Example #1
0
def deprecatedConceptDatesFile(modelManager, abbrNs, latestTaxonomyDoc):
    cntlr = modelManager.cntlr
    _fileName = resourcesFilePath(
        modelManager,
        abbrNs.partition("/")[0] + "-deprecated-concepts.json")
    _deprecatedLabelRole = latestTaxonomyDoc["deprecatedLabelRole"]
    _deprecatedDateMatchPattern = latestTaxonomyDoc["deprecationDatePattern"]
    if os.path.exists(_fileName):
        return _fileName
    # load labels and store file name
    modelManager.addToLog(_("loading {} deprecated concepts into {}").format(
        abbrNs, _fileName),
                          messageCode="info")
    deprecatedConceptDates = {}
    from arelle import ModelXbrl
    for latestTaxonomyLabelFile in flattenSequence(
            latestTaxonomyDoc["deprecatedLabels"]):
        # load without SEC/EFM validation (doc file would not be acceptable)
        priorValidateDisclosureSystem = modelManager.validateDisclosureSystem
        modelManager.validateDisclosureSystem = False
        deprecationsInstance = ModelXbrl.load(
            modelManager,
            # "http://xbrl.fasb.org/us-gaap/2012/elts/us-gaap-doc-2012-01-31.xml",
            # load from zip (especially after caching) is incredibly faster
            openFileSource(latestTaxonomyLabelFile, cntlr),
            _("built deprecations table in cache"))
        modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
        if deprecationsInstance is None:
            modelManager.addToLog(_("%(name)s documentation not loaded"),
                                  messageCode="arelle:notLoaded",
                                  messageArgs={
                                      "modelXbrl": val,
                                      "name": _abbrNs
                                  })
        else:
            # load deprecations
            for labelRel in deprecationsInstance.relationshipSet(
                    XbrlConst.conceptLabel).modelRelationships:
                modelLabel = labelRel.toModelObject
                conceptName = labelRel.fromModelObject.name
                if modelLabel.role == _deprecatedLabelRole:
                    match = _deprecatedDateMatchPattern.match(modelLabel.text)
                    if match is not None:
                        date = match.group(1)
                        if date:
                            deprecatedConceptDates[conceptName] = date

            jsonStr = _STR_UNICODE(
                json.dumps(
                    OrderedDict(
                        ((k, v)
                         for k, v in sorted(deprecatedConceptDates.items())
                         )),  # sort in json file
                    ensure_ascii=False,
                    indent=0))  # might not be unicode in 2.7
            saveFile(cntlr, _fileName, jsonStr)  # 2.7 gets unicode this way
            deprecationsInstance.close()
            del deprecationsInstance  # dereference closed modelXbrl
Example #2
0
def _make_cache(val, ugt, cntlr, ugt_default_dimensions_json_file):
    """
    Creates a new caches for the Taxonomy default dimensions

    :param val: ValidateXbrl to be validated
    :type val: :class: '~arelle.ValidateXbrl.ValidateXbrl'
    :param ugt: Taxonomy to check
    :type ugt: str
    :param ugt_default_dimensions_json_file: location to save json default
        dimensions
    :type ugt_default_dimensions_json_file: str
    :return: no explicit return, but saves caches for dqc_us_0041
    :rtype: None
    """
    started_at = time.time()
    ugt_entry_xsd = ugt["entryXsd"]
    val.usgaapDefaultDimensions = {}
    prior_validate_disclosure_system = (
        val.modelXbrl.modelManager.validateDisclosureSystem)
    val.modelXbrl.modelManager.validateDisclosureSystem = False
    ugt_entry_xsd_instance = (
        ModelXbrl.load(
            val.modelXbrl.modelManager,
            openFileSource(ugt_entry_xsd, cntlr),
            _("opened us-gaap entry xsd")  # noqa
        ))
    val.modelXbrl.modelManager.validateDisclosureSystem = (
        prior_validate_disclosure_system)

    if ugt_entry_xsd_instance is None:
        val.modelXbrl.error(
            "arelle:notLoaded",
            _("US-GAAP entry xsd not loaded: %(file)s"),  # noqa
            modelXbrl=val,
            file=os.path.basename(ugt_entry_xsd))

    else:
        model_relationships = (ugt_entry_xsd_instance.relationshipSet(
            XbrlConst.dimensionDefault).modelRelationships)
        for default_dim_rel in model_relationships:
            if _default_dim_rel_is_instance(default_dim_rel):
                from_name = default_dim_rel.fromModelObject.name
                to_name = default_dim_rel.toModelObject.name
                val.usgaapDefaultDimensions[from_name] = to_name
        json_str = str(
            json.dumps(val.usgaapDefaultDimensions,
                       ensure_ascii=False,
                       indent=0))  # might not be unicode in 2.7
        # 2.7 gets unicode this way
        saveFile(cntlr, ugt_default_dimensions_json_file, json_str)
        ugt_entry_xsd_instance.close()
        del ugt_entry_xsd_instance  # dereference closed modelXbrl
    val.modelXbrl.profileStat(
        _("build default dimensions cache"),  # noqa
        time.time() - started_at)
Example #3
0
def _create_config(val):
    """
    Creates the configs needed for dqc_us_0001

    :param val: ValidateXbrl needed in order to save the cache
    :type val: :class: '~arelle.ValidateXbrl.ValidateXbrl'
    :return: no explicit return but creates and saves configs in
        dqc_us_rule\resources\DQC_US_0001
    :rtype: None
    """
    val.ugtNamespace = None
    cntlr = val.modelXbrl.modelManager.cntlr
    year = _EARLIEST_US_GAAP_YEAR
    config = _load_config(_DQC_01_AXIS_FILE)
    # Create a list of axes in the base config file

    for ugt in _UGT_DOCS:
        # create taxonomy specific name
        config_json_file = os.path.join(os.path.dirname(__file__), 'resources',
                                        'DQC_US_0001',
                                        'dqc_0001_{}.json'.format(str(year)))
        # copy the base config file
        working_json_file = config
        ugtEntryXsd = ugt["entryXsd"]
        prior_vds = val.modelXbrl.modelManager.validateDisclosureSystem
        val.modelXbrl.modelManager.validateDisclosureSystem = False
        dimLoadingInstance = ModelXbrl.load(val.modelXbrl.modelManager,
                                            openFileSource(ugtEntryXsd, cntlr),
                                            ("built us-gaap member cache"))
        val.modelXbrl.modelManager.validateDisclosureSystem = prior_vds

        for axis, info in config.items():
            info['defined_members'] = defaultdict(set)
            axisConcept = dimLoadingInstance.nameConcepts.get(axis,
                                                              (None, ))[0]
            if axisConcept is not None:
                working_json_file[axis]['defined_members'] = sorted(
                    _tr_mem(val, ugt, dimLoadingInstance, axisConcept,
                            XbrlConst.dimensionDomain, None, set()))
        json_str = str(
            json.dumps(OrderedDict(sorted(working_json_file.items())),
                       ensure_ascii=False,
                       indent=4))
        saveFile(cntlr, config_json_file, json_str)
        dimLoadingInstance.close()
        del dimLoadingInstance
        year += 1
Example #4
0
def deprecatedConceptDatesFile(modelManager, abbrNs, latestTaxonomyDoc):
    if latestTaxonomyDoc is None:
        return None
    if not abbrNs: # none for an unexpected namespace pattern
        return None
    cntlr = modelManager.cntlr
    _fileName = resourcesFilePath(modelManager, abbrNs.partition("/")[0] + "-deprecated-concepts.json")
    _deprecatedLabelRole = latestTaxonomyDoc["deprecatedLabelRole"]
    _deprecatedDateMatchPattern = latestTaxonomyDoc["deprecationDatePattern"]
    if os.path.exists(_fileName):
        return _fileName
    # load labels and store file name
    modelManager.addToLog(_("loading {} deprecated concepts into {}").format(abbrNs, _fileName), messageCode="info")
    deprecatedConceptDates = {}
    # load without SEC/EFM validation (doc file would not be acceptable)
    priorValidateDisclosureSystem = modelManager.validateDisclosureSystem
    modelManager.validateDisclosureSystem = False
    from arelle import ModelXbrl
    deprecationsInstance = ModelXbrl.load(modelManager, 
          # "http://xbrl.fasb.org/us-gaap/2012/elts/us-gaap-doc-2012-01-31.xml",
          # load from zip (especially after caching) is incredibly faster
          openFileSource(latestTaxonomyDoc["deprecatedLabels"], cntlr), 
          _("built deprecations table in cache"))
    modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
    if deprecationsInstance is None:
        modelManager.addToLog(
            _("%(name)s documentation not loaded"),
            messageCode="arelle:notLoaded", messageArgs={"modelXbrl": val, "name":_abbrNs})
    else:   
        # load deprecations
        for labelRel in deprecationsInstance.relationshipSet(XbrlConst.conceptLabel).modelRelationships:
            modelLabel = labelRel.toModelObject
            conceptName = labelRel.fromModelObject.name
            if modelLabel.role == _deprecatedLabelRole:
                match = _deprecatedDateMatchPattern.match(modelLabel.text)
                if match is not None:
                    date = match.group(1)
                    if date:
                        deprecatedConceptDates[conceptName] = date
        jsonStr = _STR_UNICODE(json.dumps(deprecatedConceptDates, ensure_ascii=False, indent=0)) # might not be unicode in 2.7
        saveFile(cntlr, _fileName, jsonStr)  # 2.7 gets unicode this way
        deprecationsInstance.close()
        del deprecationsInstance # dereference closed modelXbrl
def _create_cache(val):
    """
    Creates the caches needed for dqc_us_0018

    :param val: ValidateXbrl needed in order to save the cache
    :type val: :class: '~arelle.ValidateXbrl.ValidateXbrl'
    :return: no explicit return but creates and saves a cache in
        dqc_us_rule\resources\DQC_US_0018
    :rtype: None
    """
    val.ugtNamespace = None
    cntlr = val.modelXbrl.modelManager.cntlr
    year = _EARLIEST_US_GAAP_YEAR

    for ugt in ugtDocs:
        deprecations_json_file = os.path.join(
            os.path.dirname(__file__),
            'resources',
            'DQC_US_0018',
            '{}_deprecated-concepts.json'.format(str(year))
        )

        if not os.path.isfile(deprecations_json_file):
            ugt_doc_lb = ugt["docLB"]
            val.usgaapDeprecations = {}
            disclosure_system = (
                val.modelXbrl.modelManager.validateDisclosureSystem
            )

            prior_validate_disclosure_system = disclosure_system
            val.modelXbrl.modelManager.validateDisclosureSystem = False
            deprecations_instance = ModelXbrl.load(
                val.modelXbrl.modelManager,
                openFileSource(ugt_doc_lb, cntlr),
                _("built deprecations table in cache")  # noqa
            )
            val.modelXbrl.modelManager.validateDisclosureSystem = (
                prior_validate_disclosure_system
            )

            if deprecations_instance is not None:
                dep_label = 'http://www.xbrl.org/2009/role/deprecatedLabel'
                dep_date_label = (
                    'http://www.xbrl.org/2009/role/deprecatedDateLabel'
                )
                concept_label = XbrlConst.conceptLabel
                relationship_set = (
                    deprecations_instance.relationshipSet(concept_label)
                )
                model_relationships = relationship_set.modelRelationships

                for labelRel in model_relationships:
                    model_documentation = labelRel.toModelObject
                    concept = labelRel.fromModelObject.name

                    if model_documentation.role == dep_label:
                        val.usgaapDeprecations[concept] = (
                            model_documentation.text,
                            val.usgaapDeprecations.get(concept, ('', ''))[0]
                        )
                    elif model_documentation.role == dep_date_label:
                        val.usgaapDeprecations[concept] = (
                            model_documentation.text,
                            val.usgaapDeprecations.get(concept, ('', ''))[1]
                        )
                json_str = str(
                    json.dumps(
                        val.usgaapDeprecations,
                        ensure_ascii=False, indent=0
                    )
                )  # might not be unicode in 2.7
                saveFile(cntlr, deprecations_json_file, json_str)
                deprecations_instance.close()
                del deprecations_instance  # dereference closed modelXbrl
        year += 1
def setup(val):
    val.linroleDefinitionIsDisclosure = re.compile(r"-\s+Disclosure\s+-\s",
                                                   re.IGNORECASE)
    val.linkroleDefinitionStatementSheet = re.compile(r"[^-]+-\s+Statement\s+-\s+.*", # no restriction to type of statement
                                                      re.IGNORECASE)
    val.ugtNamespace = None
    cntlr = val.modelXbrl.modelManager.cntlr
    # load deprecated concepts for filed year of us-gaap
    for ugt in ugtDocs:
        ugtNamespace = ugt["namespace"]
        if ugtNamespace in val.modelXbrl.namespaceDocs and len(val.modelXbrl.namespaceDocs[ugtNamespace]) > 0:
            val.ugtNamespace = ugtNamespace
            usgaapDoc = val.modelXbrl.namespaceDocs[ugtNamespace][0]
            deprecationsJsonFile = usgaapDoc.filepathdir + os.sep + "deprecated-concepts.json"
            file = None
            try:
                file = openFileStream(cntlr, deprecationsJsonFile, 'rt', encoding='utf-8')
                val.usgaapDeprecations = json.load(file)
                file.close()
            except Exception:
                if file:
                    file.close()
                val.modelXbrl.modelManager.addToLog(_("loading us-gaap {0} deprecated concepts into cache").format(ugt["year"]))
                startedAt = time.time()
                ugtDocLB = ugt["docLB"]
                val.usgaapDeprecations = {}
                # load without SEC/EFM validation (doc file would not be acceptable)
                priorValidateDisclosureSystem = val.modelXbrl.modelManager.validateDisclosureSystem
                val.modelXbrl.modelManager.validateDisclosureSystem = False
                deprecationsInstance = ModelXbrl.load(val.modelXbrl.modelManager, 
                      # "http://xbrl.fasb.org/us-gaap/2012/elts/us-gaap-doc-2012-01-31.xml",
                      # load from zip (especially after caching) is incredibly faster
                      openFileSource(ugtDocLB, cntlr), 
                      _("built deprecations table in cache"))
                val.modelXbrl.modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
                if deprecationsInstance is None:
                    val.modelXbrl.error("arelle:notLoaded",
                        _("US-GAAP documentation not loaded: %(file)s"),
                        modelXbrl=val, file=os.path.basename(ugtDocLB))
                else:   
                    # load deprecations
                    for labelRel in deprecationsInstance.relationshipSet(XbrlConst.conceptLabel).modelRelationships:
                        modelDocumentation = labelRel.toModelObject
                        conceptName = labelRel.fromModelObject.name
                        if modelDocumentation.role == 'http://www.xbrl.org/2009/role/deprecatedLabel':
                            val.usgaapDeprecations[conceptName] = (val.usgaapDeprecations.get(conceptName, ('',''))[0], modelDocumentation.text)
                        elif modelDocumentation.role == 'http://www.xbrl.org/2009/role/deprecatedDateLabel':
                            val.usgaapDeprecations[conceptName] = (modelDocumentation.text, val.usgaapDeprecations.get(conceptName, ('',''))[1])
                    jsonStr = _STR_UNICODE(json.dumps(val.usgaapDeprecations, ensure_ascii=False, indent=0)) # might not be unicode in 2.7
                    saveFile(cntlr, deprecationsJsonFile, jsonStr)  # 2.7 gets unicode this way
                    deprecationsInstance.close()
                    del deprecationsInstance # dereference closed modelXbrl
                val.modelXbrl.profileStat(_("build us-gaap deprecated concepts cache"), time.time() - startedAt)
            ugtCalcsJsonFile = usgaapDoc.filepathdir + os.sep + "ugt-calculations.json"
            ugtDefaultDimensionsJsonFile = usgaapDoc.filepathdir + os.sep + "ugt-default-dimensions.json"
            file = None
            try:
                file = openFileStream(cntlr, ugtCalcsJsonFile, 'rt', encoding='utf-8')
                val.usgaapCalculations = json.load(file)
                file.close()
                file = openFileStream(cntlr, ugtDefaultDimensionsJsonFile, 'rt', encoding='utf-8')
                val.usgaapDefaultDimensions = json.load(file)
                file.close()
            except Exception:
                if file:
                    file.close()
                val.modelXbrl.modelManager.addToLog(_("loading us-gaap {0} calculations and default dimensions into cache").format(ugt["year"]))
                startedAt = time.time()
                ugtEntryXsd = ugt["entryXsd"]
                val.usgaapCalculations = {}
                val.usgaapDefaultDimensions = {}
                # load without SEC/EFM validation (doc file would not be acceptable)
                priorValidateDisclosureSystem = val.modelXbrl.modelManager.validateDisclosureSystem
                val.modelXbrl.modelManager.validateDisclosureSystem = False
                calculationsInstance = ModelXbrl.load(val.modelXbrl.modelManager, 
                      # "http://xbrl.fasb.org/us-gaap/2012/entire/us-gaap-entryPoint-std-2012-01-31.xsd",
                      # load from zip (especially after caching) is incredibly faster
                      openFileSource(ugtEntryXsd, cntlr), 
                      _("built us-gaap calculations cache"))
                val.modelXbrl.modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
                if calculationsInstance is None:
                    val.modelXbrl.error("arelle:notLoaded",
                        _("US-GAAP calculations not loaded: %(file)s"),
                        modelXbrl=val, file=os.path.basename(ugtEntryXsd))
                else:   
                    # load calculations
                    for ELR in calculationsInstance.relationshipSet(XbrlConst.summationItem).linkRoleUris:
                        elrRelSet = calculationsInstance.relationshipSet(XbrlConst.summationItem, ELR)
                        definition = ""
                        for roleType in calculationsInstance.roleTypes.get(ELR,()):
                            definition = roleType.definition
                            break
                        isStatementSheet = bool(val.linkroleDefinitionStatementSheet.match(definition))
                        elrUgtCalcs = {"#roots": [c.name for c in elrRelSet.rootConcepts],
                                       "#definition": definition,
                                       "#isStatementSheet": isStatementSheet}
                        for relFrom, rels in elrRelSet.fromModelObjects().items():
                            elrUgtCalcs[relFrom.name] = [rel.toModelObject.name for rel in rels]
                        val.usgaapCalculations[ELR] = elrUgtCalcs
                    jsonStr = _STR_UNICODE(json.dumps(val.usgaapCalculations, ensure_ascii=False, indent=0)) # might not be unicode in 2.7
                    saveFile(cntlr, ugtCalcsJsonFile, jsonStr)  # 2.7 gets unicode this way
                    # load default dimensions
                    for defaultDimRel in calculationsInstance.relationshipSet(XbrlConst.dimensionDefault).modelRelationships:
                        if defaultDimRel.fromModelObject is not None and defaultDimRel.toModelObject is not None:
                            val.usgaapDefaultDimensions[defaultDimRel.fromModelObject.name] = defaultDimRel.toModelObject.name
                    jsonStr = _STR_UNICODE(json.dumps(val.usgaapDefaultDimensions, ensure_ascii=False, indent=0)) # might not be unicode in 2.7
                    saveFile(cntlr, ugtDefaultDimensionsJsonFile, jsonStr)  # 2.7 gets unicode this way
                    calculationsInstance.close()
                    del calculationsInstance # dereference closed modelXbrl
                val.modelXbrl.profileStat(_("build us-gaap calculations and default dimensions cache"), time.time() - startedAt)
            break
    val.deprecatedFactConcepts = defaultdict(list)
    val.deprecatedDimensions = defaultdict(list)
    val.deprecatedMembers = defaultdict(list)
Example #7
0
def _create_cache(val):
    """
    Creates the caches needed for dqc_us_0018

    :param val: ValidateXbrl needed in order to save the cache
    :type val: :class: '~arelle.ValidateXbrl.ValidateXbrl'
    :return: no explicit return but creates and saves a cache in
        dqc_us_rule\resources\DQC_US_0018
    :rtype: None
    """
    val.ugtNamespace = None
    cntlr = val.modelXbrl.modelManager.cntlr
    year = _EARLIEST_US_GAAP_YEAR

    for ugt in ugtDocs:
        deprecations_json_file = os.path.join(
            os.path.dirname(__file__),
            'resources',
            'DQC_US_0018',
            '{}_deprecated-concepts.json'.format(str(year))
        )

        if not os.path.isfile(deprecations_json_file):
            ugt_doc_lb = ugt["docLB"]
            val.usgaapDeprecations = {}
            disclosure_system = (
                val.modelXbrl.modelManager.validateDisclosureSystem
            )

            prior_validate_disclosure_system = disclosure_system
            val.modelXbrl.modelManager.validateDisclosureSystem = False
            deprecations_instance = ModelXbrl.load(
                val.modelXbrl.modelManager,
                openFileSource(ugt_doc_lb, cntlr),
                _("built deprecations table in cache")  # noqa
            )
            val.modelXbrl.modelManager.validateDisclosureSystem = (
                prior_validate_disclosure_system
            )

            if deprecations_instance is not None:
                dep_label = 'http://www.xbrl.org/2009/role/deprecatedLabel'
                dep_date_label = (
                    'http://www.xbrl.org/2009/role/deprecatedDateLabel'
                )
                concept_label = XbrlConst.conceptLabel
                relationship_set = (
                    deprecations_instance.relationshipSet(concept_label)
                )
                model_relationships = relationship_set.modelRelationships

                for labelRel in model_relationships:
                    model_documentation = labelRel.toModelObject
                    concept = labelRel.fromModelObject.name

                    if model_documentation.role == dep_label:
                        val.usgaapDeprecations[concept] = (
                            model_documentation.text,
                            val.usgaapDeprecations.get(concept, ('', ''))[0]
                        )
                    elif model_documentation.role == dep_date_label:
                        val.usgaapDeprecations[concept] = (
                            model_documentation.text,
                            val.usgaapDeprecations.get(concept, ('', ''))[1]
                        )
                json_str = str(
                    json.dumps(
                        val.usgaapDeprecations,
                        ensure_ascii=False, indent=0
                    )
                )  # might not be unicode in 2.7
                saveFile(cntlr, deprecations_json_file, json_str)
                deprecations_instance.close()
                del deprecations_instance  # dereference closed modelXbrl
        year += 1
Example #8
0
def buildUgtFullRelsFiles(modelXbrl, dqcRules):
    from arelle import ModelXbrl
    modelManager = modelXbrl.modelManager
    cntlr = modelXbrl.modelManager.cntlr
    conceptRule = (
        "http://fasb.org/dqcrules/arcrole/concept-rule",  # FASB arcrule
        "http://fasb.org/dqcrules/arcrole/rule-concept")
    rule0015 = "http://fasb.org/us-gaap/role/dqc/0015"
    # load without SEC/EFM validation (doc file would not be acceptable)
    priorValidateDisclosureSystem = modelManager.validateDisclosureSystem
    modelManager.validateDisclosureSystem = False
    for ugtAbbr, (ugtEntireUrl, dqcrtUrl) in latestEntireUgt.items():
        modelManager.addToLog(_("loading {} Entire UGT {}").format(
            ugtAbbr, ugtEntireUrl),
                              messageCode="info")
        ugtRels = {}
        ugtRels["calcs"] = ugtCalcs = {}
        ugtRels["axes"] = ugtAxes = defaultdict(set)
        ugtInstance = ModelXbrl.load(
            modelManager,
            # "http://xbrl.fasb.org/us-gaap/2012/elts/us-gaap-doc-2012-01-31.xml",
            # load from zip (especially after caching) is incredibly faster
            openFileSource(ugtEntireUrl, cntlr),
            _("built dqcrt table in cache"))
        if ugtInstance is None:
            modelManager.addToLog(_("%(name)s documentation not loaded"),
                                  messageCode="arelle:notLoaded",
                                  messageArgs={
                                      "modelXbrl": val,
                                      "name": ugtAbbr
                                  })
        else:
            # load signwarnings from DQC 0015
            calcRelSet = ugtInstance.relationshipSet(XbrlConst.summationItem)
            for rel in calcRelSet.modelRelationships:
                _fromQn = rel.fromModelObject.qname
                _toQn = rel.toModelObject.qname
                ugtCalcs.setdefault(rel.weight, {}).setdefault(
                    _fromQn.prefix,
                    {}).setdefault(_fromQn.localName,
                                   {}).setdefault(_toQn.prefix,
                                                  set()).add(_toQn.localName)
            for w in ugtCalcs.values():
                for fNs in w.values():
                    for fLn in fNs.values():
                        for tNs in fLn.keys():
                            fLn[tNs] = sorted(
                                fLn[tNs])  # change set to array for json
            dimDomRelSet = ugtInstance.relationshipSet(
                XbrlConst.dimensionDomain)
            axesOfInterest = set()
            for rule in dqcRules["DQC.US.0001"]["rules"].values():
                axesOfInterest.add(rule["axis"])
                for ruleAxesEntry in ("additional-axes", "unallowed-axes"):
                    for additionalAxis in rule.get(ruleAxesEntry, ()):
                        axesOfInterest.add(additionalAxis)
            for rel in dimDomRelSet.modelRelationships:
                axisConcept = rel.fromModelObject
                if axisConcept.name in axesOfInterest:
                    addDomMems(rel, ugtAxes[axisConcept.name], True)
            for axis in tuple(ugtAxes.keys()):
                ugtAxes[axis] = sorted(
                    ugtAxes[axis])  # change set to array for json
            ugtInstance.close()
            del ugtInstance  # dereference closed modelXbrl

            if dqcrtUrl:  # none for 2019
                modelManager.addToLog(_("loading {} DQC Rules {}").format(
                    ugtAbbr, dqcrtUrl),
                                      messageCode="info")
                dqcrtInstance = ModelXbrl.load(
                    modelManager,
                    # "http://xbrl.fasb.org/us-gaap/2012/elts/us-gaap-doc-2012-01-31.xml",
                    # load from zip (especially after caching) is incredibly faster
                    openFileSource(dqcrtUrl, cntlr),
                    _("built dqcrt table in cache"))
                if dqcrtInstance is None:
                    modelManager.addToLog(
                        _("%(name)s documentation not loaded"),
                        messageCode="arelle:notLoaded",
                        messageArgs={
                            "modelXbrl": val,
                            "name": ugtAbbr
                        })
                else:
                    ugtRels["DQC.US.0015"] = dqc0015 = defaultdict(dict)
                    # load DQC 0015
                    dqcRelSet = dqcrtInstance.relationshipSet(
                        (
                            "http://fasb.org/dqcrules/arcrole/concept-rule",  # FASB arcrule
                            "http://fasb.org/dqcrules/arcrole/rule-concept"),
                        "http://fasb.org/us-gaap/role/dqc/0015")
                    for dqc0015obj, headEltName in (
                        ("conceptNames", "Dqc_0015_ListOfElements"),
                        ("excludedMemberNames",
                         "Dqc_0015_ExcludeNonNegMembersAbstract"),
                        ("excludedAxesMembers",
                         "Dqc_0015_ExcludeNonNegAxisAbstract"),
                        ("excludedAxesMembers",
                         "Dqc_0015_ExcludeNonNegAxisMembersAbstract"),
                        ("excludedMemberStrings",
                         "Dqc_0015_ExcludeNonNegMemberStringsAbstract")):
                        headElts = dqcrtInstance.nameConcepts.get(
                            headEltName, ())
                        for headElt in headElts:
                            if dqc0015obj == "excludedMemberStrings":
                                for refRel in dqcrtInstance.relationshipSet(
                                        XbrlConst.conceptReference
                                ).fromModelObject(headElt):
                                    for refPart in refRel.toModelObject.iterchildren(
                                            "{*}allowableSubString"):
                                        for subStr in refPart.text.split():
                                            dqc0015[dqc0015obj].setdefault(
                                                "*", []).append(
                                                    subStr
                                                )  # applies to any namespace
                            else:
                                for ruleRel in dqcRelSet.fromModelObject(
                                        headElt):
                                    elt = ruleRel.toModelObject
                                    if dqc0015obj in ("conceptNames",
                                                      "excludedMemberNames"):
                                        dqc0015[dqc0015obj].setdefault(
                                            elt.qname.prefix,
                                            []).append(elt.name)
                                    else:
                                        l = dqc0015[dqc0015obj].setdefault(
                                            elt.qname.prefix,
                                            {}).setdefault(elt.name, {})
                                        if headEltName == "Dqc_0015_ExcludeNonNegAxisAbstract":
                                            l["*"] = None
                                        else:
                                            for memRel in dqcRelSet.fromModelObject(
                                                    elt):
                                                l.setdefault(
                                                    memRel.toModelObject.qname.
                                                    prefix, []).append(
                                                        memRel.toModelObject.
                                                        name)
                    dqc0015["conceptRuleIDs"] = conceptRuleIDs = {}
                    for rel in dqcrtInstance.relationshipSet(
                            XbrlConst.conceptReference).modelRelationships:
                        if rel.toModelObject.role == "http://fasb.org/us-gaap/role/dqc/ruleID":
                            conceptRuleIDs.setdefault(
                                elt.qname.prefix,
                                {})[rel.fromModelObject.name] = int(
                                    rel.toModelObject.stringValue.rpartition(
                                        ".")[2])

                dqcrtInstance.close()
                del dqcrtInstance  # dereference closed modelXbrl

                def sortDqcLists(obj):
                    if isinstance(obj, list):
                        obj.sort()
                    elif isinstance(obj, dict):
                        for objVal in obj.values():
                            sortDqcLists(objVal)

                sortDqcLists(dqc0015)
            jsonStr = _STR_UNICODE(
                json.dumps(ugtRels, ensure_ascii=False,
                           indent=2))  # might not be unicode in 2.7
            _ugtRelsFileName = resourcesFilePath(
                modelManager,
                "us-gaap-rels-{}.json".format(ugtAbbr.rpartition("/")[2]))
            saveFile(cntlr, _ugtRelsFileName,
                     jsonStr)  # 2.7 gets unicode this way

    modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
Example #9
0
def loadDqc0015signwarningRules(modelXbrl):
    conceptRule = "http://fasb.org/dqcrules/arcrole/concept-rule" # FASB arcrule
    rule0015 = "http://fasb.org/us-gaap/role/dqc/0015"
    modelManager = modelXbrl.modelManager
    cntlr = modelXbrl.modelManager.cntlr
    # check for cached completed signwarnings
    _signwarningsFileName = resourcesFilePath(modelManager, "signwarnings.json")
    if os.path.exists(_signwarningsFileName): 
        _file = openFileStream(modelManager.cntlr, _signwarningsFileName, 'rt', encoding='utf-8')
        signwarnings = json.load(_file) # {localName: date, ...}
        _file.close()
        return signwarnings
    # load template rules
    _fileName = resourcesFilePath(modelManager, "signwarnings-template.json")
    if _fileName:
        _file = openFileStream(modelXbrl.modelManager.cntlr, _fileName, 'rt', encoding='utf-8')
        signwarnings = json.load(_file, object_pairs_hook=OrderedDict) # {localName: date, ...}
        _file.close()

    # load rules and add to signwarnings template
    for dqcAbbr, dqcrtUrl in latestDqcrtDocs.items():
        modelManager.addToLog(_("loading {} DQC Rules {}").format(dqcAbbr, dqcrtUrl), messageCode="info")
        # load without SEC/EFM validation (doc file would not be acceptable)
        priorValidateDisclosureSystem = modelManager.validateDisclosureSystem
        modelManager.validateDisclosureSystem = False
        from arelle import ModelXbrl
        dqcrtInstance = ModelXbrl.load(modelManager, 
              # "http://xbrl.fasb.org/us-gaap/2012/elts/us-gaap-doc-2012-01-31.xml",
              # load from zip (especially after caching) is incredibly faster
              openFileSource(dqcrtUrl, cntlr), 
              _("built dqcrt table in cache"))
        modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
        if dqcrtInstance is None:
            modelManager.addToLog(
                _("%(name)s documentation not loaded"),
                messageCode="arelle:notLoaded", messageArgs={"modelXbrl": val, "name":dqcAbbr})
        else:   
            # load signwarnings from DQC 0015
            dqcRelSet = dqcrtInstance.relationshipSet(conceptRule, rule0015)
            for signWrnObj, headEltName in (("conceptNames", "Dqc_0015_ListOfElements"),
                                            ("excludedMemberNames", "Dqc_0015_ExcludeNonNegMembersAbstract"),
                                            ("excludedAxesMembers", "Dqc_0015_ExcludeNonNegAxisAbstract"),
                                            ("excludedAxesMembers", "Dqc_0015_ExcludeNonNegAxisMembersAbstract"),
                                            ("excludedMemberStrings", "Dqc_0015_ExcludeNonNegMemberStringsAbstract")):
                headElts = dqcrtInstance.nameConcepts.get(headEltName,())
                for headElt in headElts:
                    if signWrnObj == "excludedMemberStrings":
                        for refRel in dqcrtInstance.relationshipSet(XbrlConst.conceptReference).fromModelObject(headElt):
                            for refPart in refRel.toModelObject.iterchildren("{*}allowableSubString"):
                                for subStr in refPart.text.split():
                                    signwarnings[signWrnObj].setdefault(nsAbbr, []).append(subStr)
                    else:
                        for ruleRel in dqcRelSet.fromModelObject(headElt):
                            elt = ruleRel.toModelObject
                            nsAbbr = abbreviatedNamespace(elt.qname.namespaceURI)
                            if signWrnObj in ("conceptNames", "excludedMemberNames"):
                                signwarnings[signWrnObj].setdefault(nsAbbr, []).append(elt.name)
                            else:
                                l = signwarnings[signWrnObj].setdefault(nsAbbr, {}).setdefault(elt.name, [])
                                if headEltName == "Dqc_0015_ExcludeNonNegAxisAbstract":
                                    l.append("*")
                                else:
                                    for memRel in dqcRelSet.fromModelObject(elt):
                                        l.append(memRel.toModelObject.name)
            jsonStr = _STR_UNICODE(json.dumps(signwarnings, ensure_ascii=False, indent=2)) # might not be unicode in 2.7
            saveFile(cntlr, _signwarningsFileName, jsonStr)  # 2.7 gets unicode this way
            dqcrtInstance.close()
            del dqcrtInstance # dereference closed modelXbrl
    return signwarnings
def setup(val):
    if not val.validateLoggingSemantic:  # all checks herein are SEMANTIC
        return

    val.linroleDefinitionIsDisclosure = re.compile(r"-\s+Disclosure\s+-\s",
                                                   re.IGNORECASE)
    val.linkroleDefinitionStatementSheet = re.compile(r"[^-]+-\s+Statement\s+-\s+.*", # no restriction to type of statement
                                                      re.IGNORECASE)
    val.ugtNamespace = None
    cntlr = val.modelXbrl.modelManager.cntlr
    # load deprecated concepts for filed year of us-gaap
    for ugt in ugtDocs:
        ugtNamespace = ugt["namespace"]
        if ugtNamespace in val.modelXbrl.namespaceDocs and len(val.modelXbrl.namespaceDocs[ugtNamespace]) > 0:
            val.ugtNamespace = ugtNamespace
            usgaapDoc = val.modelXbrl.namespaceDocs[ugtNamespace][0]
            deprecationsJsonFile = usgaapDoc.filepathdir + os.sep + "deprecated-concepts.json"
            file = None
            try:
                file = openFileStream(cntlr, deprecationsJsonFile, 'rt', encoding='utf-8')
                val.usgaapDeprecations = json.load(file)
                file.close()
            except Exception:
                if file:
                    file.close()
                val.modelXbrl.modelManager.addToLog(_("loading us-gaap {0} deprecated concepts into cache").format(ugt["year"]))
                startedAt = time.time()
                ugtDocLB = ugt["docLB"]
                val.usgaapDeprecations = {}
                # load without SEC/EFM validation (doc file would not be acceptable)
                priorValidateDisclosureSystem = val.modelXbrl.modelManager.validateDisclosureSystem
                val.modelXbrl.modelManager.validateDisclosureSystem = False
                deprecationsInstance = ModelXbrl.load(val.modelXbrl.modelManager, 
                      # "http://xbrl.fasb.org/us-gaap/2012/elts/us-gaap-doc-2012-01-31.xml",
                      # load from zip (especially after caching) is incredibly faster
                      openFileSource(ugtDocLB, cntlr), 
                      _("built deprecations table in cache"))
                val.modelXbrl.modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
                if deprecationsInstance is None:
                    val.modelXbrl.error("arelle:notLoaded",
                        _("US-GAAP documentation not loaded: %(file)s"),
                        modelXbrl=val, file=os.path.basename(ugtDocLB))
                else:   
                    # load deprecations
                    for labelRel in deprecationsInstance.relationshipSet(XbrlConst.conceptLabel).modelRelationships:
                        modelDocumentation = labelRel.toModelObject
                        conceptName = labelRel.fromModelObject.name
                        if modelDocumentation.role == 'http://www.xbrl.org/2009/role/deprecatedLabel':
                            val.usgaapDeprecations[conceptName] = (val.usgaapDeprecations.get(conceptName, ('',''))[0], modelDocumentation.text)
                        elif modelDocumentation.role == 'http://www.xbrl.org/2009/role/deprecatedDateLabel':
                            val.usgaapDeprecations[conceptName] = (modelDocumentation.text, val.usgaapDeprecations.get(conceptName, ('',''))[1])
                    jsonStr = _STR_UNICODE(json.dumps(val.usgaapDeprecations, ensure_ascii=False, indent=0)) # might not be unicode in 2.7
                    saveFile(cntlr, deprecationsJsonFile, jsonStr)  # 2.7 gets unicode this way
                    deprecationsInstance.close()
                    del deprecationsInstance # dereference closed modelXbrl
                val.modelXbrl.profileStat(_("build us-gaap deprecated concepts cache"), time.time() - startedAt)
            ugtCalcsJsonFile = usgaapDoc.filepathdir + os.sep + "ugt-calculations.json"
            ugtDefaultDimensionsJsonFile = usgaapDoc.filepathdir + os.sep + "ugt-default-dimensions.json"
            file = None
            try:
                file = openFileStream(cntlr, ugtCalcsJsonFile, 'rt', encoding='utf-8')
                val.usgaapCalculations = json.load(file)
                file.close()
                file = openFileStream(cntlr, ugtDefaultDimensionsJsonFile, 'rt', encoding='utf-8')
                val.usgaapDefaultDimensions = json.load(file)
                file.close()
            except Exception:
                if file:
                    file.close()
                val.modelXbrl.modelManager.addToLog(_("loading us-gaap {0} calculations and default dimensions into cache").format(ugt["year"]))
                startedAt = time.time()
                ugtEntryXsd = ugt["entryXsd"]
                val.usgaapCalculations = {}
                val.usgaapDefaultDimensions = {}
                # load without SEC/EFM validation (doc file would not be acceptable)
                priorValidateDisclosureSystem = val.modelXbrl.modelManager.validateDisclosureSystem
                val.modelXbrl.modelManager.validateDisclosureSystem = False
                calculationsInstance = ModelXbrl.load(val.modelXbrl.modelManager, 
                      # "http://xbrl.fasb.org/us-gaap/2012/entire/us-gaap-entryPoint-std-2012-01-31.xsd",
                      # load from zip (especially after caching) is incredibly faster
                      openFileSource(ugtEntryXsd, cntlr), 
                      _("built us-gaap calculations cache"))
                val.modelXbrl.modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
                if calculationsInstance is None:
                    val.modelXbrl.error("arelle:notLoaded",
                        _("US-GAAP calculations not loaded: %(file)s"),
                        modelXbrl=val, file=os.path.basename(ugtEntryXsd))
                else:   
                    # load calculations
                    for ELR in calculationsInstance.relationshipSet(XbrlConst.summationItem).linkRoleUris:
                        elrRelSet = calculationsInstance.relationshipSet(XbrlConst.summationItem, ELR)
                        definition = ""
                        for roleType in calculationsInstance.roleTypes.get(ELR,()):
                            definition = roleType.definition
                            break
                        isStatementSheet = bool(val.linkroleDefinitionStatementSheet.match(definition))
                        elrUgtCalcs = {"#roots": [c.name for c in elrRelSet.rootConcepts],
                                       "#definition": definition,
                                       "#isStatementSheet": isStatementSheet}
                        for relFrom, rels in elrRelSet.fromModelObjects().items():
                            elrUgtCalcs[relFrom.name] = [rel.toModelObject.name for rel in rels]
                        val.usgaapCalculations[ELR] = elrUgtCalcs
                    jsonStr = _STR_UNICODE(json.dumps(val.usgaapCalculations, ensure_ascii=False, indent=0)) # might not be unicode in 2.7
                    saveFile(cntlr, ugtCalcsJsonFile, jsonStr)  # 2.7 gets unicode this way
                    # load default dimensions
                    for defaultDimRel in calculationsInstance.relationshipSet(XbrlConst.dimensionDefault).modelRelationships:
                        if defaultDimRel.fromModelObject is not None and defaultDimRel.toModelObject is not None:
                            val.usgaapDefaultDimensions[defaultDimRel.fromModelObject.name] = defaultDimRel.toModelObject.name
                    jsonStr = _STR_UNICODE(json.dumps(val.usgaapDefaultDimensions, ensure_ascii=False, indent=0)) # might not be unicode in 2.7
                    saveFile(cntlr, ugtDefaultDimensionsJsonFile, jsonStr)  # 2.7 gets unicode this way
                    calculationsInstance.close()
                    del calculationsInstance # dereference closed modelXbrl
                val.modelXbrl.profileStat(_("build us-gaap calculations and default dimensions cache"), time.time() - startedAt)
            break
    val.deprecatedFactConcepts = defaultdict(list)
    val.deprecatedDimensions = defaultdict(list)
    val.deprecatedMembers = defaultdict(list)
def _make_cache(val, ugt, cntlr, ugt_default_dimensions_json_file):
    """
    Creates a new caches for the Taxonomy default dimensions

    :param val: ValidateXbrl to be validated
    :type val: :class: '~arelle.ValidateXbrl.ValidateXbrl'
    :param ugt: Taxonomy to check
    :type ugt: str
    :param ugt_default_dimensions_json_file: location to save json default
        dimensions
    :type ugt_default_dimensions_json_file: str
    :return: no explicit return, but saves caches for dqc_us_0041
    :rtype: None
    """
    started_at = time.time()
    ugt_entry_xsd = ugt["entryXsd"]
    val.usgaapDefaultDimensions = {}
    prior_validate_disclosure_system = (
        val.modelXbrl.modelManager.validateDisclosureSystem
    )
    val.modelXbrl.modelManager.validateDisclosureSystem = False
    ugt_entry_xsd_instance = (
        ModelXbrl.load(
            val.modelXbrl.modelManager,
            openFileSource(ugt_entry_xsd, cntlr),
            _("opened us-gaap entry xsd")  # noqa
        )
    )
    val.modelXbrl.modelManager.validateDisclosureSystem = (
        prior_validate_disclosure_system
    )

    if ugt_entry_xsd_instance is None:
        val.modelXbrl.error(
            "arelle:notLoaded",
            _("US-GAAP entry xsd not loaded: %(file)s"),  # noqa
            modelXbrl=val,
            file=os.path.basename(ugt_entry_xsd)
        )

    else:
        model_relationships = (
            ugt_entry_xsd_instance.relationshipSet(
                XbrlConst.dimensionDefault
            ).modelRelationships
        )
        for default_dim_rel in model_relationships:
            if _default_dim_rel_is_instance(default_dim_rel):
                from_name = default_dim_rel.fromModelObject.name
                to_name = default_dim_rel.toModelObject.name
                val.usgaapDefaultDimensions[from_name] = to_name
        json_str = str(
            json.dumps(
                val.usgaapDefaultDimensions,
                ensure_ascii=False,
                indent=0
            )
        )  # might not be unicode in 2.7
        # 2.7 gets unicode this way
        saveFile(cntlr, ugt_default_dimensions_json_file, json_str)
        ugt_entry_xsd_instance.close()
        del ugt_entry_xsd_instance  # dereference closed modelXbrl
    val.modelXbrl.profileStat(
        _("build default dimensions cache"),  # noqa
        time.time() - started_at
    )