def deprecatedConceptDatesFile(modelManager, abbrNs, latestTaxonomyDoc):
cntlr = modelManager.cntlr
_fileName = resourcesFilePath(
modelManager,
abbrNs.partition("/")[0] + "-deprecated-concepts.json")
_deprecatedLabelRole = latestTaxonomyDoc["deprecatedLabelRole"]
_deprecatedDateMatchPattern = latestTaxonomyDoc["deprecationDatePattern"]
if os.path.exists(_fileName):
return _fileName
# load labels and store file name
modelManager.addToLog(_("loading {} deprecated concepts into {}").format(
abbrNs, _fileName),
messageCode="info")
deprecatedConceptDates = {}
from arelle import ModelXbrl
for latestTaxonomyLabelFile in flattenSequence(
latestTaxonomyDoc["deprecatedLabels"]):
# load without SEC/EFM validation (doc file would not be acceptable)
priorValidateDisclosureSystem = modelManager.validateDisclosureSystem
modelManager.validateDisclosureSystem = False
deprecationsInstance = ModelXbrl.load(
modelManager,
# "http://xbrl.fasb.org/us-gaap/2012/elts/us-gaap-doc-2012-01-31.xml",
# load from zip (especially after caching) is incredibly faster
openFileSource(latestTaxonomyLabelFile, cntlr),
_("built deprecations table in cache"))
modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
if deprecationsInstance is None:
modelManager.addToLog(_("%(name)s documentation not loaded"),
messageCode="arelle:notLoaded",
messageArgs={
"modelXbrl": val,
"name": _abbrNs
})
else:
# load deprecations
for labelRel in deprecationsInstance.relationshipSet(
XbrlConst.conceptLabel).modelRelationships:
modelLabel = labelRel.toModelObject
conceptName = labelRel.fromModelObject.name
if modelLabel.role == _deprecatedLabelRole:
match = _deprecatedDateMatchPattern.match(modelLabel.text)
if match is not None:
date = match.group(1)
if date:
deprecatedConceptDates[conceptName] = date
jsonStr = _STR_UNICODE(
json.dumps(
OrderedDict(
((k, v)
for k, v in sorted(deprecatedConceptDates.items())
)), # sort in json file
ensure_ascii=False,
indent=0)) # might not be unicode in 2.7
saveFile(cntlr, _fileName, jsonStr) # 2.7 gets unicode this way
deprecationsInstance.close()
del deprecationsInstance # dereference closed modelXbrl
def _make_cache(val, ugt, cntlr, ugt_default_dimensions_json_file):
"""
Creates a new caches for the Taxonomy default dimensions
:param val: ValidateXbrl to be validated
:type val: :class: '~arelle.ValidateXbrl.ValidateXbrl'
:param ugt: Taxonomy to check
:type ugt: str
:param ugt_default_dimensions_json_file: location to save json default
dimensions
:type ugt_default_dimensions_json_file: str
:return: no explicit return, but saves caches for dqc_us_0041
:rtype: None
"""
started_at = time.time()
ugt_entry_xsd = ugt["entryXsd"]
val.usgaapDefaultDimensions = {}
prior_validate_disclosure_system = (
val.modelXbrl.modelManager.validateDisclosureSystem)
val.modelXbrl.modelManager.validateDisclosureSystem = False
ugt_entry_xsd_instance = (
ModelXbrl.load(
val.modelXbrl.modelManager,
openFileSource(ugt_entry_xsd, cntlr),
_("opened us-gaap entry xsd") # noqa
))
val.modelXbrl.modelManager.validateDisclosureSystem = (
prior_validate_disclosure_system)
if ugt_entry_xsd_instance is None:
val.modelXbrl.error(
"arelle:notLoaded",
_("US-GAAP entry xsd not loaded: %(file)s"), # noqa
modelXbrl=val,
file=os.path.basename(ugt_entry_xsd))
else:
model_relationships = (ugt_entry_xsd_instance.relationshipSet(
XbrlConst.dimensionDefault).modelRelationships)
for default_dim_rel in model_relationships:
if _default_dim_rel_is_instance(default_dim_rel):
from_name = default_dim_rel.fromModelObject.name
to_name = default_dim_rel.toModelObject.name
val.usgaapDefaultDimensions[from_name] = to_name
json_str = str(
json.dumps(val.usgaapDefaultDimensions,
ensure_ascii=False,
indent=0)) # might not be unicode in 2.7
# 2.7 gets unicode this way
saveFile(cntlr, ugt_default_dimensions_json_file, json_str)
ugt_entry_xsd_instance.close()
del ugt_entry_xsd_instance # dereference closed modelXbrl
val.modelXbrl.profileStat(
_("build default dimensions cache"), # noqa
time.time() - started_at)
def _create_config(val):
"""
Creates the configs needed for dqc_us_0001
:param val: ValidateXbrl needed in order to save the cache
:type val: :class: '~arelle.ValidateXbrl.ValidateXbrl'
:return: no explicit return but creates and saves configs in
dqc_us_rule\resources\DQC_US_0001
:rtype: None
"""
val.ugtNamespace = None
cntlr = val.modelXbrl.modelManager.cntlr
year = _EARLIEST_US_GAAP_YEAR
config = _load_config(_DQC_01_AXIS_FILE)
# Create a list of axes in the base config file
for ugt in _UGT_DOCS:
# create taxonomy specific name
config_json_file = os.path.join(os.path.dirname(__file__), 'resources',
'DQC_US_0001',
'dqc_0001_{}.json'.format(str(year)))
# copy the base config file
working_json_file = config
ugtEntryXsd = ugt["entryXsd"]
prior_vds = val.modelXbrl.modelManager.validateDisclosureSystem
val.modelXbrl.modelManager.validateDisclosureSystem = False
dimLoadingInstance = ModelXbrl.load(val.modelXbrl.modelManager,
openFileSource(ugtEntryXsd, cntlr),
("built us-gaap member cache"))
val.modelXbrl.modelManager.validateDisclosureSystem = prior_vds
for axis, info in config.items():
info['defined_members'] = defaultdict(set)
axisConcept = dimLoadingInstance.nameConcepts.get(axis,
(None, ))[0]
if axisConcept is not None:
working_json_file[axis]['defined_members'] = sorted(
_tr_mem(val, ugt, dimLoadingInstance, axisConcept,
XbrlConst.dimensionDomain, None, set()))
json_str = str(
json.dumps(OrderedDict(sorted(working_json_file.items())),
ensure_ascii=False,
indent=4))
saveFile(cntlr, config_json_file, json_str)
dimLoadingInstance.close()
del dimLoadingInstance
year += 1
def deprecatedConceptDatesFile(modelManager, abbrNs, latestTaxonomyDoc):
if latestTaxonomyDoc is None:
return None
if not abbrNs: # none for an unexpected namespace pattern
return None
cntlr = modelManager.cntlr
_fileName = resourcesFilePath(modelManager, abbrNs.partition("/")[0] + "-deprecated-concepts.json")
_deprecatedLabelRole = latestTaxonomyDoc["deprecatedLabelRole"]
_deprecatedDateMatchPattern = latestTaxonomyDoc["deprecationDatePattern"]
if os.path.exists(_fileName):
return _fileName
# load labels and store file name
modelManager.addToLog(_("loading {} deprecated concepts into {}").format(abbrNs, _fileName), messageCode="info")
deprecatedConceptDates = {}
# load without SEC/EFM validation (doc file would not be acceptable)
priorValidateDisclosureSystem = modelManager.validateDisclosureSystem
modelManager.validateDisclosureSystem = False
from arelle import ModelXbrl
deprecationsInstance = ModelXbrl.load(modelManager,
# "http://xbrl.fasb.org/us-gaap/2012/elts/us-gaap-doc-2012-01-31.xml",
# load from zip (especially after caching) is incredibly faster
openFileSource(latestTaxonomyDoc["deprecatedLabels"], cntlr),
_("built deprecations table in cache"))
modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
if deprecationsInstance is None:
modelManager.addToLog(
_("%(name)s documentation not loaded"),
messageCode="arelle:notLoaded", messageArgs={"modelXbrl": val, "name":_abbrNs})
else:
# load deprecations
for labelRel in deprecationsInstance.relationshipSet(XbrlConst.conceptLabel).modelRelationships:
modelLabel = labelRel.toModelObject
conceptName = labelRel.fromModelObject.name
if modelLabel.role == _deprecatedLabelRole:
match = _deprecatedDateMatchPattern.match(modelLabel.text)
if match is not None:
date = match.group(1)
if date:
deprecatedConceptDates[conceptName] = date
jsonStr = _STR_UNICODE(json.dumps(deprecatedConceptDates, ensure_ascii=False, indent=0)) # might not be unicode in 2.7
saveFile(cntlr, _fileName, jsonStr) # 2.7 gets unicode this way
deprecationsInstance.close()
del deprecationsInstance # dereference closed modelXbrl
def _create_cache(val):
"""
Creates the caches needed for dqc_us_0018
:param val: ValidateXbrl needed in order to save the cache
:type val: :class: '~arelle.ValidateXbrl.ValidateXbrl'
:return: no explicit return but creates and saves a cache in
dqc_us_rule\resources\DQC_US_0018
:rtype: None
"""
val.ugtNamespace = None
cntlr = val.modelXbrl.modelManager.cntlr
year = _EARLIEST_US_GAAP_YEAR
for ugt in ugtDocs:
deprecations_json_file = os.path.join(
os.path.dirname(__file__),
'resources',
'DQC_US_0018',
'{}_deprecated-concepts.json'.format(str(year))
)
if not os.path.isfile(deprecations_json_file):
ugt_doc_lb = ugt["docLB"]
val.usgaapDeprecations = {}
disclosure_system = (
val.modelXbrl.modelManager.validateDisclosureSystem
)
prior_validate_disclosure_system = disclosure_system
val.modelXbrl.modelManager.validateDisclosureSystem = False
deprecations_instance = ModelXbrl.load(
val.modelXbrl.modelManager,
openFileSource(ugt_doc_lb, cntlr),
_("built deprecations table in cache") # noqa
)
val.modelXbrl.modelManager.validateDisclosureSystem = (
prior_validate_disclosure_system
)
if deprecations_instance is not None:
dep_label = 'http://www.xbrl.org/2009/role/deprecatedLabel'
dep_date_label = (
'http://www.xbrl.org/2009/role/deprecatedDateLabel'
)
concept_label = XbrlConst.conceptLabel
relationship_set = (
deprecations_instance.relationshipSet(concept_label)
)
model_relationships = relationship_set.modelRelationships
for labelRel in model_relationships:
model_documentation = labelRel.toModelObject
concept = labelRel.fromModelObject.name
if model_documentation.role == dep_label:
val.usgaapDeprecations[concept] = (
model_documentation.text,
val.usgaapDeprecations.get(concept, ('', ''))[0]
)
elif model_documentation.role == dep_date_label:
val.usgaapDeprecations[concept] = (
model_documentation.text,
val.usgaapDeprecations.get(concept, ('', ''))[1]
)
json_str = str(
json.dumps(
val.usgaapDeprecations,
ensure_ascii=False, indent=0
)
) # might not be unicode in 2.7
saveFile(cntlr, deprecations_json_file, json_str)
deprecations_instance.close()
del deprecations_instance # dereference closed modelXbrl
year += 1
def setup(val):
val.linroleDefinitionIsDisclosure = re.compile(r"-\s+Disclosure\s+-\s",
re.IGNORECASE)
val.linkroleDefinitionStatementSheet = re.compile(r"[^-]+-\s+Statement\s+-\s+.*", # no restriction to type of statement
re.IGNORECASE)
val.ugtNamespace = None
cntlr = val.modelXbrl.modelManager.cntlr
# load deprecated concepts for filed year of us-gaap
for ugt in ugtDocs:
ugtNamespace = ugt["namespace"]
if ugtNamespace in val.modelXbrl.namespaceDocs and len(val.modelXbrl.namespaceDocs[ugtNamespace]) > 0:
val.ugtNamespace = ugtNamespace
usgaapDoc = val.modelXbrl.namespaceDocs[ugtNamespace][0]
deprecationsJsonFile = usgaapDoc.filepathdir + os.sep + "deprecated-concepts.json"
file = None
try:
file = openFileStream(cntlr, deprecationsJsonFile, 'rt', encoding='utf-8')
val.usgaapDeprecations = json.load(file)
file.close()
except Exception:
if file:
file.close()
val.modelXbrl.modelManager.addToLog(_("loading us-gaap {0} deprecated concepts into cache").format(ugt["year"]))
startedAt = time.time()
ugtDocLB = ugt["docLB"]
val.usgaapDeprecations = {}
# load without SEC/EFM validation (doc file would not be acceptable)
priorValidateDisclosureSystem = val.modelXbrl.modelManager.validateDisclosureSystem
val.modelXbrl.modelManager.validateDisclosureSystem = False
deprecationsInstance = ModelXbrl.load(val.modelXbrl.modelManager,
# "http://xbrl.fasb.org/us-gaap/2012/elts/us-gaap-doc-2012-01-31.xml",
# load from zip (especially after caching) is incredibly faster
openFileSource(ugtDocLB, cntlr),
_("built deprecations table in cache"))
val.modelXbrl.modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
if deprecationsInstance is None:
val.modelXbrl.error("arelle:notLoaded",
_("US-GAAP documentation not loaded: %(file)s"),
modelXbrl=val, file=os.path.basename(ugtDocLB))
else:
# load deprecations
for labelRel in deprecationsInstance.relationshipSet(XbrlConst.conceptLabel).modelRelationships:
modelDocumentation = labelRel.toModelObject
conceptName = labelRel.fromModelObject.name
if modelDocumentation.role == 'http://www.xbrl.org/2009/role/deprecatedLabel':
val.usgaapDeprecations[conceptName] = (val.usgaapDeprecations.get(conceptName, ('',''))[0], modelDocumentation.text)
elif modelDocumentation.role == 'http://www.xbrl.org/2009/role/deprecatedDateLabel':
val.usgaapDeprecations[conceptName] = (modelDocumentation.text, val.usgaapDeprecations.get(conceptName, ('',''))[1])
jsonStr = _STR_UNICODE(json.dumps(val.usgaapDeprecations, ensure_ascii=False, indent=0)) # might not be unicode in 2.7
saveFile(cntlr, deprecationsJsonFile, jsonStr) # 2.7 gets unicode this way
deprecationsInstance.close()
del deprecationsInstance # dereference closed modelXbrl
val.modelXbrl.profileStat(_("build us-gaap deprecated concepts cache"), time.time() - startedAt)
ugtCalcsJsonFile = usgaapDoc.filepathdir + os.sep + "ugt-calculations.json"
ugtDefaultDimensionsJsonFile = usgaapDoc.filepathdir + os.sep + "ugt-default-dimensions.json"
file = None
try:
file = openFileStream(cntlr, ugtCalcsJsonFile, 'rt', encoding='utf-8')
val.usgaapCalculations = json.load(file)
file.close()
file = openFileStream(cntlr, ugtDefaultDimensionsJsonFile, 'rt', encoding='utf-8')
val.usgaapDefaultDimensions = json.load(file)
file.close()
except Exception:
if file:
file.close()
val.modelXbrl.modelManager.addToLog(_("loading us-gaap {0} calculations and default dimensions into cache").format(ugt["year"]))
startedAt = time.time()
ugtEntryXsd = ugt["entryXsd"]
val.usgaapCalculations = {}
val.usgaapDefaultDimensions = {}
# load without SEC/EFM validation (doc file would not be acceptable)
priorValidateDisclosureSystem = val.modelXbrl.modelManager.validateDisclosureSystem
val.modelXbrl.modelManager.validateDisclosureSystem = False
calculationsInstance = ModelXbrl.load(val.modelXbrl.modelManager,
# "http://xbrl.fasb.org/us-gaap/2012/entire/us-gaap-entryPoint-std-2012-01-31.xsd",
# load from zip (especially after caching) is incredibly faster
openFileSource(ugtEntryXsd, cntlr),
_("built us-gaap calculations cache"))
val.modelXbrl.modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
if calculationsInstance is None:
val.modelXbrl.error("arelle:notLoaded",
_("US-GAAP calculations not loaded: %(file)s"),
modelXbrl=val, file=os.path.basename(ugtEntryXsd))
else:
# load calculations
for ELR in calculationsInstance.relationshipSet(XbrlConst.summationItem).linkRoleUris:
elrRelSet = calculationsInstance.relationshipSet(XbrlConst.summationItem, ELR)
definition = ""
for roleType in calculationsInstance.roleTypes.get(ELR,()):
definition = roleType.definition
break
isStatementSheet = bool(val.linkroleDefinitionStatementSheet.match(definition))
elrUgtCalcs = {"#roots": [c.name for c in elrRelSet.rootConcepts],
"#definition": definition,
"#isStatementSheet": isStatementSheet}
for relFrom, rels in elrRelSet.fromModelObjects().items():
elrUgtCalcs[relFrom.name] = [rel.toModelObject.name for rel in rels]
val.usgaapCalculations[ELR] = elrUgtCalcs
jsonStr = _STR_UNICODE(json.dumps(val.usgaapCalculations, ensure_ascii=False, indent=0)) # might not be unicode in 2.7
saveFile(cntlr, ugtCalcsJsonFile, jsonStr) # 2.7 gets unicode this way
# load default dimensions
for defaultDimRel in calculationsInstance.relationshipSet(XbrlConst.dimensionDefault).modelRelationships:
if defaultDimRel.fromModelObject is not None and defaultDimRel.toModelObject is not None:
val.usgaapDefaultDimensions[defaultDimRel.fromModelObject.name] = defaultDimRel.toModelObject.name
jsonStr = _STR_UNICODE(json.dumps(val.usgaapDefaultDimensions, ensure_ascii=False, indent=0)) # might not be unicode in 2.7
saveFile(cntlr, ugtDefaultDimensionsJsonFile, jsonStr) # 2.7 gets unicode this way
calculationsInstance.close()
del calculationsInstance # dereference closed modelXbrl
val.modelXbrl.profileStat(_("build us-gaap calculations and default dimensions cache"), time.time() - startedAt)
break
val.deprecatedFactConcepts = defaultdict(list)
val.deprecatedDimensions = defaultdict(list)
val.deprecatedMembers = defaultdict(list)
def _create_cache(val):
"""
Creates the caches needed for dqc_us_0018
:param val: ValidateXbrl needed in order to save the cache
:type val: :class: '~arelle.ValidateXbrl.ValidateXbrl'
:return: no explicit return but creates and saves a cache in
dqc_us_rule\resources\DQC_US_0018
:rtype: None
"""
val.ugtNamespace = None
cntlr = val.modelXbrl.modelManager.cntlr
year = _EARLIEST_US_GAAP_YEAR
for ugt in ugtDocs:
deprecations_json_file = os.path.join(
os.path.dirname(__file__),
'resources',
'DQC_US_0018',
'{}_deprecated-concepts.json'.format(str(year))
)
if not os.path.isfile(deprecations_json_file):
ugt_doc_lb = ugt["docLB"]
val.usgaapDeprecations = {}
disclosure_system = (
val.modelXbrl.modelManager.validateDisclosureSystem
)
prior_validate_disclosure_system = disclosure_system
val.modelXbrl.modelManager.validateDisclosureSystem = False
deprecations_instance = ModelXbrl.load(
val.modelXbrl.modelManager,
openFileSource(ugt_doc_lb, cntlr),
_("built deprecations table in cache") # noqa
)
val.modelXbrl.modelManager.validateDisclosureSystem = (
prior_validate_disclosure_system
)
if deprecations_instance is not None:
dep_label = 'http://www.xbrl.org/2009/role/deprecatedLabel'
dep_date_label = (
'http://www.xbrl.org/2009/role/deprecatedDateLabel'
)
concept_label = XbrlConst.conceptLabel
relationship_set = (
deprecations_instance.relationshipSet(concept_label)
)
model_relationships = relationship_set.modelRelationships
for labelRel in model_relationships:
model_documentation = labelRel.toModelObject
concept = labelRel.fromModelObject.name
if model_documentation.role == dep_label:
val.usgaapDeprecations[concept] = (
model_documentation.text,
val.usgaapDeprecations.get(concept, ('', ''))[0]
)
elif model_documentation.role == dep_date_label:
val.usgaapDeprecations[concept] = (
model_documentation.text,
val.usgaapDeprecations.get(concept, ('', ''))[1]
)
json_str = str(
json.dumps(
val.usgaapDeprecations,
ensure_ascii=False, indent=0
)
) # might not be unicode in 2.7
saveFile(cntlr, deprecations_json_file, json_str)
deprecations_instance.close()
del deprecations_instance # dereference closed modelXbrl
year += 1
def buildUgtFullRelsFiles(modelXbrl, dqcRules):
from arelle import ModelXbrl
modelManager = modelXbrl.modelManager
cntlr = modelXbrl.modelManager.cntlr
conceptRule = (
"http://fasb.org/dqcrules/arcrole/concept-rule", # FASB arcrule
"http://fasb.org/dqcrules/arcrole/rule-concept")
rule0015 = "http://fasb.org/us-gaap/role/dqc/0015"
# load without SEC/EFM validation (doc file would not be acceptable)
priorValidateDisclosureSystem = modelManager.validateDisclosureSystem
modelManager.validateDisclosureSystem = False
for ugtAbbr, (ugtEntireUrl, dqcrtUrl) in latestEntireUgt.items():
modelManager.addToLog(_("loading {} Entire UGT {}").format(
ugtAbbr, ugtEntireUrl),
messageCode="info")
ugtRels = {}
ugtRels["calcs"] = ugtCalcs = {}
ugtRels["axes"] = ugtAxes = defaultdict(set)
ugtInstance = ModelXbrl.load(
modelManager,
# "http://xbrl.fasb.org/us-gaap/2012/elts/us-gaap-doc-2012-01-31.xml",
# load from zip (especially after caching) is incredibly faster
openFileSource(ugtEntireUrl, cntlr),
_("built dqcrt table in cache"))
if ugtInstance is None:
modelManager.addToLog(_("%(name)s documentation not loaded"),
messageCode="arelle:notLoaded",
messageArgs={
"modelXbrl": val,
"name": ugtAbbr
})
else:
# load signwarnings from DQC 0015
calcRelSet = ugtInstance.relationshipSet(XbrlConst.summationItem)
for rel in calcRelSet.modelRelationships:
_fromQn = rel.fromModelObject.qname
_toQn = rel.toModelObject.qname
ugtCalcs.setdefault(rel.weight, {}).setdefault(
_fromQn.prefix,
{}).setdefault(_fromQn.localName,
{}).setdefault(_toQn.prefix,
set()).add(_toQn.localName)
for w in ugtCalcs.values():
for fNs in w.values():
for fLn in fNs.values():
for tNs in fLn.keys():
fLn[tNs] = sorted(
fLn[tNs]) # change set to array for json
dimDomRelSet = ugtInstance.relationshipSet(
XbrlConst.dimensionDomain)
axesOfInterest = set()
for rule in dqcRules["DQC.US.0001"]["rules"].values():
axesOfInterest.add(rule["axis"])
for ruleAxesEntry in ("additional-axes", "unallowed-axes"):
for additionalAxis in rule.get(ruleAxesEntry, ()):
axesOfInterest.add(additionalAxis)
for rel in dimDomRelSet.modelRelationships:
axisConcept = rel.fromModelObject
if axisConcept.name in axesOfInterest:
addDomMems(rel, ugtAxes[axisConcept.name], True)
for axis in tuple(ugtAxes.keys()):
ugtAxes[axis] = sorted(
ugtAxes[axis]) # change set to array for json
ugtInstance.close()
del ugtInstance # dereference closed modelXbrl
if dqcrtUrl: # none for 2019
modelManager.addToLog(_("loading {} DQC Rules {}").format(
ugtAbbr, dqcrtUrl),
messageCode="info")
dqcrtInstance = ModelXbrl.load(
modelManager,
# "http://xbrl.fasb.org/us-gaap/2012/elts/us-gaap-doc-2012-01-31.xml",
# load from zip (especially after caching) is incredibly faster
openFileSource(dqcrtUrl, cntlr),
_("built dqcrt table in cache"))
if dqcrtInstance is None:
modelManager.addToLog(
_("%(name)s documentation not loaded"),
messageCode="arelle:notLoaded",
messageArgs={
"modelXbrl": val,
"name": ugtAbbr
})
else:
ugtRels["DQC.US.0015"] = dqc0015 = defaultdict(dict)
# load DQC 0015
dqcRelSet = dqcrtInstance.relationshipSet(
(
"http://fasb.org/dqcrules/arcrole/concept-rule", # FASB arcrule
"http://fasb.org/dqcrules/arcrole/rule-concept"),
"http://fasb.org/us-gaap/role/dqc/0015")
for dqc0015obj, headEltName in (
("conceptNames", "Dqc_0015_ListOfElements"),
("excludedMemberNames",
"Dqc_0015_ExcludeNonNegMembersAbstract"),
("excludedAxesMembers",
"Dqc_0015_ExcludeNonNegAxisAbstract"),
("excludedAxesMembers",
"Dqc_0015_ExcludeNonNegAxisMembersAbstract"),
("excludedMemberStrings",
"Dqc_0015_ExcludeNonNegMemberStringsAbstract")):
headElts = dqcrtInstance.nameConcepts.get(
headEltName, ())
for headElt in headElts:
if dqc0015obj == "excludedMemberStrings":
for refRel in dqcrtInstance.relationshipSet(
XbrlConst.conceptReference
).fromModelObject(headElt):
for refPart in refRel.toModelObject.iterchildren(
"{*}allowableSubString"):
for subStr in refPart.text.split():
dqc0015[dqc0015obj].setdefault(
"*", []).append(
subStr
) # applies to any namespace
else:
for ruleRel in dqcRelSet.fromModelObject(
headElt):
elt = ruleRel.toModelObject
if dqc0015obj in ("conceptNames",
"excludedMemberNames"):
dqc0015[dqc0015obj].setdefault(
elt.qname.prefix,
[]).append(elt.name)
else:
l = dqc0015[dqc0015obj].setdefault(
elt.qname.prefix,
{}).setdefault(elt.name, {})
if headEltName == "Dqc_0015_ExcludeNonNegAxisAbstract":
l["*"] = None
else:
for memRel in dqcRelSet.fromModelObject(
elt):
l.setdefault(
memRel.toModelObject.qname.
prefix, []).append(
memRel.toModelObject.
name)
dqc0015["conceptRuleIDs"] = conceptRuleIDs = {}
for rel in dqcrtInstance.relationshipSet(
XbrlConst.conceptReference).modelRelationships:
if rel.toModelObject.role == "http://fasb.org/us-gaap/role/dqc/ruleID":
conceptRuleIDs.setdefault(
elt.qname.prefix,
{})[rel.fromModelObject.name] = int(
rel.toModelObject.stringValue.rpartition(
".")[2])
dqcrtInstance.close()
del dqcrtInstance # dereference closed modelXbrl
def sortDqcLists(obj):
if isinstance(obj, list):
obj.sort()
elif isinstance(obj, dict):
for objVal in obj.values():
sortDqcLists(objVal)
sortDqcLists(dqc0015)
jsonStr = _STR_UNICODE(
json.dumps(ugtRels, ensure_ascii=False,
indent=2)) # might not be unicode in 2.7
_ugtRelsFileName = resourcesFilePath(
modelManager,
"us-gaap-rels-{}.json".format(ugtAbbr.rpartition("/")[2]))
saveFile(cntlr, _ugtRelsFileName,
jsonStr) # 2.7 gets unicode this way
modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
def loadDqc0015signwarningRules(modelXbrl):
conceptRule = "http://fasb.org/dqcrules/arcrole/concept-rule" # FASB arcrule
rule0015 = "http://fasb.org/us-gaap/role/dqc/0015"
modelManager = modelXbrl.modelManager
cntlr = modelXbrl.modelManager.cntlr
# check for cached completed signwarnings
_signwarningsFileName = resourcesFilePath(modelManager, "signwarnings.json")
if os.path.exists(_signwarningsFileName):
_file = openFileStream(modelManager.cntlr, _signwarningsFileName, 'rt', encoding='utf-8')
signwarnings = json.load(_file) # {localName: date, ...}
_file.close()
return signwarnings
# load template rules
_fileName = resourcesFilePath(modelManager, "signwarnings-template.json")
if _fileName:
_file = openFileStream(modelXbrl.modelManager.cntlr, _fileName, 'rt', encoding='utf-8')
signwarnings = json.load(_file, object_pairs_hook=OrderedDict) # {localName: date, ...}
_file.close()
# load rules and add to signwarnings template
for dqcAbbr, dqcrtUrl in latestDqcrtDocs.items():
modelManager.addToLog(_("loading {} DQC Rules {}").format(dqcAbbr, dqcrtUrl), messageCode="info")
# load without SEC/EFM validation (doc file would not be acceptable)
priorValidateDisclosureSystem = modelManager.validateDisclosureSystem
modelManager.validateDisclosureSystem = False
from arelle import ModelXbrl
dqcrtInstance = ModelXbrl.load(modelManager,
# "http://xbrl.fasb.org/us-gaap/2012/elts/us-gaap-doc-2012-01-31.xml",
# load from zip (especially after caching) is incredibly faster
openFileSource(dqcrtUrl, cntlr),
_("built dqcrt table in cache"))
modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
if dqcrtInstance is None:
modelManager.addToLog(
_("%(name)s documentation not loaded"),
messageCode="arelle:notLoaded", messageArgs={"modelXbrl": val, "name":dqcAbbr})
else:
# load signwarnings from DQC 0015
dqcRelSet = dqcrtInstance.relationshipSet(conceptRule, rule0015)
for signWrnObj, headEltName in (("conceptNames", "Dqc_0015_ListOfElements"),
("excludedMemberNames", "Dqc_0015_ExcludeNonNegMembersAbstract"),
("excludedAxesMembers", "Dqc_0015_ExcludeNonNegAxisAbstract"),
("excludedAxesMembers", "Dqc_0015_ExcludeNonNegAxisMembersAbstract"),
("excludedMemberStrings", "Dqc_0015_ExcludeNonNegMemberStringsAbstract")):
headElts = dqcrtInstance.nameConcepts.get(headEltName,())
for headElt in headElts:
if signWrnObj == "excludedMemberStrings":
for refRel in dqcrtInstance.relationshipSet(XbrlConst.conceptReference).fromModelObject(headElt):
for refPart in refRel.toModelObject.iterchildren("{*}allowableSubString"):
for subStr in refPart.text.split():
signwarnings[signWrnObj].setdefault(nsAbbr, []).append(subStr)
else:
for ruleRel in dqcRelSet.fromModelObject(headElt):
elt = ruleRel.toModelObject
nsAbbr = abbreviatedNamespace(elt.qname.namespaceURI)
if signWrnObj in ("conceptNames", "excludedMemberNames"):
signwarnings[signWrnObj].setdefault(nsAbbr, []).append(elt.name)
else:
l = signwarnings[signWrnObj].setdefault(nsAbbr, {}).setdefault(elt.name, [])
if headEltName == "Dqc_0015_ExcludeNonNegAxisAbstract":
l.append("*")
else:
for memRel in dqcRelSet.fromModelObject(elt):
l.append(memRel.toModelObject.name)
jsonStr = _STR_UNICODE(json.dumps(signwarnings, ensure_ascii=False, indent=2)) # might not be unicode in 2.7
saveFile(cntlr, _signwarningsFileName, jsonStr) # 2.7 gets unicode this way
dqcrtInstance.close()
del dqcrtInstance # dereference closed modelXbrl
return signwarnings
def setup(val):
if not val.validateLoggingSemantic: # all checks herein are SEMANTIC
return
val.linroleDefinitionIsDisclosure = re.compile(r"-\s+Disclosure\s+-\s",
re.IGNORECASE)
val.linkroleDefinitionStatementSheet = re.compile(r"[^-]+-\s+Statement\s+-\s+.*", # no restriction to type of statement
re.IGNORECASE)
val.ugtNamespace = None
cntlr = val.modelXbrl.modelManager.cntlr
# load deprecated concepts for filed year of us-gaap
for ugt in ugtDocs:
ugtNamespace = ugt["namespace"]
if ugtNamespace in val.modelXbrl.namespaceDocs and len(val.modelXbrl.namespaceDocs[ugtNamespace]) > 0:
val.ugtNamespace = ugtNamespace
usgaapDoc = val.modelXbrl.namespaceDocs[ugtNamespace][0]
deprecationsJsonFile = usgaapDoc.filepathdir + os.sep + "deprecated-concepts.json"
file = None
try:
file = openFileStream(cntlr, deprecationsJsonFile, 'rt', encoding='utf-8')
val.usgaapDeprecations = json.load(file)
file.close()
except Exception:
if file:
file.close()
val.modelXbrl.modelManager.addToLog(_("loading us-gaap {0} deprecated concepts into cache").format(ugt["year"]))
startedAt = time.time()
ugtDocLB = ugt["docLB"]
val.usgaapDeprecations = {}
# load without SEC/EFM validation (doc file would not be acceptable)
priorValidateDisclosureSystem = val.modelXbrl.modelManager.validateDisclosureSystem
val.modelXbrl.modelManager.validateDisclosureSystem = False
deprecationsInstance = ModelXbrl.load(val.modelXbrl.modelManager,
# "http://xbrl.fasb.org/us-gaap/2012/elts/us-gaap-doc-2012-01-31.xml",
# load from zip (especially after caching) is incredibly faster
openFileSource(ugtDocLB, cntlr),
_("built deprecations table in cache"))
val.modelXbrl.modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
if deprecationsInstance is None:
val.modelXbrl.error("arelle:notLoaded",
_("US-GAAP documentation not loaded: %(file)s"),
modelXbrl=val, file=os.path.basename(ugtDocLB))
else:
# load deprecations
for labelRel in deprecationsInstance.relationshipSet(XbrlConst.conceptLabel).modelRelationships:
modelDocumentation = labelRel.toModelObject
conceptName = labelRel.fromModelObject.name
if modelDocumentation.role == 'http://www.xbrl.org/2009/role/deprecatedLabel':
val.usgaapDeprecations[conceptName] = (val.usgaapDeprecations.get(conceptName, ('',''))[0], modelDocumentation.text)
elif modelDocumentation.role == 'http://www.xbrl.org/2009/role/deprecatedDateLabel':
val.usgaapDeprecations[conceptName] = (modelDocumentation.text, val.usgaapDeprecations.get(conceptName, ('',''))[1])
jsonStr = _STR_UNICODE(json.dumps(val.usgaapDeprecations, ensure_ascii=False, indent=0)) # might not be unicode in 2.7
saveFile(cntlr, deprecationsJsonFile, jsonStr) # 2.7 gets unicode this way
deprecationsInstance.close()
del deprecationsInstance # dereference closed modelXbrl
val.modelXbrl.profileStat(_("build us-gaap deprecated concepts cache"), time.time() - startedAt)
ugtCalcsJsonFile = usgaapDoc.filepathdir + os.sep + "ugt-calculations.json"
ugtDefaultDimensionsJsonFile = usgaapDoc.filepathdir + os.sep + "ugt-default-dimensions.json"
file = None
try:
file = openFileStream(cntlr, ugtCalcsJsonFile, 'rt', encoding='utf-8')
val.usgaapCalculations = json.load(file)
file.close()
file = openFileStream(cntlr, ugtDefaultDimensionsJsonFile, 'rt', encoding='utf-8')
val.usgaapDefaultDimensions = json.load(file)
file.close()
except Exception:
if file:
file.close()
val.modelXbrl.modelManager.addToLog(_("loading us-gaap {0} calculations and default dimensions into cache").format(ugt["year"]))
startedAt = time.time()
ugtEntryXsd = ugt["entryXsd"]
val.usgaapCalculations = {}
val.usgaapDefaultDimensions = {}
# load without SEC/EFM validation (doc file would not be acceptable)
priorValidateDisclosureSystem = val.modelXbrl.modelManager.validateDisclosureSystem
val.modelXbrl.modelManager.validateDisclosureSystem = False
calculationsInstance = ModelXbrl.load(val.modelXbrl.modelManager,
# "http://xbrl.fasb.org/us-gaap/2012/entire/us-gaap-entryPoint-std-2012-01-31.xsd",
# load from zip (especially after caching) is incredibly faster
openFileSource(ugtEntryXsd, cntlr),
_("built us-gaap calculations cache"))
val.modelXbrl.modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
if calculationsInstance is None:
val.modelXbrl.error("arelle:notLoaded",
_("US-GAAP calculations not loaded: %(file)s"),
modelXbrl=val, file=os.path.basename(ugtEntryXsd))
else:
# load calculations
for ELR in calculationsInstance.relationshipSet(XbrlConst.summationItem).linkRoleUris:
elrRelSet = calculationsInstance.relationshipSet(XbrlConst.summationItem, ELR)
definition = ""
for roleType in calculationsInstance.roleTypes.get(ELR,()):
definition = roleType.definition
break
isStatementSheet = bool(val.linkroleDefinitionStatementSheet.match(definition))
elrUgtCalcs = {"#roots": [c.name for c in elrRelSet.rootConcepts],
"#definition": definition,
"#isStatementSheet": isStatementSheet}
for relFrom, rels in elrRelSet.fromModelObjects().items():
elrUgtCalcs[relFrom.name] = [rel.toModelObject.name for rel in rels]
val.usgaapCalculations[ELR] = elrUgtCalcs
jsonStr = _STR_UNICODE(json.dumps(val.usgaapCalculations, ensure_ascii=False, indent=0)) # might not be unicode in 2.7
saveFile(cntlr, ugtCalcsJsonFile, jsonStr) # 2.7 gets unicode this way
# load default dimensions
for defaultDimRel in calculationsInstance.relationshipSet(XbrlConst.dimensionDefault).modelRelationships:
if defaultDimRel.fromModelObject is not None and defaultDimRel.toModelObject is not None:
val.usgaapDefaultDimensions[defaultDimRel.fromModelObject.name] = defaultDimRel.toModelObject.name
jsonStr = _STR_UNICODE(json.dumps(val.usgaapDefaultDimensions, ensure_ascii=False, indent=0)) # might not be unicode in 2.7
saveFile(cntlr, ugtDefaultDimensionsJsonFile, jsonStr) # 2.7 gets unicode this way
calculationsInstance.close()
del calculationsInstance # dereference closed modelXbrl
val.modelXbrl.profileStat(_("build us-gaap calculations and default dimensions cache"), time.time() - startedAt)
break
val.deprecatedFactConcepts = defaultdict(list)
val.deprecatedDimensions = defaultdict(list)
val.deprecatedMembers = defaultdict(list)
def _make_cache(val, ugt, cntlr, ugt_default_dimensions_json_file):
"""
Creates a new caches for the Taxonomy default dimensions
:param val: ValidateXbrl to be validated
:type val: :class: '~arelle.ValidateXbrl.ValidateXbrl'
:param ugt: Taxonomy to check
:type ugt: str
:param ugt_default_dimensions_json_file: location to save json default
dimensions
:type ugt_default_dimensions_json_file: str
:return: no explicit return, but saves caches for dqc_us_0041
:rtype: None
"""
started_at = time.time()
ugt_entry_xsd = ugt["entryXsd"]
val.usgaapDefaultDimensions = {}
prior_validate_disclosure_system = (
val.modelXbrl.modelManager.validateDisclosureSystem
)
val.modelXbrl.modelManager.validateDisclosureSystem = False
ugt_entry_xsd_instance = (
ModelXbrl.load(
val.modelXbrl.modelManager,
openFileSource(ugt_entry_xsd, cntlr),
_("opened us-gaap entry xsd") # noqa
)
)
val.modelXbrl.modelManager.validateDisclosureSystem = (
prior_validate_disclosure_system
)
if ugt_entry_xsd_instance is None:
val.modelXbrl.error(
"arelle:notLoaded",
_("US-GAAP entry xsd not loaded: %(file)s"), # noqa
modelXbrl=val,
file=os.path.basename(ugt_entry_xsd)
)
else:
model_relationships = (
ugt_entry_xsd_instance.relationshipSet(
XbrlConst.dimensionDefault
).modelRelationships
)
for default_dim_rel in model_relationships:
if _default_dim_rel_is_instance(default_dim_rel):
from_name = default_dim_rel.fromModelObject.name
to_name = default_dim_rel.toModelObject.name
val.usgaapDefaultDimensions[from_name] = to_name
json_str = str(
json.dumps(
val.usgaapDefaultDimensions,
ensure_ascii=False,
indent=0
)
) # might not be unicode in 2.7
# 2.7 gets unicode this way
saveFile(cntlr, ugt_default_dimensions_json_file, json_str)
ugt_entry_xsd_instance.close()
del ugt_entry_xsd_instance # dereference closed modelXbrl
val.modelXbrl.profileStat(
_("build default dimensions cache"), # noqa
time.time() - started_at
)