def quickbooksGLrequest(qbReport=None, file=None):
from arelle.CntlrQuickBooks import supportedQbReports, qbRequest
from arelle.ModelValue import dateTime
errors = []
requestPathParts = request.urlparts[2].split('/')
viewRequested = "view" == requestPathParts[-1]
media = request.query.media or 'html'
fromDate = request.query.fromDate
toDate = request.query.toDate
if qbReport not in supportedQbReports:
errors.append(_("QuickBooks report '{0}' is not supported (please select from: {1})").format(
qbReport, ', '.join(supportedQbReports)))
if media not in ('xml', 'xhtml', 'html'):
errors.append(_("Media '{0}' is not supported for xbrl-gl (please select xhtml, html or xml)").format(media))
if not fromDate or dateTime(fromDate) is None:
errors.append(_("FromDate '{0}' missing or not valid").format(fromDate))
if not toDate or dateTime(toDate) is None:
errors.append(_("ToDate '{0}' missing or not valid").format(toDate))
if errors:
return errorReport(errors, media)
ticket = qbRequest(qbReport, fromDate, toDate, file)
result = htmlBody(tableRows([_("Request queued for QuickBooks...")], header=_("Quickbooks Request")), script='''
<script type="text/javascript">
<!--
var timer = setInterval("autoRefresh()", 1000 * 10);
function autoRefresh(){{location.href = "/rest/quickbooks/response?ticket={0}&media={1}&view={2}";}}
//-->
</script>
'''.format(ticket, media, viewRequested))
return result
def fn_dateTime(xc, p, contextItem, args):
if len(args) != 2: raise XPathContext.FunctionNumArgs()
date = anytypeArg(xc, args, 0, "xs:date", missingArgFallback=())
time = anytypeArg(xc, args, 1, "xs:time", missingArgFallback=())
if date is None or time is None:
return ()
return dateTime(date) + dayTimeDuration(time)
def parsed_value(fact: ModelFact):
if fact is None:
return None
concept = fact.concept # type: ModelConcept
if concept is None or concept.isTuple or fact.isNil:
return None
if concept.isFraction:
num, den = map(fractions.Fraction, fact.fractionValue)
return num / den
val = fact.value.strip()
if concept.isInteger:
return int(val)
elif concept.isNumeric:
dec = fact.decimals
if dec is None or dec == "INF": # show using decimals or reported format
dec = len(val.partition(".")[2])
else: # max decimals at 28
dec = max(
min(int(dec), 28), -28
) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here
num = roundValue(val, fact.precision,
dec) # round using reported decimals
return num
elif concept.baseXbrliType == 'dateItemType':
return dateTime(val)
elif concept.baseXbrliType == 'booleanItemType':
return val.lower() in ('1', 'true')
elif concept.isTextBlock:
return ' '.join(val.split())
return val
def checkEntries(self):
errors = []
if not self.cellFeed.value in rssFeeds and not isValidAbsolute(self.cellFeed.value):
errors.append(_("RSS feed field contents invalid"))
try:
if self.cellMatchText.value:
re.compile(self.cellMatchText.value)
except Exception as err:
errors.append(_("Match text field contents error {0}").format(err))
if self.cellFormulaFile.value and not os.path.exists(self.cellFormulaFile.value):
errors.append(_("Formula file not found {0}").format(self.cellFormulaFile.value))
if self.cellLogFile.value and not os.path.exists(os.path.dirname(self.cellLogFile.value)):
errors.append(_("Log file directory not found {0}").format(self.cellLogFile.value))
if self.cellEmailAddress.value:
if not emailPattern.match(self.cellEmailAddress.value):
errors.append(_("E-mail address format error").format(self.cellLogFile.value))
if not self.options.get("smtpEmailSettings"):
errors.append(_("Please enter E-mail server settings (SMTP)"))
if self.cellLatestPubDate.value and dateTime(self.cellLatestPubDate.value) is None:
errors.append(_("Latest pub date field contents invalid"))
if errors:
messagebox.showwarning(_("Dialog validation error(s)"),
"\n ".join(errors), parent=self)
return False
return True
def validateNode(modelXbrl, elt, node, baseXsdType, value):
if baseXsdType:
try:
if baseXsdType in ("decimal", "float", "double"):
node.xValue = float(value)
elif baseXsdType in ("integer", ):
node.xValue = int(value)
elif baseXsdType == "boolean":
if value in ("true", "1"): node.xValue = True
elif value in ("false", "0"): node.xValue = False
else: raise ValueError
elif baseXsdType == "QName":
node.xValue = qname(elt, value, castException=ValueError)
elif baseXsdType in ("normalizedString", "token", "language",
"NMTOKEN", "Name", "NCName", "ID", "IDREF",
"ENTITY"):
node.xValue = value.strip()
elif baseXsdType == "dateTime":
node.xValue = dateTime(value,
type=DATETIME,
castException=ValueError)
elif baseXsdType == "date":
node.xValue = dateTime(value,
type=DATE,
castException=ValueError)
else:
node.xValue = value
node.xValid = VALID
except ValueError:
if node.nodeType == 1:
modelXbrl.error(
_("Element {0} type {1} value error: {2}").format(
elt.tagName, baseXsdType, value), "err",
"xmlSchema:valueError")
else:
modelXbrl.error(
_("Element {0} attribute {1} type {2} value error: {3}"
).format(elt.tagName, node.name, baseXsdType, value),
"err", "xmlSchema:valueError")
node.xValue = None
node.xValid = INVALID
else:
node.xValue = None
node.xValid = UNKNOWN
def validateNode(modelXbrl, elt, node, baseXsdType, value):
if baseXsdType:
try:
if baseXsdType in ("decimal", "float", "double"):
node.xValue = float(value)
elif baseXsdType in ("integer",):
node.xValue = int(value)
elif baseXsdType == "boolean":
if value in ("true", "1"): node.xValue = True
elif value in ("false", "0"): node.xValue = False
else: raise ValueError
elif baseXsdType == "QName":
node.xValue = qname(elt, value, castException=ValueError)
elif baseXsdType in ("normalizedString","token","language","NMTOKEN","Name","NCName","ID","IDREF","ENTITY"):
node.xValue = value.strip()
elif baseXsdType == "dateTime":
node.xValue = dateTime(value, type=DATETIME, castException=ValueError)
elif baseXsdType == "date":
node.xValue = dateTime(value, type=DATE, castException=ValueError)
else:
node.xValue = value
node.xValid = VALID
except ValueError:
if node.nodeType == 1:
modelXbrl.error(
_("Element {0} type {1} value error: {2}").format(
elt.tagName,
baseXsdType,
value),
"err", "xmlSchema:valueError")
else:
modelXbrl.error(
_("Element {0} attribute {1} type {2} value error: {3}").format(
elt.tagName,
node.name,
baseXsdType,
value),
"err", "xmlSchema:valueError")
node.xValue = None
node.xValid = INVALID
else:
node.xValue = None
node.xValid = UNKNOWN
def quickbooksGLrequest(qbReport=None, file=None):
"""Initiate request to QuickBooks server for *get* requests to */rest/quickbooks/<qbReport>/xbrl-gl/...*.
:returns: html, xml, csv, text -- Return per media type argument and request arguments
"""
from arelle.CntlrQuickBooks import supportedQbReports, qbRequest
from arelle.ModelValue import dateTime
errors = []
requestPathParts = request.urlparts[2].split("/")
viewRequested = "view" == requestPathParts[-1]
media = request.query.media or "html"
fromDate = request.query.fromDate
toDate = request.query.toDate
if qbReport not in supportedQbReports:
errors.append(
_("QuickBooks report '{0}' is not supported (please select from: {1})").format(
qbReport, ", ".join(supportedQbReports)
)
)
if media not in ("xml", "xhtml", "html"):
errors.append(_("Media '{0}' is not supported for xbrl-gl (please select xhtml, html or xml)").format(media))
if not fromDate or dateTime(fromDate) is None:
errors.append(_("FromDate '{0}' missing or not valid").format(fromDate))
if not toDate or dateTime(toDate) is None:
errors.append(_("ToDate '{0}' missing or not valid").format(toDate))
if errors:
return errorReport(errors, media)
ticket = qbRequest(qbReport, fromDate, toDate, file)
result = htmlBody(
tableRows([_("Request queued for QuickBooks...")], header=_("Quickbooks Request")),
script="""
<script type="text/javascript">
<!--
var timer = setInterval("autoRefresh()", 1000 * 10);
function autoRefresh(){{location.href = "/rest/quickbooks/response?ticket={0}&media={1}&view={2}";}}
//-->
</script>
""".format(
ticket, media, viewRequested
),
)
return result
def checkEntries(self):
errors = []
if not self.cellEntityIdentScheme.value:
errors.append(_("Entity scheme invalid"))
if not self.cellEntityIdentValue.value:
errors.append(_("Entity identifier value invalid"))
if not self.cellStartDate.value or dateTime(self.cellStartDate.value) is None:
errors.append(_("Start date invalid"))
if not self.cellEndDate.value or dateTime(self.cellEndDate.value) is None:
errors.append(_("End date invalid"))
if self.cellMonetaryUnit.value not in monetaryUnits:
errors.append(_("Monetary unit invalid"))
if not decimalsPattern.match(self.cellMonetaryDecimals.value):
errors.append(_("Monetary decimals invalid"))
if not decimalsPattern.match(self.cellNonMonetaryDecimals.value):
errors.append(_("Non-monetary decimals invalid"))
if errors:
messagebox.showwarning(_("Dialog validation error(s)"),
"\n ".join(errors), parent=self)
return False
return True
def period_datetime(p, args, periodElement, addOneDay):
if len(args) != 1: raise XPathContext.FunctionNumArgs()
if len(args[0]) != 1: raise XPathContext.FunctionArgType(1,"xbrl:period")
period = args[0][0]
if (isinstance(period,xml.dom.Node) and period.nodeType == 1 and
period.localName == "period" and period.namespaceURI == XbrlConst.xbrli):
child = XmlUtil.child(period, XbrlConst.xbrli, periodElement)
if child:
return dateTime( child, addOneDay=addOneDay, type=DATETIME)
elif periodElement == "instant":
raise XPathContext.XPathException(p, 'xfie:PeriodIsNotInstant', _('Period is not instant'))
else:
raise XPathContext.XPathException(p, 'xfie:PeriodIsForever', _('Period is forever'))
raise XPathContext.FunctionArgType(1,"xbrl:period")
def period_datetime(p, args, periodElement, addOneDay):
if len(args) != 1: raise XPathContext.FunctionNumArgs()
if len(args[0]) != 1: raise XPathContext.FunctionArgType(1, "xbrl:period")
period = args[0][0]
if (isinstance(period, xml.dom.Node) and period.nodeType == 1
and period.localName == "period"
and period.namespaceURI == XbrlConst.xbrli):
child = XmlUtil.child(period, XbrlConst.xbrli, periodElement)
if child:
return dateTime(child, addOneDay=addOneDay, type=DATETIME)
elif periodElement == "instant":
raise XPathContext.XPathException(p, 'xfie:PeriodIsNotInstant',
_('Period is not instant'))
else:
raise XPathContext.XPathException(p, 'xfie:PeriodIsForever',
_('Period is forever'))
raise XPathContext.FunctionArgType(1, "xbrl:period")
def validateValue(modelXbrl, elt, attrTag, baseXsdType, value, isNillable=False, isNil=False, facets=None):
if baseXsdType:
try:
u'''
if (len(value) == 0 and attrTag is None and not isNillable and
baseXsdType not in ("anyType", "string", "normalizedString", "token", "NMTOKEN", "anyURI", "noContent")):
raise ValueError("missing value for not nillable element")
'''
xValid = VALID
whitespaceReplace = (baseXsdType == u"normalizedString")
whitespaceCollapse = (not whitespaceReplace and baseXsdType != u"string")
isList = baseXsdType in set([u"IDREFS", u"ENTITIES", u"NMTOKENS"])
if isList:
baseXsdType = baseXsdType[:-1] # remove plural
pattern = baseXsdTypePatterns.get(baseXsdType)
if facets:
if u"pattern" in facets:
pattern = facets[u"pattern"]
# note multiple patterns are or'ed togetner, which isn't yet implemented!
if u"whiteSpace" in facets:
whitespaceReplace, whitespaceCollapse = {u"preserve":(False,False), u"replace":(True,False), u"collapse":(False,True)}[facets[u"whiteSpace"]]
if whitespaceReplace:
value = normalizeWhitespacePattern.sub(u' ', value)
elif whitespaceCollapse:
value = collapseWhitespacePattern.sub(u' ', value.strip())
if baseXsdType == u"noContent":
if len(value) > 0 and not value.isspace():
raise ValueError(u"value content not permitted")
# note that sValue and xValue are not innerText but only text elements on specific element (or attribute)
xValue = sValue = None
xValid = VALID_NO_CONTENT # notify others that element may contain subelements (for stringValue needs)
elif not value and isNil and isNillable: # rest of types get None if nil/empty value
xValue = sValue = None
else:
if pattern is not None:
if ((isList and any(pattern.match(v) is None for v in value.split())) or
(not isList and pattern.match(value) is None)):
raise ValueError(u"pattern facet " + facets[u"pattern"].pattern if facets and u"pattern" in facets else u"pattern mismatch")
if facets:
if u"enumeration" in facets and value not in facets[u"enumeration"]:
raise ValueError(u"{0} is not in {1}".format(value, facets[u"enumeration"]))
if u"length" in facets and len(value) != facets[u"length"]:
raise ValueError(u"length {0}, expected {1}".format(len(value), facets[u"length"]))
if u"minLength" in facets and len(value) < facets[u"minLength"]:
raise ValueError(u"length {0}, minLength {1}".format(len(value), facets[u"minLength"]))
if u"maxLength" in facets and len(value) > facets[u"maxLength"]:
raise ValueError(u"length {0}, maxLength {1}".format(len(value), facets[u"maxLength"]))
if baseXsdType in set([u"string", u"normalizedString", u"language", u"token", u"NMTOKEN",u"Name",u"NCName",u"IDREF",u"ENTITY"]):
xValue = sValue = value
elif baseXsdType == u"ID":
xValue = sValue = value
xValid = VALID_ID
elif baseXsdType == u"anyURI":
if value: # allow empty strings to be valid anyURIs
if UrlUtil.relativeUrlPattern.match(value) is None:
raise ValueError(u"IETF RFC 2396 4.3 syntax")
# encode PSVI xValue similarly to Xerces and other implementations
xValue = anyURI(UrlUtil.anyUriQuoteForPSVI(value))
sValue = value
elif baseXsdType in (u"decimal", u"float", u"double"):
if baseXsdType == u"decimal":
if decimalPattern.match(value) is None:
raise ValueError(u"lexical pattern mismatch")
xValue = Decimal(value)
sValue = float(value) # s-value uses Number (float) representation
else:
if floatPattern.match(value) is None:
raise ValueError(u"lexical pattern mismatch")
xValue = sValue = float(value)
if facets:
if u"totalDigits" in facets and len(value.replace(u".",u"")) > facets[u"totalDigits"]:
raise ValueError(u"totalDigits facet {0}".format(facets[u"totalDigits"]))
if u"fractionDigits" in facets and ( u'.' in value and
len(value[value.index(u'.') + 1:]) > facets[u"fractionDigits"]):
raise ValueError(u"fraction digits facet {0}".format(facets[u"fractionDigits"]))
if u"maxInclusive" in facets and xValue > facets[u"maxInclusive"]:
raise ValueError(u" > maxInclusive {0}".format(facets[u"maxInclusive"]))
if u"maxExclusive" in facets and xValue >= facets[u"maxExclusive"]:
raise ValueError(u" >= maxInclusive {0}".format(facets[u"maxExclusive"]))
if u"minInclusive" in facets and xValue < facets[u"minInclusive"]:
raise ValueError(u" < minInclusive {0}".format(facets[u"minInclusive"]))
if u"minExclusive" in facets and xValue <= facets[u"minExclusive"]:
raise ValueError(u" <= minExclusive {0}".format(facets[u"minExclusive"]))
elif baseXsdType in set([u"integer",
u"nonPositiveInteger",u"negativeInteger",u"nonNegativeInteger",u"positiveInteger",
u"long",u"unsignedLong",
u"int",u"unsignedInt",
u"short",u"unsignedShort",
u"byte",u"unsignedByte"]):
xValue = sValue = _INT(value)
if ((baseXsdType in set([u"nonNegativeInteger",u"unsignedLong",u"unsignedInt"])
and xValue < 0) or
(baseXsdType == u"nonPositiveInteger" and xValue > 0) or
(baseXsdType == u"positiveInteger" and xValue <= 0) or
(baseXsdType == u"byte" and not -128 <= xValue < 127) or
(baseXsdType == u"unsignedByte" and not 0 <= xValue < 255) or
(baseXsdType == u"short" and not -32768 <= xValue < 32767) or
(baseXsdType == u"unsignedShort" and not 0 <= xValue < 65535) or
(baseXsdType == u"positiveInteger" and xValue <= 0)):
raise ValueError(u"{0} is not {1}".format(value, baseXsdType))
if facets:
if u"totalDigits" in facets and len(value.replace(u".",u"")) > facets[u"totalDigits"]:
raise ValueError(u"totalDigits facet {0}".format(facets[u"totalDigits"]))
if u"fractionDigits" in facets and ( u'.' in value and
len(value[value.index(u'.') + 1:]) > facets[u"fractionDigits"]):
raise ValueError(u"fraction digits facet {0}".format(facets[u"fractionDigits"]))
if u"maxInclusive" in facets and xValue > facets[u"maxInclusive"]:
raise ValueError(u" > maxInclusive {0}".format(facets[u"maxInclusive"]))
if u"maxExclusive" in facets and xValue >= facets[u"maxExclusive"]:
raise ValueError(u" >= maxInclusive {0}".format(facets[u"maxExclusive"]))
if u"minInclusive" in facets and xValue < facets[u"minInclusive"]:
raise ValueError(u" < minInclusive {0}".format(facets[u"minInclusive"]))
if u"minExclusive" in facets and xValue <= facets[u"minExclusive"]:
raise ValueError(u" <= minExclusive {0}".format(facets[u"minExclusive"]))
elif baseXsdType == u"boolean":
if value in (u"true", u"1"):
xValue = sValue = True
elif value in (u"false", u"0"):
xValue = sValue = False
else: raise ValueError
elif baseXsdType == u"QName":
xValue = qnameEltPfxName(elt, value, prefixException=ValueError)
#xValue = qname(elt, value, castException=ValueError, prefixException=ValueError)
sValue = value
u''' not sure here, how are explicitDimensions validated, but bad units not?
if xValue.namespaceURI in modelXbrl.namespaceDocs:
if (xValue not in modelXbrl.qnameConcepts and
xValue not in modelXbrl.qnameTypes and
xValue not in modelXbrl.qnameAttributes and
xValue not in modelXbrl.qnameAttributeGroups):
raise ValueError("qname not defined " + str(xValue))
'''
elif baseXsdType in (u"XBRLI_DECIMALSUNION", u"XBRLI_PRECISIONUNION"):
xValue = sValue = value if value == u"INF" else _INT(value)
elif baseXsdType in (u"XBRLI_NONZERODECIMAL"):
xValue = sValue = _INT(value)
if xValue == 0:
raise ValueError(u"invalid value")
elif baseXsdType == u"XBRLI_DATEUNION":
xValue = dateTime(value, type=DATEUNION, castException=ValueError)
sValue = value
elif baseXsdType == u"dateTime":
xValue = dateTime(value, type=DATETIME, castException=ValueError)
sValue = value
elif baseXsdType == u"date":
xValue = dateTime(value, type=DATE, castException=ValueError)
sValue = value
elif baseXsdType == u"regex-pattern":
# for facet compiling
try:
sValue = value
if value in xmlSchemaPatterns:
xValue = xmlSchemaPatterns[value]
else:
if ur"\i" in value or ur"\c" in value:
value = value.replace(ur"\i", iNameChar).replace(ur"\c", cNameChar)
xValue = re_compile(value + u"$") # must match whole string
except Exception, err:
raise ValueError(err)
else:
if baseXsdType in lexicalPatterns:
match = lexicalPatterns[baseXsdType].match(value)
if match is None:
raise ValueError(u"lexical pattern mismatch")
if baseXsdType == u"gMonthDay":
month, day, zSign, zHrMin, zHr, zMin = match.groups()
if int(day) > {2:29, 4:30, 6:30, 9:30, 11:30, 1:31, 3:31, 5:31, 7:31, 8:31, 10:31, 12:31}[int(month)]:
raise ValueError(u"invalid day {0} for month {1}".format(day, month))
xValue = gMonthDay(month, day)
elif baseXsdType == u"gYearMonth":
year, month, zSign, zHrMin, zHr, zMin = match.groups()
xValue = gYearMonth(year, month)
elif baseXsdType == u"gYear":
year, zSign, zHrMin, zHr, zMin = match.groups()
xValue = gYear(year)
elif baseXsdType == u"gMonth":
month, zSign, zHrMin, zHr, zMin = match.groups()
xValue = gMonth(month)
elif baseXsdType == u"gDay":
day, zSign, zHrMin, zHr, zMin = match.groups()
xValue = gDay(day)
else:
xValue = value
else: # no lexical pattern, forget compiling value
xValue = value
sValue = value
def validateValue(modelXbrl,
elt,
attrTag,
baseXsdType,
value,
isNillable=False,
isNil=False,
facets=None):
if baseXsdType:
try:
'''
if (len(value) == 0 and attrTag is None and not isNillable and
baseXsdType not in ("anyType", "string", "normalizedString", "token", "NMTOKEN", "anyURI", "noContent")):
raise ValueError("missing value for not nillable element")
'''
xValid = VALID
whitespaceReplace = (baseXsdType == "normalizedString")
whitespaceCollapse = (not whitespaceReplace
and baseXsdType != "string")
isList = baseXsdType in {"IDREFS", "ENTITIES", "NMTOKENS"}
if isList:
baseXsdType = baseXsdType[:-1] # remove plural
if facets:
if "minLength" not in facets:
facets = facets.copy()
facets["minLength"] = 1
else:
facets = {"minLength": 1}
pattern = baseXsdTypePatterns.get(baseXsdType)
if facets:
if "pattern" in facets:
pattern = facets["pattern"]
# note multiple patterns are or'ed togetner, which isn't yet implemented!
if "whiteSpace" in facets:
whitespaceReplace, whitespaceCollapse = {
"preserve": (False, False),
"replace": (True, False),
"collapse": (False, True)
}[facets["whiteSpace"]]
if whitespaceReplace:
value = normalizeWhitespacePattern.sub(
' ', value) # replace tab, line feed, return with space
elif whitespaceCollapse:
value = collapseWhitespacePattern.sub(' ', value).strip(
' '
) # collapse multiple spaces, tabs, line feeds and returns to single space
if baseXsdType == "noContent":
if len(value) > 0 and not entirelyWhitespacePattern.match(
value): # only xml schema pattern whitespaces removed
raise ValueError("value content not permitted")
# note that sValue and xValue are not innerText but only text elements on specific element (or attribute)
xValue = sValue = None
xValid = VALID_NO_CONTENT # notify others that element may contain subelements (for stringValue needs)
elif not value and isNil and isNillable: # rest of types get None if nil/empty value
xValue = sValue = None
else:
if pattern is not None:
if ((isList and any(
pattern.match(v) is None for v in value.split()))
or (not isList and pattern.match(value) is None)):
raise ValueError(
"pattern facet " +
facets["pattern"].pattern if facets
and "pattern" in facets else "pattern mismatch")
if facets:
if "enumeration" in facets and value not in facets[
"enumeration"]:
raise ValueError("{0} is not in {1}".format(
value, facets["enumeration"].keys()))
if "length" in facets and len(value) != facets["length"]:
raise ValueError("length {0}, expected {1}".format(
len(value), facets["length"]))
if "minLength" in facets and len(
value) < facets["minLength"]:
raise ValueError("length {0}, minLength {1}".format(
len(value), facets["minLength"]))
if "maxLength" in facets and len(
value) > facets["maxLength"]:
raise ValueError("length {0}, maxLength {1}".format(
len(value), facets["maxLength"]))
if baseXsdType in {
"string", "normalizedString", "language",
"languageOrEmpty", "token", "NMTOKEN", "Name",
"NCName", "IDREF", "ENTITY"
}:
xValue = sValue = value
elif baseXsdType == "ID":
xValue = sValue = value
xValid = VALID_ID
elif baseXsdType == "anyURI":
if value: # allow empty strings to be valid anyURIs
if UrlUtil.relativeUrlPattern.match(value) is None:
raise ValueError("IETF RFC 2396 4.3 syntax")
# encode PSVI xValue similarly to Xerces and other implementations
xValue = anyURI(UrlUtil.anyUriQuoteForPSVI(value))
sValue = value
elif baseXsdType in ("decimal", "float", "double",
"XBRLI_NONZERODECIMAL"):
if baseXsdType in ("decimal", "XBRLI_NONZERODECIMAL"):
if decimalPattern.match(value) is None:
raise ValueError("lexical pattern mismatch")
xValue = Decimal(value)
sValue = float(
value
) # s-value uses Number (float) representation
if sValue == 0 and baseXsdType == "XBRLI_NONZERODECIMAL":
raise ValueError("zero is not allowed")
else:
if floatPattern.match(value) is None:
raise ValueError("lexical pattern mismatch")
xValue = sValue = float(value)
if facets:
if "totalDigits" in facets and len(
value.replace(".",
"")) > facets["totalDigits"]:
raise ValueError("totalDigits facet {0}".format(
facets["totalDigits"]))
if "fractionDigits" in facets and (
'.' in value
and len(value[value.index('.') + 1:]) >
facets["fractionDigits"]):
raise ValueError(
"fraction digits facet {0}".format(
facets["fractionDigits"]))
if "maxInclusive" in facets and xValue > facets[
"maxInclusive"]:
raise ValueError(" > maxInclusive {0}".format(
facets["maxInclusive"]))
if "maxExclusive" in facets and xValue >= facets[
"maxExclusive"]:
raise ValueError(" >= maxInclusive {0}".format(
facets["maxExclusive"]))
if "minInclusive" in facets and xValue < facets[
"minInclusive"]:
raise ValueError(" < minInclusive {0}".format(
facets["minInclusive"]))
if "minExclusive" in facets and xValue <= facets[
"minExclusive"]:
raise ValueError(" <= minExclusive {0}".format(
facets["minExclusive"]))
elif baseXsdType in {
"integer", "nonPositiveInteger", "negativeInteger",
"nonNegativeInteger", "positiveInteger", "long",
"unsignedLong", "int", "unsignedInt", "short",
"unsignedShort", "byte", "unsignedByte"
}:
xValue = sValue = _INT(value)
if ((baseXsdType in {
"nonNegativeInteger", "unsignedLong", "unsignedInt"
} and xValue < 0) or
(baseXsdType == "nonPositiveInteger" and xValue > 0) or
(baseXsdType == "positiveInteger" and xValue <= 0) or
(baseXsdType == "byte" and not -128 <= xValue < 127)
or (baseXsdType == "unsignedByte"
and not 0 <= xValue < 255)
or (baseXsdType == "short"
and not -32768 <= xValue < 32767)
or (baseXsdType == "unsignedShort"
and not 0 <= xValue < 65535) or
(baseXsdType == "positiveInteger" and xValue <= 0)):
raise ValueError("{0} is not {1}".format(
value, baseXsdType))
if facets:
if "totalDigits" in facets and len(
value.replace(".",
"")) > facets["totalDigits"]:
raise ValueError("totalDigits facet {0}".format(
facets["totalDigits"]))
if "fractionDigits" in facets and (
'.' in value
and len(value[value.index('.') + 1:]) >
facets["fractionDigits"]):
raise ValueError(
"fraction digits facet {0}".format(
facets["fractionDigits"]))
if "maxInclusive" in facets and xValue > facets[
"maxInclusive"]:
raise ValueError(" > maxInclusive {0}".format(
facets["maxInclusive"]))
if "maxExclusive" in facets and xValue >= facets[
"maxExclusive"]:
raise ValueError(" >= maxInclusive {0}".format(
facets["maxExclusive"]))
if "minInclusive" in facets and xValue < facets[
"minInclusive"]:
raise ValueError(" < minInclusive {0}".format(
facets["minInclusive"]))
if "minExclusive" in facets and xValue <= facets[
"minExclusive"]:
raise ValueError(" <= minExclusive {0}".format(
facets["minExclusive"]))
elif baseXsdType == "boolean":
if value in ("true", "1"):
xValue = sValue = True
elif value in ("false", "0"):
xValue = sValue = False
else:
raise ValueError
elif baseXsdType == "QName":
xValue = qnameEltPfxName(elt,
value,
prefixException=ValueError)
#xValue = qname(elt, value, castException=ValueError, prefixException=ValueError)
sValue = value
''' not sure here, how are explicitDimensions validated, but bad units not?
if xValue.namespaceURI in modelXbrl.namespaceDocs:
if (xValue not in modelXbrl.qnameConcepts and
xValue not in modelXbrl.qnameTypes and
xValue not in modelXbrl.qnameAttributes and
xValue not in modelXbrl.qnameAttributeGroups):
raise ValueError("qname not defined " + str(xValue))
'''
elif baseXsdType == "enumerationHrefs":
xValue = [qnameHref(href) for href in value.split()]
sValue = value
elif baseXsdType == "enumerationQNames":
xValue = [
qnameEltPfxName(elt, qn, prefixException=ValueError)
for qn in value.split()
]
sValue = value
elif baseXsdType in ("XBRLI_DECIMALSUNION",
"XBRLI_PRECISIONUNION"):
xValue = sValue = value if value == "INF" else _INT(value)
elif baseXsdType in ("XBRLI_NONZERODECIMAL"):
xValue = sValue = _INT(value)
if xValue == 0:
raise ValueError("invalid value")
elif baseXsdType == "regex-pattern":
# for facet compiling
try:
sValue = value
if value in xmlSchemaPatterns:
xValue = xmlSchemaPatterns[value]
else:
if r"\i" in value or r"\c" in value:
value = value.replace(r"[\i-[:]]", iNameChar).replace(r"\i", iNameChar) \
.replace(r"[\c-[:]]", cMinusCNameChar).replace(r"\c", cNameChar)
xValue = re_compile(value +
"$") # must match whole string
except Exception as err:
raise ValueError(err)
elif baseXsdType == "fraction":
numeratorStr, denominatorStr = elt.fractionValue
if numeratorStr == INVALIDixVALUE or denominatorStr == INVALIDixVALUE:
sValue = xValue = INVALIDixVALUE
xValid = INVALID
else:
sValue = value
numeratorNum = float(numeratorStr)
denominatorNum = float(denominatorStr)
if numeratorNum.is_integer(
) and denominatorNum.is_integer():
xValue = Fraction(int(numeratorNum),
int(denominatorNum))
else:
xValue = Fraction(numeratorNum / denominatorNum)
else:
if baseXsdType in lexicalPatterns:
match = lexicalPatterns[baseXsdType].match(value)
if match is None:
raise ValueError("lexical pattern mismatch")
if baseXsdType == "XBRLI_DATEUNION":
xValue = dateTime(value,
type=DATEUNION,
castException=ValueError)
sValue = value
elif baseXsdType == "dateTime":
xValue = dateTime(value,
type=DATETIME,
castException=ValueError)
sValue = value
elif baseXsdType == "date":
xValue = dateTime(value,
type=DATE,
castException=ValueError)
sValue = value
elif baseXsdType == "gMonthDay":
month, day, zSign, zHrMin, zHr, zMin = match.groups(
)
if int(day) > {
2: 29,
4: 30,
6: 30,
9: 30,
11: 30,
1: 31,
3: 31,
5: 31,
7: 31,
8: 31,
10: 31,
12: 31
}[int(month)]:
raise ValueError(
"invalid day {0} for month {1}".format(
day, month))
xValue = gMonthDay(month, day)
elif baseXsdType == "gYearMonth":
year, month, zSign, zHrMin, zHr, zMin = match.groups(
)
xValue = gYearMonth(year, month)
elif baseXsdType == "gYear":
year, zSign, zHrMin, zHr, zMin = match.groups()
xValue = gYear(year)
elif baseXsdType == "gMonth":
month, zSign, zHrMin, zHr, zMin = match.groups()
xValue = gMonth(month)
elif baseXsdType == "gDay":
day, zSign, zHrMin, zHr, zMin = match.groups()
xValue = gDay(day)
elif baseXsdType == "duration":
xValue = isoDuration(value)
else:
xValue = value
else: # no lexical pattern, forget compiling value
xValue = value
sValue = value
except (ValueError, InvalidOperation) as err:
if ModelInlineValueObject is not None and isinstance(
elt, ModelInlineValueObject):
errElt = "{0} fact {1}".format(elt.elementQname, elt.qname)
else:
errElt = elt.elementQname
if attrTag:
modelXbrl.error(
"xmlSchema:valueError",
_("Element %(element)s attribute %(attribute)s type %(typeName)s value error: %(value)s, %(error)s"
),
modelObject=elt,
element=errElt,
attribute=XmlUtil.clarkNotationToPrefixedName(
elt, attrTag, isAttribute=True),
typeName=baseXsdType,
value=strTruncate(value, 30),
error=err)
else:
modelXbrl.error(
"xmlSchema:valueError",
_("Element %(element)s type %(typeName)s value error: %(value)s, %(error)s"
),
modelObject=elt,
element=errElt,
typeName=baseXsdType,
value=strTruncate(value, 30),
error=err)
xValue = None
sValue = value
xValid = INVALID
else:
xValue = sValue = None
xValid = UNKNOWN
if attrTag:
try: # dynamically allocate attributes (otherwise given shared empty set)
xAttributes = elt.xAttributes
except AttributeError:
elt.xAttributes = xAttributes = {}
xAttributes[attrTag] = ModelAttribute(elt, attrTag, xValid, xValue,
sValue, value)
else:
elt.xValid = xValid
elt.xValue = xValue
elt.sValue = sValue
from arelle.ModelValue import qname, dateTime, DATE
from arelle.ValidateXbrlCalcs import inferredDecimals, rangeValue, insignificantDigits
from arelle.XbrlConst import xbrli, qnXbrliXbrl
try:
import regex as re
except ImportError:
import re
from collections import defaultdict
memNameNumPattern = re.compile(r"^([A-Za-z-]+)([0-9]+)$")
compTxmyNamespacePattern = re.compile(
r"http://www.govtalk.gov.uk/uk/fr/tax/uk-hmrc-ct/[0-9-]{10}")
# capture background-image or list-style-image with URL function
styleImgUrlPattern = re.compile(r"[a-z]+-image:\s*url[(][^)]+[)]")
EMPTYDICT = {}
_6_APR_2008 = dateTime("2008-04-06", type=DATE)
commonMandatoryItems = {
"EntityCurrentLegalOrRegisteredName", "StartDateForPeriodCoveredByReport",
"EndDateForPeriodCoveredByReport", "BalanceSheetDate"
}
mandatoryItems = {
"ukGAAP": commonMandatoryItems | {
"DateApprovalAccounts", "NameDirectorSigningAccounts", "EntityDormant",
"EntityTrading", "DateSigningDirectorsReport", "DirectorSigningReport"
},
"charities": commonMandatoryItems | {
"DateAuthorisationFinancialStatementsForIssue",
"DirectorSigningFinancialStatements", "EntityDormantTruefalse",
"EntityTradingStatus", "AccountingStandardsApplied",
"AccountingStandardsApplied"
def atomize(self, p, x):
# sequence
if isinstance(x, SEQUENCE_TYPES):
sequence = []
for item in self.flattenSequence(x):
atomizedItem = self.atomize(p, item)
if atomizedItem != []:
sequence.append(atomizedItem)
return sequence
# individual items
if isinstance(x, _RANGE):
return x
baseXsdType = None
e = None
if isinstance(x, ModelFact):
if x.isTuple:
raise XPathException(p, 'err:FOTY0012', _('Atomizing tuple {0} that does not have a typed value').format(x))
if x.isNil:
return []
baseXsdType = x.concept.baseXsdType
v = x.value # resolves default value
e = x
elif isinstance(x, ModelAttribute): # ModelAttribute is a tuple (below), check this first!
return x.xValue
else:
if isinstance(x, ModelObject):
e = x
if e is not None:
if getattr(e, "xValid", 0) == VALID_NO_CONTENT:
raise XPathException(p, 'err:FOTY0012', _('Atomizing element {0} that does not have a typed value').format(x))
if e.get("{http://www.w3.org/2001/XMLSchema-instance}nil") == "true":
return []
try:
if e.xValid >= VALID:
return e.xValue
except AttributeError:
pass
modelXbrl = x.modelXbrl
modelConcept = modelXbrl.qnameConcepts.get(qname(x))
if modelConcept is not None:
baseXsdType = modelConcept.baseXsdType
v = x.stringValue
if baseXsdType in ("float", "double"):
try:
x = float(v)
except ValueError:
raise XPathException(p, 'err:FORG0001', _('Atomizing {0} to a {1} does not have a proper value').format(x,baseXsdType))
elif baseXsdType == "decimal":
try:
x = Decimal(v)
except InvalidOperation:
raise XPathException(p, 'err:FORG0001', _('Atomizing {0} to decimal does not have a proper value'))
elif baseXsdType in ("integer",
"nonPositiveInteger","negativeInteger","nonNegativeInteger","positiveInteger",
"long","unsignedLong",
"int","unsignedInt",
"short","unsignedShort",
"byte","unsignedByte"):
try:
x = _INT(v)
except ValueError:
raise XPathException(p, 'err:FORG0001', _('Atomizing {0} to an integer does not have a proper value').format(x))
elif baseXsdType == "boolean":
x = (v == "true" or v == "1")
elif baseXsdType == "QName" and e is not None:
x = qname(e, v)
elif baseXsdType == "anyURI":
x = anyURI(v.strip())
elif baseXsdType in ("normalizedString","token","language","NMTOKEN","Name","NCName","ID","IDREF","ENTITY"):
x = v.strip()
elif baseXsdType == "XBRLI_DATEUNION":
x = dateTime(v, type=DATEUNION)
elif baseXsdType == "date":
x = dateTime(v, type=DATE)
elif baseXsdType == "dateTime":
x = dateTime(v, type=DATETIME)
elif baseXsdType == "noContent":
x = None # can't be atomized
elif baseXsdType:
x = str(v)
return x
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri):
from openpyxl import load_workbook
from arelle import ModelDocument, ModelXbrl, XmlUtil
from arelle.ModelDocument import ModelDocumentReference
from arelle.ModelValue import qname
try:
currentAction = "initializing"
startedAt = time.time()
if os.path.isabs(oimFile):
# allow relative filenames to loading directory
priorCWD = os.getcwd()
os.chdir(os.path.dirname(oimFile))
else:
priorCWD = None
currentAction = "determining file type"
isJSON = oimFile.endswith(".json") and not oimFile.endswith("-metadata.json")
isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json")
isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls")
isCSVorXL = isCSV or isXL
instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl"
if isJSON:
currentAction = "loading and parsing JSON OIM file"
with io.open(oimFile, 'rt', encoding='utf-8') as f:
oimObject = json.load(f)
missing = [t for t in ("dtsReferences", "prefixes", "facts") if t not in oimObject]
if missing:
raise OIMException("oime:missingJSONelements",
_("The required elements %(missing)s {} in JSON input"),
missing = ", ".join(missing))
currentAction = "identifying JSON objects"
dtsReferences = oimObject["dtsReferences"]
prefixes = oimObject["prefixes"]
facts = oimObject["facts"]
footnotes = oimOject["facts"] # shares this object
elif isCSV:
currentAction = "identifying CSV input tables"
if sys.version[0] >= '3':
csvOpenMode = 'w'
csvOpenNewline = ''
else:
csvOpenMode = 'wb' # for 2.7
csvOpenNewline = None
oimFileBase = None
if "-facts" in oimFile:
oimFileBase = oimFile.partition("-facts")[0]
else:
for suffix in ("-dtsReferences.csv", "-defaults.csv", "-prefixes.csv", "-footnotes.csv", "-metadata.json"):
if oimFile.endswith(suffix):
oimFileBase = oimFile[:-length(suffix)]
break
if oimFileBase is None:
raise OIMException("oime:missingCSVtables",
_("Unable to identify CSV tables file name pattern"))
if (not os.path.exists(oimFileBase + "-dtsReferences.csv") or
not os.path.exists(oimFileBase + "-prefixes.csv")):
raise OIMException("oime:missingCSVtables",
_("Unable to identify CSV tables for dtsReferences or prefixes"))
instanceFileName = oimFileBase + ".xbrl"
currentAction = "loading CSV dtsReferences table"
dtsReferences = []
with io.open(oimFileBase + "-dtsReferences.csv", 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
else:
dtsReferences.append(dict((header[j], col) for j, col in enumerate(row)))
currentAction = "loading CSV prefixes table"
prefixes = {}
with io.open(oimFileBase + "-prefixes.csv", 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = dict((col,i) for i,col in enumerate(row))
else:
prefixes[row[header["prefix"]]] = row[header["URI"]]
currentAction = "loading CSV defaults table"
defaults = {}
if os.path.exists(oimFileBase + "-defaults.csv"):
with io.open(oimFileBase + "-defaults.csv", 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
fileCol = row.index("file")
else:
defaults[row[fileCol]] = dict((header[j], col) for j, col in enumerate(row) if j != fileCol)
currentAction = "loading CSV facts tables"
facts = []
_dir = os.path.dirname(oimFileBase)
for filename in os.listdir(_dir):
filepath = os.path.join(_dir, filename)
if "-facts" in filename and filepath.startswith(oimFileBase):
tableDefaults = defaults.get(filename, {})
with io.open(filepath, 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
else:
fact = {}
fact.update(tableDefaults)
for j, col in enumerate(row):
if col is not None:
if header[j].endswith("Value"):
if col: # ignore empty columns (= null CSV value)
fact["value"] = col
else:
fact[header[j]] = col
facts.append(fact)
footnotes = []
if os.path.exists(oimFileBase + "-footnotes.csv"):
with io.open(oimFileBase + "-footnotes.csv", 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
else:
footnotes.append(dict((header[j], col) for j, col in enumerate(row) if col))
elif isXL:
oimWb = load_workbook(oimFile, read_only=True, data_only=True)
sheetNames = oimWb.get_sheet_names()
if (not any(sheetName == "prefixes" for sheetName in sheetNames) or
not any(sheetName == "dtsReferences" for sheetName in sheetNames) or
not any("facts" in sheetName for sheetName in sheetNames)):
if priorCWD: os.chdir(priorCWD)
return None
try:
dtsReferences = []
for i, row in oimWb["dtsReferences"]:
if i == 0:
header = [col.value for col in row]
else:
dtsReferences.append(dict((header[j], col.value) for j, col in enumerate(row)))
prefixes = {}
for i, row in oimWb["dtsReferences"]:
if i == 0:
header = dict((col.value,i) for i,col in enumerate(row))
else:
prefixes[row[header["prefix"]].value] = row[header["URI"].value]
defaults = {}
for i, row in oimW.get("defaults", ()):
if i == 0:
header = dict((col.value,i) for i,col in enumerate(row))
fileCol = header["file"]
else:
defaults[row[fileCol].value] = dict((header[j], col.value) for j, col in enumerate(row) if j != fileCol)
facts = []
_dir = os.path.dirpart(oimFileBase)
for sheetName in sheetNames:
if sheetName == "facts" or "-facts" in sheetName:
for i, row in oimWb[sheetName]:
if i == 0:
header = [col.value for col in row]
else:
fact = {}
fact.update(tableDefaults)
for j, col in enumerate(row):
if col.value is not None:
if header[j].endswith("Value"):
fact["value"] = str(col.value)
else:
fact[header[j]] = str(col.value)
facts.append(fact)
footnotes = []
for i, row in oimWb.get("footnotes", ()):
if i == 0:
header = dict((col.value,i) for i,col in enumerate(row) if col.value)
else:
footnotes.append(dict((header[j], col.value) for j, col in enumerate(row) if col.value))
except Exception as ex:
if priorCWD: os.chdir(priorCWD)
return None
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults
# create the instance document
modelXbrl.blockDpmDBrecursion = True
modelXbrl.modelDocument = createModelDocument(
modelXbrl,
Type.INSTANCE,
instanceFileName,
schemaRefs=[dtsRef["href"] for dtsRef in dtsReferences if dtsRef["type"] == "schema"],
isEntry=True,
initialComment="extracted from OIM {}".format(mappedUri),
documentEncoding="utf-8")
cntxTbl = {}
unitTbl = {}
for fact in facts:
conceptQn = qname(fact["oim:concept"], prefixes)
concept = modelXbrl.qnameConcepts.get(conceptQn)
entityAsQn = qname(fact["oim:entity"], prefixes)
if "oim:period" in fact:
periodStart = fact["oim:period"]["start"]
periodEnd = fact["oim:period"]["end"]
else:
periodStart = fact["oim:periodStart"]
periodEnd = fact["oim:periodEnd"]
cntxKey = ( # hashable context key
("periodType", concept.periodType),
("entity", entityAsQn),
("periodStart", periodStart),
("periodEnd", periodEnd)) + tuple(sorted(
(dimName, dimVal)
for dimName, dimVal in fact.items()
if ":" in dimName and not dimName.startswith("oim:")))
if cntxKey in cntxTbl:
_cntx = cntxTbl[cntxKey]
else:
cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
qnameDims = {}
for dimName, dimVal in fact.items():
if ":" in dimName and not dimName.startswith("oim:"):
dimQname = qname(dimName, prefixes)
dimConcept = modelXbrl.qnameConcepts.get(dimQname)
if ":" in dimVal and dimVal.partition[':'][0] in prefixes:
mem = qname(dimVal, prefixes) # explicit dim
elif dimConcept.isTypedDimension:
# a modelObject xml element is needed for all of the instance functions to manage the typed dim
mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False)
qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment")
_cntx = modelXbrl.createContext(
entityAsQn.namespaceURI,
entityAsQn.localName,
concept.periodType,
None if concept.periodType == "instnat" else dateTime(periodStart, type=DATETIME),
dateTime(periodEnd, type=DATETIME),
None, # no dimensional validity checking (like formula does)
qnameDims, [], [],
id=cntxId)
cntxTbl[cntxKey] = _cntx
if "oim:unit" in fact:
unitKey = fact["oim:unit"]
if unitKey in unitTbl:
_unit = unitTbl[unitKey]
else:
_mul, _sep, _div = unitKey.partition('/')
if _mul.startswith('('):
_mul = _mul[1:-1]
mulQns = [qname(u, prefixes) for u in _mul.split('*') if u]
if _div.startswith('('):
_div = _div[1:-1]
divQns = [qname(u, prefixes) for u in _div.split('*') if u]
unitId = 'u-{:02}'.format(len(unitTbl) + 1)
for _measures in mulQns, divQns:
for _measure in _measures:
addQnameValue(modelXbrl.modelDocument, _measure)
_unit = modelXbrl.createUnit(mulQns, divQns, id=unitId)
unitTbl[unitKey] = _unit
else:
_unit = None
attrs = {"contextRef": _cntx.id}
if fact.get("value") is None:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact["value"]
if fact.get("id"):
attrs["id"] = fact["id"]
if concept.isNumeric:
if _unit is not None:
attrs["unitRef"] = _unit.id
if "accuracy" in fact:
attrs["decimals"] = fact["accuracy"]
# is value a QName?
if concept.baseXbrliType == "QName":
addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes))
f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
footnoteLinks = {} # ELR elements
factLocs = {} # index by (linkrole, factId)
footnoteNbr = 0
locNbr = 0
for factOrFootnote in footnotes:
if "factId" in factOrFootnote:
factId = factOrFootnote["factId"]
factFootnotes = (factOrFootnote,) # CSV or XL
elif "id" in factOrFootnote and "footnotes" in factOrFootnote:
factId = factOrFootnote["id"]
factFootnotes = factOrFootnote["footnotes"]
else:
factFootnotes = ()
for footnote in factFootnotes:
linkrole = footnote.get("group")
arcrole = footnote.get("footnoteType")
if not factId or not linkrole or not arcrole or not (
footnote.get("factRef") or footnote.get("footnote")):
# invalid footnote
continue
if linkrole not in footnoteLinks:
footnoteLinks[linkrole] = addChild(modelXbrl.modelDocument.xmlRootElement,
XbrlConst.qnLinkFootnoteLink,
attributes={"{http://www.w3.org/1999/xlink}type": "extended",
"{http://www.w3.org/1999/xlink}role": linkrole})
footnoteLink = footnoteLinks[linkrole]
if (linkrole, factId) not in factLocs:
locNbr += 1
locLabel = "l_{:02}".format(locNbr)
factLocs[(linkrole, factId)] = locLabel
addChild(footnoteLink, XbrlConst.qnLinkLoc,
attributes={XLINKTYPE: "locator",
XLINKHREF: "#" + factId,
XLINKLABEL: locLabel})
locLabel = factLocs[(linkrole, factId)]
if footnote.get("footnote"):
footnoteNbr += 1
footnoteLabel = "f_{:02}".format(footnoteNbr)
attrs = {XLINKTYPE: "resource",
XLINKLABEL: footnoteLabel}
if footnote.get("language"):
attrs[XMLLANG] = footnote["language"]
# note, for HTML will need to build an element structure
addChild(footnoteLink, XbrlConst.qnLinkFootnote, attributes=attrs, text=footnote["footnote"])
elif footnote.get("factRef"):
factRef = footnote.get("factRef")
if (linkrole, factRef) not in factLocs:
locNbr += 1
locLabel = "f_{:02}".format(footnoteNbr)
factLoc[(linkrole, factRef)] = locLabel
addChild(footnoteLink, XbrlConst.qnLinkLoc,
attributes={XLINKTYPE: "locator",
XLINKHREF: "#" + factRef,
XLINKLABEL: locLabel})
footnoteLabel = factLoc[(linkrole, factId)]
footnoteArc = addChild(footnoteLink,
XbrlConst.qnLinkFootnoteArc,
attributes={XLINKTYPE: "arc",
XLINKARCROLE: arcrole,
XLINKFROM: locLabel,
XLINKTO: footnoteLabel})
#cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt),
# messageCode="loadFromExcel:info")
except Exception as ex:
if ex is OIMException:
modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs)
else:
modelXbrl.error("Error while %(action)s, error %(error)s",
modelObject=modelXbrl, action=currentAction, error=ex)
if priorCWD:
os.chdir(priorCWD) # restore prior current working directory
return modelXbrl.modelDocument
def validateValue(modelXbrl,
elt,
attrTag,
baseXsdType,
value,
isNillable=False,
isNil=False,
facets=None):
if baseXsdType:
try:
'''
if (len(value) == 0 and attrTag is None and not isNillable and
baseXsdType not in ("anyType", "string", "normalizedString", "token", "NMTOKEN", "anyURI", "noContent")):
raise ValueError("missing value for not nillable element")
'''
xValid = VALID
whitespaceReplace = (baseXsdType == "normalizedString")
whitespaceCollapse = (not whitespaceReplace
and baseXsdType != "string")
pattern = baseXsdTypePatterns.get(baseXsdType)
if facets:
if "pattern" in facets:
pattern = facets["pattern"]
# note multiple patterns are or'ed togetner, which isn't yet implemented!
if "whiteSpace" in facets:
whitespaceReplace, whitespaceCollapse = {
"preserve": (False, False),
"replace": (True, False),
"collapse": (False, True)
}[facets["whiteSpace"]]
if whitespaceReplace:
value = normalizeWhitespacePattern.sub(' ', value)
elif whitespaceCollapse:
value = collapseWhitespacePattern.sub(' ', value.strip())
if pattern is not None and pattern.match(value) is None:
raise ValueError("pattern facet " +
facets["pattern"].pattern if facets and
"pattern" in facets else "pattern mismatch")
if facets:
if "enumeration" in facets and value not in facets[
"enumeration"]:
raise ValueError("{0} is not in {1}".format(
value, facets["enumeration"]))
if "length" in facets and len(value) != facets["length"]:
raise ValueError("length {0}, expected {1}".format(
len(value), facets["length"]))
if "minLength" in facets and len(value) < facets["minLength"]:
raise ValueError("length {0}, minLength {1}".format(
len(value), facets["minLength"]))
if "maxLength" in facets and len(value) > facets["maxLength"]:
raise ValueError("length {0}, maxLength {1}".format(
len(value), facets["maxLength"]))
if baseXsdType == "noContent":
if len(value) > 0 and not value.isspace():
raise ValueError("value content not permitted")
xValue = sValue = None
elif baseXsdType in {
"string", "normalizedString", "language", "token",
"NMTOKEN", "Name", "NCName", "IDREF", "ENTITY"
}:
xValue = sValue = value
elif baseXsdType == "ID":
xValue = sValue = value
xValid = VALID_ID
elif baseXsdType == "anyURI":
if value: # allow empty strings to be valid anyURIs
if UrlUtil.relativeUrlPattern.match(value) is None:
raise ValueError("IETF RFC 2396 4.3 syntax")
# encode PSVI xValue similarly to Xerces and other implementations
xValue = anyURI(UrlUtil.anyUriQuoteForPSVI(value))
sValue = value
elif not value and isNil and isNillable: # rest of types get None if nil/empty value
xValue = sValue = None
elif baseXsdType in ("decimal", "float", "double"):
xValue = sValue = float(value)
if facets:
if "totalDigits" in facets and len(value.replace(
".", "")) > facets["totalDigits"]:
raise ValueError("totalDigits facet {0}".format(
facets["totalDigits"]))
if "fractionDigits" in facets and (
'.' in value and len(value[value.index('.') + 1:])
> facets["fractionDigits"]):
raise ValueError("fraction digits facet {0}".format(
facets["fractionDigits"]))
if "maxInclusive" in facets and xValue > facets[
"maxInclusive"]:
raise ValueError(" > maxInclusive {0}".format(
facets["maxInclusive"]))
if "maxExclusive" in facets and xValue >= facets[
"maxExclusive"]:
raise ValueError(" >= maxInclusive {0}".format(
facets["maxExclusive"]))
if "minInclusive" in facets and xValue < facets[
"minInclusive"]:
raise ValueError(" < minInclusive {0}".format(
facets["minInclusive"]))
if "minExclusive" in facets and xValue <= facets[
"minExclusive"]:
raise ValueError(" <= minExclusive {0}".format(
facets["minExclusive"]))
elif baseXsdType in {
"integer", "nonPositiveInteger", "negativeInteger",
"nonNegativeInteger", "positiveInteger", "long",
"unsignedLong", "int", "unsignedInt", "short",
"unsignedShort", "byte", "unsignedByte"
}:
xValue = sValue = _INT(value)
if ((baseXsdType
in {"nonNegativeInteger", "unsignedLong", "unsignedInt"}
and xValue < 0)
or (baseXsdType == "nonPositiveInteger" and xValue > 0)
or (baseXsdType == "positiveInteger" and xValue <= 0)
or (baseXsdType == "byte" and not -128 <= xValue < 127)
or
(baseXsdType == "unsignedByte" and not 0 <= xValue < 255)
or
(baseXsdType == "short" and not -32768 <= xValue < 32767)
or (baseXsdType == "unsignedShort"
and not 0 <= xValue < 65535)
or (baseXsdType == "positiveInteger" and xValue <= 0)):
raise ValueError("{0} is not {1}".format(
value, baseXsdType))
if facets:
if "totalDigits" in facets and len(value.replace(
".", "")) > facets["totalDigits"]:
raise ValueError("totalDigits facet {0}".format(
facets["totalDigits"]))
if "fractionDigits" in facets and (
'.' in value and len(value[value.index('.') + 1:])
> facets["fractionDigits"]):
raise ValueError("fraction digits facet {0}".format(
facets["fractionDigits"]))
if "maxInclusive" in facets and xValue > facets[
"maxInclusive"]:
raise ValueError(" > maxInclusive {0}".format(
facets["maxInclusive"]))
if "maxExclusive" in facets and xValue >= facets[
"maxExclusive"]:
raise ValueError(" >= maxInclusive {0}".format(
facets["maxExclusive"]))
if "minInclusive" in facets and xValue < facets[
"minInclusive"]:
raise ValueError(" < minInclusive {0}".format(
facets["minInclusive"]))
if "minExclusive" in facets and xValue <= facets[
"minExclusive"]:
raise ValueError(" <= minExclusive {0}".format(
facets["minExclusive"]))
elif baseXsdType == "boolean":
if value in ("true", "1"):
xValue = sValue = True
elif value in ("false", "0"):
xValue = sValue = False
else:
raise ValueError
elif baseXsdType == "QName":
xValue = qname(elt,
value,
castException=ValueError,
prefixException=ValueError)
sValue = value
''' not sure here, how are explicitDimensions validated, but bad units not?
if xValue.namespaceURI in modelXbrl.namespaceDocs:
if (xValue not in modelXbrl.qnameConcepts and
xValue not in modelXbrl.qnameTypes and
xValue not in modelXbrl.qnameAttributes and
xValue not in modelXbrl.qnameAttributeGroups):
raise ValueError("qname not defined " + str(xValue))
'''
elif baseXsdType in ("XBRLI_DECIMALSUNION",
"XBRLI_PRECISIONUNION"):
xValue = sValue = value if value == "INF" else _INT(value)
elif baseXsdType in ("XBRLI_NONZERODECIMAL"):
xValue = sValue = _INT(value)
if xValue == 0:
raise ValueError("invalid value")
elif baseXsdType == "XBRLI_DATEUNION":
xValue = dateTime(value,
type=DATEUNION,
castException=ValueError)
sValue = value
elif baseXsdType == "dateTime":
xValue = dateTime(value,
type=DATETIME,
castException=ValueError)
sValue = value
elif baseXsdType == "date":
xValue = dateTime(value, type=DATE, castException=ValueError)
sValue = value
elif baseXsdType == "regex-pattern":
# for facet compiling
try:
sValue = value
if value in xmlSchemaPatterns:
xValue = xmlSchemaPatterns[value]
else:
if r"\i" in value or r"\c" in value:
value = value.replace(r"\i", iNameChar).replace(
r"\c", cNameChar)
xValue = re.compile(value +
"$") # must match whole string
except Exception as err:
raise ValueError(err)
else:
if baseXsdType in lexicalPatterns:
match = lexicalPatterns[baseXsdType].match(value)
if match is None:
raise ValueError("lexical pattern mismatch")
if baseXsdType == "gMonthDay":
month, day, zSign, zHrMin, zHr, zMin = match.groups()
if int(day) > {
2: 29,
4: 30,
6: 30,
9: 30,
11: 30,
1: 31,
3: 31,
5: 31,
7: 31,
8: 31,
10: 31,
12: 31
}[int(month)]:
raise ValueError(
"invalid day {0} for month {1}".format(
day, month))
xValue = value
sValue = value
except ValueError as err:
if ModelInlineFact is not None and isinstance(
elt, ModelInlineFact):
errElt = "{0} fact {1}".format(elt.elementQname, elt.qname)
else:
errElt = elt.elementQname
if attrTag:
modelXbrl.error(
"xmlSchema:valueError",
_("Element %(element)s attribute %(attribute)s type %(typeName)s value error: %(value)s, %(error)s"
),
modelObject=elt,
element=errElt,
attribute=XmlUtil.clarkNotationToPrefixedName(
elt, attrTag, isAttribute=True),
typeName=baseXsdType,
value=value,
error=err)
else:
modelXbrl.error(
"xmlSchema:valueError",
_("Element %(element)s type %(typeName)s value error: %(value)s, %(error)s"
),
modelObject=elt,
element=errElt,
typeName=baseXsdType,
value=value,
error=err)
xValue = None
sValue = value
xValid = INVALID
else:
xValue = sValue = None
xValid = UNKNOWN
if attrTag:
try: # dynamically allocate attributes (otherwise given shared empty set)
xAttributes = elt.xAttributes
except AttributeError:
elt.xAttributes = xAttributes = {}
xAttributes[attrTag] = ModelAttribute(elt, attrTag, xValid, xValue,
sValue, value)
else:
elt.xValid = xValid
elt.xValue = xValue
elt.sValue = sValue
def atomize(self, p, x):
# sequence
if isinstance(x, (tuple, list, set)):
sequence = []
for item in self.flattenSequence(x):
atomizedItem = self.atomize(p, item)
if atomizedItem != []:
sequence.append(atomizedItem)
return sequence
# individual items
if isinstance(x, _RANGE):
return x
baseXsdType = None
e = None
if isinstance(x, ModelFact):
if x.isTuple:
raise XPathException(
p, 'err:FOTY0012',
_('Atomizing tuple {0} that does not have a typed value').
format(x))
if x.isNil:
return []
baseXsdType = x.concept.baseXsdType
v = x.value # resolves default value
e = x
elif isinstance(
x, ModelAttribute
): # ModelAttribute is a tuple (below), check this first!
return x.xValue
else:
if isinstance(x, ModelObject):
e = x
if e is not None:
if e.get("{http://www.w3.org/2001/XMLSchema-instance}nil"
) == "true":
return []
try:
if e.xValid >= VALID:
return e.xValue
except AttributeError:
pass
modelXbrl = x.modelXbrl
modelConcept = modelXbrl.qnameConcepts.get(qname(x))
if modelConcept is not None:
baseXsdType = modelConcept.baseXsdType
v = XmlUtil.text(x)
if baseXsdType in ("decimal", "float", "double"):
try:
x = float(v)
except ValueError:
raise XPathException(
p, 'err:FORG0001',
_('Atomizing {0} to a {1} does not have a proper value').
format(x, baseXsdType))
elif baseXsdType in ("integer", ):
try:
x = _INT(v)
except ValueError:
raise XPathException(
p, 'err:FORG0001',
_('Atomizing {0} to an integer does not have a proper value'
).format(x))
elif baseXsdType == "boolean":
x = (v == "true" or v == "1")
elif baseXsdType == "QName" and e is not None:
x = qname(e, v)
elif baseXsdType == "anyURI":
x = anyURI(v.strip())
elif baseXsdType in ("normalizedString", "token", "language",
"NMTOKEN", "Name", "NCName", "ID", "IDREF",
"ENTITY"):
x = v.strip()
elif baseXsdType == "XBRLI_DATEUNION":
x = dateTime(v, type=DATEUNION)
elif baseXsdType == "date":
x = dateTime(v, type=DATE)
elif baseXsdType == "dateTime":
x = dateTime(v, type=DATETIME)
elif baseXsdType:
x = str(v)
return x
def insertDataPoints(self):
instanceId = self.instanceId
self.showStatus("deleting prior data points of this report")
for tableName in ("dFact", "dFilingIndicator", "dProcessingContext", "dProcessingFact"):
self.execute("DELETE FROM {0} WHERE {0}.InstanceID = {1}"
.format( self.dbTableName(tableName), instanceId),
close=False, fetch=False)
self.showStatus("insert data points")
# contexts
def dimValKey(cntx, typedDim=False):
return '|'.join(sorted("{}({})".format(dim.dimensionQname,
dim.memberQname if dim.isExplicit
else xmlstring(dim.typedMember, stripXmlns=True) if typedDim
else '*' )
for dim in cntx.qnameDims.values()))
def dimNameKey(cntx):
return '|'.join(sorted("{}".format(dim.dimensionQname)
for dim in cntx.qnameDims.values()))
contextSortedAllDims = dict((cntx.id, dimValKey(cntx)) # has (*) for typed dim
for cntx in self.modelXbrl.contexts.values()
if cntx.qnameDims)
contextSortedTypedDims = dict((cntx.id, dimValKey(cntx, typedDim=True)) # typed dims with value only if typed
for cntx in self.modelXbrl.contexts.values()
if any(dim.isTyped for dim in cntx.qnameDims.values()))
contextSortedDimNames = dict((cntx.id, dimNameKey(cntx))
for cntx in self.modelXbrl.contexts.values()
if cntx.qnameDims)
def met(fact):
return "MET({})".format(fact.qname)
def metDimAllKey(fact):
key = met(fact)
if fact.contextID in contextSortedAllDims:
key += '|' + contextSortedAllDims[fact.contextID]
return key
def metDimTypedKey(fact):
if fact.contextID in contextSortedTypedDims:
key = met(fact)
if fact.contextID in contextSortedTypedDims:
key += '|' + contextSortedTypedDims[fact.contextID]
return key
return None
def metDimNameKey(fact):
key = met(fact)
if fact.contextID in contextSortedDimNames:
key += '|' + contextSortedDimNames[fact.contextID]
return key
table = self.getTable("dProcessingContext", None,
('InstanceID', 'ContextID', 'SortedDimensions', 'NotValid'),
('InstanceID', 'ContextID'),
tuple((instanceId,
cntxID,
cntxDimKey,
False
)
for cntxID, cntxDimKey in sorted(contextSortedAllDims.items())))
# contexts with typed dimensions
# dCloseFactTable
dFilingIndicators = set()
dProcessingFacts = []
dFacts = []
for f in self.modelXbrl.factsInInstance:
cntx = f.context
concept = f.concept
isNumeric = isBool = isDateTime = isText = False
if concept is not None:
if concept.isNumeric:
isNumeric = True
else:
baseXbrliType = concept.baseXbrliType
if baseXbrliType == "booleanItemType":
isBool = True
elif baseXbrliType == "dateTimeItemType": # also is dateItemType?
isDateTime = True
xValue = f.xValue
else:
if f.isNil:
xValue = None
else:
xValue = f.value
c = f.qname.localName[0]
if c == 'm':
isNumeric = True
# not validated, do own xValue
try:
xValue = Decimal(xValue)
except InvalidOperation:
xValue = Decimal('NaN')
elif c == 'd':
isDateTime = True
try:
xValue = dateTime(xValue, type=DATEUNION, castException=ValueError)
except ValueError:
pass
elif c == 'b':
isBool = True
xValue = xValue.strip()
if xValue in ("true", "1"):
xValue = True
elif xValue in ("false", "0"):
xValue = False
isText = not (isNumeric or isBool or isDateTime)
if f.qname == qnFindFilingIndicators:
for filingIndicator in f.modelTupleFacts:
if filingIndicator.qname == qnFindFilingIndicator:
dFilingIndicators.add(filingIndicator.textValue.strip())
elif cntx is not None:
dFacts.append((instanceId,
metDimAllKey(f),
metDimTypedKey(f),
str(f.unit.measures[0][0]) if isNumeric and f.unit is not None and f.unit.isSingleMeasure else None,
f.decimals,
xValue if isNumeric else None,
xValue if isDateTime else None,
xValue if isBool else None,
xValue if isText else None
))
dProcessingFacts.append((instanceId,
met(f),
f.contextID if isNumeric else None,
xValue if isText or isBool else None,
xValue if isNumeric else None,
xValue if isDateTime else None,
None))
# get filing indicator template IDs
results = self.execute("SELECT mToT.TemplateOrTableCode, mToT.TemplateOrTableID "
" FROM mModuleBusinessTemplate mBT, mTemplateOrTable mToT "
" WHERE mBT.ModuleID = {0} AND"
" mToT.TemplateOrTableID = mBT.BusinessTemplateID AND"
" mToT.TemplateOrTableCode in ({1})"
.format(self.moduleId,
', '.join("'{}'".format(filingIndicator)
for filingIndicator in dFilingIndicators)))
filingIndicatorCodeIDs = dict((code, id) for code, id in results)
if _DICT_SET(filingIndicatorCodeIDs.keys()) != dFilingIndicators:
self.modelXbrl.error("sqlDB:MissingFilingIndicators",
_("The filing indicator IDs not found for codes %(missingFilingIndicatorCodes)"),
modelObject=self.modelXbrl,
missingFilingIndicatorCodes=','.join(dFilingIndicators - _DICT_SET(filingIndicatorCodeIDs.keys())))
table = self.getTable("dFilingIndicator", None,
("InstanceID", "BusinessTemplateID"),
("InstanceID", "BusinessTemplateID"),
((instanceId,
filingIndicatorCodeId)
for filingIndicatorCodeId in sorted(filingIndicatorCodeIDs.values())))
table = self.getTable("dFact", None,
('InstanceID', 'DataPointSignature', 'DataPointSignatureWithValuesForWildcards',
'Unit', 'Decimals',
'NumericValue', 'DateTimeValue', 'BooleanValue', 'TextValue'),
('InstanceID', ),
dFacts)
table = self.getTable("dProcessingFact", None,
('InstanceID', 'Metric', 'ContextID',
'ValueTxt', 'ValueDecimal', 'ValueDate',
'Error'),
('InstanceID', ),
dProcessingFacts)
def insertDataPoints(self):
instanceId = self.instanceId
self.showStatus("deleting prior data points of this report")
for tableName in ("dFact", "dFilingIndicator", "dAvailableTable"):
self.execute("DELETE FROM {0} WHERE {0}.InstanceID = {1}"
.format( self.dbTableName(tableName), instanceId),
close=False, fetch=False)
self.showStatus("obtaining mapping table information")
result = self.execute("SELECT MetricAndDimensions, TableID FROM mTableDimensionSet WHERE ModuleID = {}"
.format(self.moduleId))
tableIDs = set()
metricAndDimensionsTableId = defaultdict(set)
for metricAndDimensions, tableID in result:
tableIDs.add(tableID)
metricAndDimensionsTableId[metricAndDimensions].add(tableID)
result = self.execute("SELECT TableID, YDimVal, ZDimVal FROM mTable WHERE TableID in ({})"
.format(', '.join(str(i) for i in sorted(tableIDs))))
yDimVal = defaultdict(dict)
zDimVal = defaultdict(dict)
for tableID, yDimVals, zDimVals in result:
for tblDimVal, dimVals in ((yDimVal, yDimVals), (zDimVal, zDimVals)):
if dimVals:
for dimVal in dimVals.split('|'):
dim, sep, val = dimVal.partition('(')
tblDimVal[tableID][dim] = val[:-1]
availableTableRows = defaultdict(set) # index (tableID, zDimVal) = set of yDimVals
self.showStatus("insert data points")
# contexts
emptySet = set()
def dimValKey(cntx, typedDim=False, behaveAsTypedDims=emptySet, restrictToDims=None):
return '|'.join(sorted("{}({})".format(dim.dimensionQname,
dim.memberQname if dim.isExplicit and dim not in behaveAsTypedDims
else dim.memberQname if typedDim and not dim.isTyped
else xmlstring(dim.typedMember, stripXmlns=True) if typedDim
else '*' )
for dim in cntx.qnameDims.values()
if not restrictToDims or str(dim.dimensionQname) in restrictToDims))
def dimNameKey(cntx):
return '|'.join(sorted("{}".format(dim.dimensionQname)
for dim in cntx.qnameDims.values()))
contextSortedAllDims = dict((cntx.id, dimValKey(cntx)) # has (*) for typed dim
for cntx in self.modelXbrl.contexts.values()
if cntx.qnameDims)
contextSortedTypedDims = dict((cntx.id, dimValKey(cntx, typedDim=True)) # typed dims with value only if typed
for cntx in self.modelXbrl.contexts.values()
if any(dim.isTyped for dim in cntx.qnameDims.values()))
contextSortedDimNames = dict((cntx.id, dimNameKey(cntx))
for cntx in self.modelXbrl.contexts.values()
if cntx.qnameDims)
def met(fact):
return "MET({})".format(fact.qname)
# key for use in dFact with * for dim that behaves as or is typed
def metDimAllKey(fact, behaveAsTypedDims=emptySet):
key = met(fact)
cntx = fact.context
if cntx.qnameDims:
key += '|' + dimValKey(cntx, behaveAsTypedDims=behaveAsTypedDims)
return key
# key for use in dFact only when there's a dim that behaves as or is typed
def metDimTypedKey(fact, behaveAsTypedDims=emptySet):
cntx = fact.context
if any(dimQname in behaveAsTypedDims for dimQname in cntx.qnameDims):
key = met(fact) + '|' + dimValKey(cntx, typedDim=True, behaveAsTypedDims=behaveAsTypedDims)
return key
return None
# key for use in dAvailable where mem and typed values show up
def metDimValKey(cntx, typedDim=False, behaveAsTypedDims=emptySet, restrictToDims=emptySet):
if "MET" in restrictToDims:
key = "MET({})|".format(restrictToDims["MET"])
else:
key = ""
key += dimValKey(cntx, typedDim=typedDim, behaveAsTypedDims=behaveAsTypedDims, restrictToDims=restrictToDims)
return key
def metDimNameKey(fact):
key = met(fact)
if fact.contextID in contextSortedDimNames:
key += '|' + contextSortedDimNames[fact.contextID]
return key
''' deprecated
table = self.getTable("dProcessingContext", None,
('InstanceID', 'ContextID', 'SortedDimensions', 'NotValid'),
('InstanceID', 'ContextID'),
tuple((instanceId,
cntxID,
cntxDimKey,
False
)
for cntxID, cntxDimKey in sorted(contextSortedAllDims.items())))
'''
# contexts with typed dimensions
# dCloseFactTable
dFilingIndicators = set()
# dProcessingFacts = []
dFacts = []
for f in self.modelXbrl.facts:
cntx = f.context
concept = f.concept
isNumeric = isBool = isDateTime = isText = False
if concept is not None:
if concept.isNumeric:
isNumeric = True
else:
baseXbrliType = concept.baseXbrliType
if baseXbrliType == "booleanItemType":
isBool = True
elif baseXbrliType == "dateTimeItemType": # also is dateItemType?
isDateTime = True
xValue = f.xValue
else:
if f.isNil:
xValue = None
else:
xValue = f.value
c = f.qname.localName[0]
if c == 'm':
isNumeric = True
# not validated, do own xValue
try:
xValue = Decimal(xValue)
except InvalidOperation:
xValue = Decimal('NaN')
elif c == 'd':
isDateTime = True
try:
xValue = dateTime(xValue, type=DATEUNION, castException=ValueError)
except ValueError:
pass
elif c == 'b':
isBool = True
xValue = xValue.strip()
if xValue in ("true", "1"):
xValue = True
elif xValue in ("false", "0"):
xValue = False
isText = not (isNumeric or isBool or isDateTime)
if f.qname == qnFindFilingIndicators:
for filingIndicator in f.modelTupleFacts:
if filingIndicator.qname == qnFindFilingIndicator:
dFilingIndicators.add(filingIndicator.textValue.strip())
elif cntx is not None:
# find which explicit dimensions act as typed
behaveAsTypedDims = set()
zDimValues = {}
tableID = None
for tableID in metricAndDimensionsTableId.get(metDimNameKey(f), ()):
yDimVals = yDimVal[tableID]
zDimVals = zDimVal[tableID]
for dimQname in cntx.qnameDims.keys():
dimStr = str(dimQname)
if (dimStr in zDimVals and zDimVals[dimStr] == "*" or
dimStr in yDimVals and yDimVals[dimStr] == "*"):
behaveAsTypedDims.add(dimQname)
zDimKey = (metDimValKey(cntx, typedDim=True, behaveAsTypedDims=behaveAsTypedDims, restrictToDims=zDimVals)
or None) # want None if no dimVal Z key
yDimKey = metDimValKey(cntx, typedDim=True, behaveAsTypedDims=behaveAsTypedDims, restrictToDims=yDimVals)
availableTableRows[tableID,zDimKey].add(yDimKey)
break
dFacts.append((instanceId,
metDimAllKey(f, behaveAsTypedDims),
metDimTypedKey(f, behaveAsTypedDims),
str(f.unit.measures[0][0]) if isNumeric and f.unit is not None and f.unit.isSingleMeasure else None,
f.decimals,
xValue if isNumeric else None,
xValue if isDateTime else None,
xValue if isBool else None,
xValue if isText else None
))
''' deprecated
dProcessingFacts.append((instanceId,
met(f),
f.contextID if isNumeric else None,
xValue if isText or isBool else None,
xValue if isNumeric else None,
xValue if isDateTime else None,
None))
'''
# availableTable processing
# get filing indicator template IDs
results = self.execute("SELECT mToT.TemplateOrTableCode, mToT.TemplateOrTableID "
" FROM mModuleBusinessTemplate mBT, mTemplateOrTable mToT "
" WHERE mBT.ModuleID = {0} AND"
" mToT.TemplateOrTableID = mBT.BusinessTemplateID AND"
" mToT.TemplateOrTableCode in ({1})"
.format(self.moduleId,
', '.join("'{}'".format(filingIndicator)
for filingIndicator in dFilingIndicators)))
filingIndicatorCodeIDs = dict((code, id) for code, id in results)
if _DICT_SET(filingIndicatorCodeIDs.keys()) != dFilingIndicators:
self.modelXbrl.error("sqlDB:MissingFilingIndicators",
_("The filing indicator IDs not found for codes %(missingFilingIndicatorCodes)"),
modelObject=self.modelXbrl,
missingFilingIndicatorCodes=','.join(dFilingIndicators - _DICT_SET(filingIndicatorCodeIDs.keys())))
self.getTable("dFilingIndicator", None,
("InstanceID", "BusinessTemplateID"),
("InstanceID", "BusinessTemplateID"),
((instanceId,
filingIndicatorCodeId)
for filingIndicatorCodeId in sorted(filingIndicatorCodeIDs.values())))
self.getTable("dFact", None,
('InstanceID', 'DataPointSignature', 'DataPointSignatureWithValuesForWildcards',
'Unit', 'Decimals',
'NumericValue', 'DateTimeValue', 'BooleanValue', 'TextValue'),
('InstanceID', ),
dFacts)
''' deprecated
table = self.getTable("dProcessingFact", None,
('InstanceID', 'Metric', 'ContextID',
'ValueTxt', 'ValueDecimal', 'ValueDate',
'Error'),
('InstanceID', ),
dProcessingFacts)
'''
self.getTable("dAvailableTable", None,
('InstanceID', 'TableID', 'ZDimVal', "NumberOfRows"),
('InstanceID', 'TableID', 'ZDimVal'),
((instanceId,
availTableKey[0], # table Id
availTableKey[1], # zDimVal
len(setOfYDimVals))
for availTableKey, setOfYDimVals in availableTableRows.items()))
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri):
from openpyxl import load_workbook
from arelle import ModelDocument, ModelXbrl, XmlUtil
from arelle.ModelDocument import ModelDocumentReference
from arelle.ModelValue import qname
try:
currentAction = "initializing"
startingErrorCount = len(modelXbrl.errors)
startedAt = time.time()
if os.path.isabs(oimFile):
# allow relative filenames to loading directory
priorCWD = os.getcwd()
os.chdir(os.path.dirname(oimFile))
else:
priorCWD = None
currentAction = "determining file type"
isJSON = oimFile.endswith(
".json") and not oimFile.endswith("-metadata.json")
isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json")
isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls")
isCSVorXL = isCSV or isXL
instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl"
if isJSON:
currentAction = "loading and parsing JSON OIM file"
def loadDict(keyValuePairs):
_dict = OrderedDict(
) # preserve fact order in resulting instance
for key, value in keyValuePairs:
if isinstance(value, dict):
if DUPJSONKEY in value:
for _errKey, _errValue, _otherValue in value[
DUPJSONKEY]:
if key == "prefixes":
modelXbrl.error(
"oime:duplicatedPrefix",
_("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"
),
modelObject=modelXbrl,
prefix=_errKey,
uri1=_errValue,
uri2=_otherValue)
del value[DUPJSONKEY]
if key in _dict:
if DUPJSONKEY not in _dict:
_dict[DUPJSONKEY] = []
_dict[DUPJSONKEY].append((key, value, _dict[key]))
else:
_dict[key] = value
return _dict
with io.open(oimFile, 'rt', encoding='utf-8') as f:
oimObject = json.load(f, object_pairs_hook=loadDict)
missing = [
t for t in ("dtsReferences", "prefixes", "facts")
if t not in oimObject
]
if missing:
raise OIMException(
"oime:missingJSONElements",
_("Required element(s) are missing from JSON input: %(missing)s"
),
missing=", ".join(missing))
currentAction = "identifying JSON objects"
dtsReferences = oimObject["dtsReferences"]
prefixesList = oimObject["prefixes"].items()
facts = oimObject["facts"]
footnotes = oimObject["facts"] # shares this object
elif isCSV:
currentAction = "identifying CSV input tables"
if sys.version[0] >= '3':
csvOpenMode = 'w'
csvOpenNewline = ''
else:
csvOpenMode = 'wb' # for 2.7
csvOpenNewline = None
oimFileBase = None
if "-facts" in oimFile:
oimFileBase = oimFile.partition("-facts")[0]
else:
for suffix in ("-dtsReferences.csv", "-defaults.csv",
"-prefixes.csv", "-footnotes.csv",
"-metadata.json"):
if oimFile.endswith(suffix):
oimFileBase = oimFile[:-len(suffix)]
break
if oimFileBase is None:
raise OIMException(
"oime:missingCSVTables",
_("Unable to identify CSV tables file name pattern"))
if (not os.path.exists(oimFileBase + "-dtsReferences.csv")
or not os.path.exists(oimFileBase + "-prefixes.csv")):
raise OIMException(
"oime:missingCSVTables",
_("Unable to identify CSV tables for dtsReferences or prefixes"
))
instanceFileName = oimFileBase + ".xbrl"
currentAction = "loading CSV dtsReferences table"
dtsReferences = []
with io.open(oimFileBase + "-dtsReferences.csv",
'rt',
encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
else:
dtsReferences.append(
dict(
(header[j], col) for j, col in enumerate(row)))
currentAction = "loading CSV prefixes table"
prefixesList = []
with io.open(oimFileBase + "-prefixes.csv",
'rt',
encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = dict((col, i) for i, col in enumerate(row))
else:
prefixesList.append(
(row[header["prefix"]], row[header["URI"]]))
defaults = {}
if os.path.exists(oimFileBase + "-defaults.csv"):
currentAction = "loading CSV defaults table"
with io.open(oimFileBase + "-defaults.csv",
'rt',
encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
fileCol = row.index("file")
else:
defaults[row[fileCol]] = dict(
(header[j], col) for j, col in enumerate(row)
if j != fileCol)
currentAction = "loading CSV facts tables"
facts = []
_dir = os.path.dirname(oimFileBase)
factsFileBasename = os.path.basename(oimFileBase) + "-facts"
for filename in os.listdir(_dir):
filepath = os.path.join(_dir, filename)
if filename.startswith(factsFileBasename):
currentAction = "loading CSV facts table {}".format(
filename)
tableDefaults = defaults.get(filename, {})
with io.open(filepath, 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
else:
fact = {}
fact.update(tableDefaults)
for j, col in enumerate(row):
if col is not None:
if header[
j]: # skip cols with no header
if header[j].endswith("Value"):
if col: # ignore empty columns (= null CSV value)
fact["value"] = col
else:
fact[header[j]] = col
facts.append(fact)
footnotes = []
if os.path.exists(oimFileBase + "-footnotes.csv"):
currentAction = "loading CSV footnotes table"
with io.open(oimFileBase + "-footnotes.csv",
'rt',
encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
else:
footnotes.append(
dict((header[j], col)
for j, col in enumerate(row) if col))
elif isXL:
currentAction = "identifying workbook input worksheets"
oimWb = load_workbook(oimFile, read_only=True, data_only=True)
sheetNames = oimWb.get_sheet_names()
if (not any(sheetName == "prefixes" for sheetName in sheetNames)
or not any(sheetName == "dtsReferences"
for sheetName in sheetNames)
or not any("facts" in sheetName
for sheetName in sheetNames)):
raise OIMException(
"oime:missingWorkbookWorksheets",
_("Unable to identify worksheet tabs for dtsReferences, prefixes or facts"
))
currentAction = "loading worksheet: dtsReferences"
dtsReferences = []
for i, row in enumerate(oimWb["dtsReferences"]):
if i == 0:
header = [col.value for col in row]
else:
dtsReferences.append(
dict((header[j], col.value)
for j, col in enumerate(row)))
currentAction = "loading worksheet: prefixes"
prefixesList = []
for i, row in enumerate(oimWb["prefixes"]):
if i == 0:
header = dict((col.value, i) for i, col in enumerate(row))
else:
prefixesList.append((row[header["prefix"]].value,
row[header["URI"]].value))
defaults = {}
if "defaults" in sheetNames:
currentAction = "loading worksheet: defaults"
for i, row in enumerate(oimWb["defaults"]):
if i == 0:
header = dict(
(col.value, i) for i, col in enumerate(row))
fileCol = header["file"]
else:
defaults[row[fileCol].value] = dict(
(header[j], col.value) for j, col in enumerate(row)
if j != fileCol)
facts = []
for sheetName in sheetNames:
if sheetName == "facts" or "-facts" in sheetName:
currentAction = "loading worksheet: {}".format(sheetName)
tableDefaults = defaults.get(sheetName, {})
for i, row in enumerate(oimWb[sheetName]):
if i == 0:
header = [col.value for col in row]
else:
fact = {}
fact.update(tableDefaults)
for j, col in enumerate(row):
if col.value is not None:
if header[j]: # skip cols with no header
if header[j].endswith("Value"):
fact["value"] = str(col.value)
else:
fact[header[j]] = str(col.value)
facts.append(fact)
footnotes = []
if "footnotes" in sheetNames:
currentAction = "loading worksheet: footnotes"
for i, row in enumerate(oimWb["footnotes"]):
if i == 0:
header = dict((col.value, i)
for i, col in enumerate(row)
if col.value)
else:
footnotes.append(
dict((header[j], col.value)
for j, col in enumerate(row) if col.value))
currentAction = "identifying default dimensions"
ValidateXbrlDimensions.loadDimensionDefaults(
modelXbrl) # needs dimension defaults
currentAction = "validating OIM"
prefixes = {}
prefixedUris = {}
for _prefix, _uri in prefixesList:
if not _prefix:
modelXbrl.error(
"oime:emptyPrefix",
_("The empty string must not be used as a prefix, uri %(uri)s"
),
modelObject=modelXbrl,
uri=_uri)
elif not NCNamePattern.match(_prefix):
modelXbrl.error(
"oime:prefixPattern",
_("The prefix %(prefix)s must match the NCName lexical pattern, uri %(uri)s"
),
modelObject=modelXbrl,
prefix=_prefix,
uri=_uri)
elif _prefix in prefixes:
modelXbrl.error(
"oime:duplicatedPrefix",
_("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"
),
modelObject=modelXbrl,
prefix=_prefix,
uri1=prefixes[_prefix],
uri2=_uri)
elif _uri in prefixedUris:
modelXbrl.error(
"oime:duplicatedUri",
_("The uri %(uri)s is used on prefix %(prefix1)s and prefix %(prefix2)s"
),
modelObject=modelXbrl,
uri=_uri,
prefix1=prefixedUris[_uri],
prefix2=_prefix)
else:
prefixes[_prefix] = _uri
prefixedUris[_uri] = _prefix
oimPrefix = None
for _nsOim in nsOim:
if _nsOim in prefixedUris:
oimPrefix = prefixedUris[_nsOim]
if not oimPrefix:
raise OIMException(
"oime:noOimPrefix",
_("The oim namespace must have a declared prefix"))
oimConcept = "{}:concept".format(oimPrefix)
oimEntity = "{}:entity".format(oimPrefix)
oimPeriod = "{}:period".format(oimPrefix)
oimPeriodStart = "{}:periodStart".format(oimPrefix)
oimPeriodEnd = "{}:periodEnd".format(oimPrefix)
oimUnit = "{}:unit".format(oimPrefix)
oimPrefix = "{}:".format(oimPrefix)
# create the instance document
currentAction = "creating instance document"
modelXbrl.blockDpmDBrecursion = True
modelXbrl.modelDocument = createModelDocument(
modelXbrl,
Type.INSTANCE,
instanceFileName,
schemaRefs=[
dtsRef["href"] for dtsRef in dtsReferences
if dtsRef["type"] == "schema"
],
isEntry=True,
initialComment="extracted from OIM {}".format(mappedUri),
documentEncoding="utf-8")
cntxTbl = {}
unitTbl = {}
currentAction = "creating facts"
for fact in facts:
conceptQn = qname(fact[oimConcept], prefixes)
concept = modelXbrl.qnameConcepts.get(conceptQn)
entityAsQn = qname(fact[oimEntity], prefixes)
if oimPeriod in fact:
periodStart = fact[oimPeriod]["start"]
periodEnd = fact[oimPeriod]["end"]
else:
periodStart = fact[oimPeriodStart]
periodEnd = fact[oimPeriodEnd]
cntxKey = ( # hashable context key
("periodType", concept.periodType), ("entity", entityAsQn),
("periodStart", periodStart),
("periodEnd", periodEnd)) + tuple(
sorted(
(dimName, dimVal)
for dimName, dimVal in fact.items() if ":" in dimName
and not dimName.startswith(oimPrefix)))
if cntxKey in cntxTbl:
_cntx = cntxTbl[cntxKey]
else:
cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
qnameDims = {}
for dimName, dimVal in fact.items():
if ":" in dimName and not dimName.startswith(
oimPrefix) and dimVal:
dimQname = qname(dimName, prefixes)
dimConcept = modelXbrl.qnameConcepts.get(dimQname)
if ":" in dimVal and dimVal.partition(
':')[0] in prefixes:
mem = qname(dimVal, prefixes) # explicit dim
elif dimConcept.isTypedDimension:
# a modelObject xml element is needed for all of the instance functions to manage the typed dim
mem = addChild(modelXbrl.modelDocument,
dimConcept.typedDomainElement.qname,
text=dimVal,
appendChild=False)
qnameDims[dimQname] = DimValuePrototype(
modelXbrl, None, dimQname, mem, "segment")
_cntx = modelXbrl.createContext(
entityAsQn.namespaceURI,
entityAsQn.localName,
concept.periodType,
None if concept.periodType == "instant" else dateTime(
periodStart, type=DATETIME),
dateTime(periodEnd, type=DATETIME),
None, # no dimensional validity checking (like formula does)
qnameDims,
[],
[],
id=cntxId)
cntxTbl[cntxKey] = _cntx
if oimUnit in fact:
unitKey = fact[oimUnit]
if unitKey in unitTbl:
_unit = unitTbl[unitKey]
else:
_unit = None
# validate unit
unitKeySub = PrefixedQName.sub(
UnitPrefixedQNameSubstitutionChar, unitKey)
if not UnitPattern.match(unitKeySub):
modelXbrl.error(
"oime:unitStringRepresentation",
_("Unit string representation is lexically invalid, %(unit)s"
),
modelObject=modelXbrl,
unit=unitKey)
else:
_mul, _sep, _div = unitKey.partition('/')
if _mul.startswith('('):
_mul = _mul[1:-1]
_muls = [u for u in _mul.split('*') if u]
if _div.startswith('('):
_div = _div[1:-1]
_divs = [u for u in _div.split('*') if u]
if _muls != sorted(_muls) or _divs != sorted(_divs):
modelXbrl.error(
"oime:unitStringRepresentation",
_("Unit string representation measures are not in alphabetical order, %(unit)s"
),
modelObject=modelXbrl,
unit=unitKey)
try:
mulQns = [
qname(
u,
prefixes,
prefixException=OIMException(
"oime:unitPrefix",
_("Unit prefix is not declared: %(unit)s"
),
unit=u)) for u in _muls
]
divQns = [
qname(
u,
prefixes,
prefixException=OIMException(
"oime:unitPrefix",
_("Unit prefix is not declared: %(unit)s"
),
unit=u)) for u in _divs
]
unitId = 'u-{:02}'.format(len(unitTbl) + 1)
for _measures in mulQns, divQns:
for _measure in _measures:
addQnameValue(modelXbrl.modelDocument,
_measure)
_unit = modelXbrl.createUnit(mulQns,
divQns,
id=unitId)
except OIMException as ex:
modelXbrl.error(ex.code,
ex.message,
modelObject=modelXbrl,
**ex.msgArgs)
unitTbl[unitKey] = _unit
else:
_unit = None
attrs = {"contextRef": _cntx.id}
if fact.get("value") is None:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact["value"]
if fact.get("id"):
attrs["id"] = fact["id"]
if concept.isNumeric:
if _unit is None:
continue # skip creating fact because unit was invalid
attrs["unitRef"] = _unit.id
if "accuracy" in fact:
attrs["decimals"] = fact["accuracy"]
# is value a QName?
if concept.baseXbrliType == "QName":
addQnameValue(modelXbrl.modelDocument,
qname(text.strip(), prefixes))
f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
currentAction = "creating footnotes"
footnoteLinks = {} # ELR elements
factLocs = {} # index by (linkrole, factId)
footnoteNbr = 0
locNbr = 0
for factOrFootnote in footnotes:
if "factId" in factOrFootnote:
factId = factOrFootnote["factId"]
factFootnotes = (factOrFootnote, ) # CSV or XL
elif "id" in factOrFootnote and "footnotes" in factOrFootnote:
factId = factOrFootnote["id"]
factFootnotes = factOrFootnote["footnotes"]
else:
factFootnotes = ()
for footnote in factFootnotes:
linkrole = footnote.get("group")
arcrole = footnote.get("footnoteType")
if not factId or not linkrole or not arcrole or not (
footnote.get("factRef") or footnote.get("footnote")):
# invalid footnote
continue
if linkrole not in footnoteLinks:
footnoteLinks[linkrole] = addChild(
modelXbrl.modelDocument.xmlRootElement,
XbrlConst.qnLinkFootnoteLink,
attributes={
"{http://www.w3.org/1999/xlink}type": "extended",
"{http://www.w3.org/1999/xlink}role": linkrole
})
footnoteLink = footnoteLinks[linkrole]
if (linkrole, factId) not in factLocs:
locNbr += 1
locLabel = "l_{:02}".format(locNbr)
factLocs[(linkrole, factId)] = locLabel
addChild(footnoteLink,
XbrlConst.qnLinkLoc,
attributes={
XLINKTYPE: "locator",
XLINKHREF: "#" + factId,
XLINKLABEL: locLabel
})
locLabel = factLocs[(linkrole, factId)]
if footnote.get("footnote"):
footnoteNbr += 1
footnoteLabel = "f_{:02}".format(footnoteNbr)
attrs = {XLINKTYPE: "resource", XLINKLABEL: footnoteLabel}
if footnote.get("language"):
attrs[XMLLANG] = footnote["language"]
# note, for HTML will need to build an element structure
addChild(footnoteLink,
XbrlConst.qnLinkFootnote,
attributes=attrs,
text=footnote["footnote"])
elif footnote.get("factRef"):
factRef = footnote.get("factRef")
if (linkrole, factRef) not in factLocs:
locNbr += 1
locLabel = "f_{:02}".format(footnoteNbr)
factLoc[(linkrole, factRef)] = locLabel
addChild(footnoteLink,
XbrlConst.qnLinkLoc,
attributes={
XLINKTYPE: "locator",
XLINKHREF: "#" + factRef,
XLINKLABEL: locLabel
})
footnoteLabel = factLoc[(linkrole, factId)]
footnoteArc = addChild(footnoteLink,
XbrlConst.qnLinkFootnoteArc,
attributes={
XLINKTYPE: "arc",
XLINKARCROLE: arcrole,
XLINKFROM: locLabel,
XLINKTO: footnoteLabel
})
currentAction = "done loading facts and footnotes"
#cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt),
# messageCode="loadFromExcel:info")
except Exception as ex:
if isinstance(ex, OIMException):
modelXbrl.error(ex.code,
ex.message,
modelObject=modelXbrl,
**ex.msgArgs)
else:
modelXbrl.error(
"arelleOIMloader:error",
"Error while %(action)s, error %(error)s\ntraceback %(traceback)s",
modelObject=modelXbrl,
action=currentAction,
error=ex,
traceback=traceback.format_tb(sys.exc_info()[2]))
if priorCWD:
os.chdir(
priorCWD
) # restore prior current working directory startingErrorCount = len(modelXbrl.errors)
if startingErrorCount < len(modelXbrl.errors):
# had errors, don't allow ModelDocument.load to continue
return OIMException("arelleOIMloader:unableToLoad",
"Unable to load due to reported errors")
return getattr(modelXbrl, "modelDocument",
None) # none if returning from exception
def current_dateTime(xc, p, contextItem, args):
from datetime import datetime
return dateTime(datetime.now(), type=DATETIME)
def validateValue(modelXbrl, elt, attrTag, baseXsdType, value, isNillable=False, facets=None):
if baseXsdType:
try:
if (len(value) == 0 and
not attrTag is None and
not isNillable and
baseXsdType not in ("anyType", "string", "normalizedString", "token", "NMTOKEN", "anyURI", "noContent")):
raise ValueError("missing value for not nillable element")
xValid = VALID
whitespaceReplace = (baseXsdType == "normalizedString")
whitespaceCollapse = (not whitespaceReplace and baseXsdType != "string")
pattern = baseXsdTypePatterns.get(baseXsdType)
if facets:
if "pattern" in facets:
pattern = facets["pattern"]
# note multiple patterns are or'ed togetner, which isn't yet implemented!
if "whiteSpace" in facets:
whitespaceReplace, whitespaceCollapse = {"preserve":(False,False), "replace":(True,False), "collapse":(False,True)}
if whitespaceReplace:
value = normalizeWhitespacePattern.sub(' ', value)
elif whitespaceCollapse:
value = collapseWhitespacePattern.sub(' ', value.strip())
if pattern is not None and pattern.match(value) is None:
raise ValueError("pattern facet " + facets["pattern"].pattern if facets and "pattern" in facets else "pattern mismatch")
if facets:
if "enumeration" in facets and value not in facets["enumeration"]:
raise ValueError("is not in {1}".format(value, facets["enumeration"]))
if "length" in facets and len(value) != facets["length"]:
raise ValueError("length facet")
if "minLength" in facets and len(value) < facets["minLength"]:
raise ValueError("minLength facet")
if "maxLength" in facets and len(value) > facets["maxLength"]:
raise ValueError("maxLength facet")
if baseXsdType == "noContent":
if len(value) > 0 and not value.isspace():
raise ValueError("value content not permitted")
xValue = sValue = None
elif baseXsdType in {"string", "normalizedString", "language", "token", "NMTOKEN","Name","NCName","IDREF","ENTITY"}:
xValue = sValue = value
elif baseXsdType == "ID":
xValue = sValue = value
xValid = VALID_ID
elif baseXsdType == "anyURI":
xValue = anyURI(value)
sValue = value
if xValue and not UrlUtil.isValid(xValue): # allow empty strings to be valid anyURIs
raise ValueError("invalid anyURI value")
elif not value: # rest of types get None if nil/empty value
xValue = sValue = None
elif baseXsdType in ("decimal", "float", "double"):
xValue = sValue = float(value)
if facets:
if "totalDigits" in facets and len(value.replace(".","")) != facets["totalDigits"]:
raise ValueError("total digits facet")
if "fractionDigits" in facets and ( '.' not in value or
len(value[value.index('.') + 1:]) != facets["fractionDigits"]):
raise ValueError("fraction digits facet")
elif baseXsdType in ("integer",):
xValue = sValue = int(value)
elif baseXsdType == "boolean":
if value in ("true", "1"):
xValue = sValue = True
elif value in ("false", "0"):
xValue = sValue = False
else: raise ValueError
elif baseXsdType == "QName":
xValue = qname(elt, value, castException=ValueError)
sValue = value
''' not sure here, how are explicitDimensions validated, but bad units not?
if xValue.namespaceURI in modelXbrl.namespaceDocs:
if (xValue not in modelXbrl.qnameConcepts and
xValue not in modelXbrl.qnameTypes and
xValue not in modelXbrl.qnameAttributes and
xValue not in modelXbrl.qnameAttributeGroups):
raise ValueError("qname not defined " + str(xValue))
'''
elif baseXsdType in ("XBRLI_DECIMALSUNION", "XBRLI_PRECISIONUNION"):
xValue = sValue = value if value == "INF" else int(value)
elif baseXsdType in ("XBRLI_NONZERODECIMAL"):
xValue = sValue = int(value)
if xValue == 0:
raise ValueError("invalid value")
elif baseXsdType == "XBRLI_DATEUNION":
xValue = dateTime(value, type=DATEUNION, castException=ValueError)
sValue = value
elif baseXsdType == "dateTime":
xValue = dateTime(value, type=DATETIME, castException=ValueError)
sValue = value
elif baseXsdType == "date":
xValue = dateTime(value, type=DATE, castException=ValueError)
sValue = value
elif baseXsdType == "regex-pattern":
# for facet compiling
try:
xValue = re.compile(value + "$") # must match whole string
sValue = value
except Exception as err:
raise ValueError(err)
else:
xValue = value
sValue = value
except ValueError as err:
if attrTag:
modelXbrl.error("xmlSchema:valueError",
_("Element %(element)s attribute %(attribute)s type %(typeName)s value error: %(value)s, %(error)s"),
modelObject=elt,
element=elt.elementQname,
attribute=XmlUtil.clarkNotationToPrefixedName(elt,attrTag,isAttribute=True),
typeName=baseXsdType,
value=value,
error=err)
else:
modelXbrl.error("xmlSchema:valueError",
_("Element %(element)s type %(typeName)s value error: %(value)s"),
modelObject=elt,
element=elt.elementQname,
typeName=baseXsdType,
value=value)
xValue = None
sValue = value
xValid = INVALID
else:
xValue = sValue = None
xValid = UNKNOWN
if attrTag:
elt.xAttributes[attrTag] = ModelAttribute(elt, attrTag, xValid, xValue, sValue, value)
else:
elt.xValid = xValid
elt.xValue = xValue
elt.sValue = sValue
def validateValue(modelXbrl, elt, attrTag, baseXsdType, value, isNillable=False, isNil=False, facets=None):
if baseXsdType:
try:
'''
if (len(value) == 0 and attrTag is None and not isNillable and
baseXsdType not in ("anyType", "string", "normalizedString", "token", "NMTOKEN", "anyURI", "noContent")):
raise ValueError("missing value for not nillable element")
'''
xValid = VALID
whitespaceReplace = (baseXsdType == "normalizedString")
whitespaceCollapse = (not whitespaceReplace and baseXsdType != "string")
isList = baseXsdType in {"IDREFS", "ENTITIES", "NMTOKENS"}
if isList:
baseXsdType = baseXsdType[:-1] # remove plural
pattern = baseXsdTypePatterns.get(baseXsdType)
if facets:
if "pattern" in facets:
pattern = facets["pattern"]
# note multiple patterns are or'ed togetner, which isn't yet implemented!
if "whiteSpace" in facets:
whitespaceReplace, whitespaceCollapse = {"preserve":(False,False), "replace":(True,False), "collapse":(False,True)}[facets["whiteSpace"]]
if whitespaceReplace:
value = normalizeWhitespacePattern.sub(' ', value)
elif whitespaceCollapse:
value = collapseWhitespacePattern.sub(' ', value.strip())
if baseXsdType == "noContent":
if len(value) > 0 and not value.isspace():
raise ValueError("value content not permitted")
# note that sValue and xValue are not innerText but only text elements on specific element (or attribute)
xValue = sValue = None
xValid = VALID_NO_CONTENT # notify others that element may contain subelements (for stringValue needs)
elif not value and isNil and isNillable: # rest of types get None if nil/empty value
xValue = sValue = None
else:
if pattern is not None:
if ((isList and any(pattern.match(v) is None for v in value.split())) or
(not isList and pattern.match(value) is None)):
raise ValueError("pattern facet " + facets["pattern"].pattern if facets and "pattern" in facets else "pattern mismatch")
if facets:
if "enumeration" in facets and value not in facets["enumeration"]:
raise ValueError("{0} is not in {1}".format(value, facets["enumeration"]))
if "length" in facets and len(value) != facets["length"]:
raise ValueError("length {0}, expected {1}".format(len(value), facets["length"]))
if "minLength" in facets and len(value) < facets["minLength"]:
raise ValueError("length {0}, minLength {1}".format(len(value), facets["minLength"]))
if "maxLength" in facets and len(value) > facets["maxLength"]:
raise ValueError("length {0}, maxLength {1}".format(len(value), facets["maxLength"]))
if baseXsdType in {"string", "normalizedString", "language", "token", "NMTOKEN","Name","NCName","IDREF","ENTITY"}:
xValue = sValue = value
elif baseXsdType == "ID":
xValue = sValue = value
xValid = VALID_ID
elif baseXsdType == "anyURI":
if value: # allow empty strings to be valid anyURIs
if UrlUtil.relativeUrlPattern.match(value) is None:
raise ValueError("IETF RFC 2396 4.3 syntax")
# encode PSVI xValue similarly to Xerces and other implementations
xValue = anyURI(UrlUtil.anyUriQuoteForPSVI(value))
sValue = value
elif baseXsdType in ("decimal", "float", "double"):
if baseXsdType == "decimal":
if decimalPattern.match(value) is None:
raise ValueError("lexical pattern mismatch")
xValue = Decimal(value)
sValue = float(value) # s-value uses Number (float) representation
else:
if floatPattern.match(value) is None:
raise ValueError("lexical pattern mismatch")
xValue = sValue = float(value)
if facets:
if "totalDigits" in facets and len(value.replace(".","")) > facets["totalDigits"]:
raise ValueError("totalDigits facet {0}".format(facets["totalDigits"]))
if "fractionDigits" in facets and ( '.' in value and
len(value[value.index('.') + 1:]) > facets["fractionDigits"]):
raise ValueError("fraction digits facet {0}".format(facets["fractionDigits"]))
if "maxInclusive" in facets and xValue > facets["maxInclusive"]:
raise ValueError(" > maxInclusive {0}".format(facets["maxInclusive"]))
if "maxExclusive" in facets and xValue >= facets["maxExclusive"]:
raise ValueError(" >= maxInclusive {0}".format(facets["maxExclusive"]))
if "minInclusive" in facets and xValue < facets["minInclusive"]:
raise ValueError(" < minInclusive {0}".format(facets["minInclusive"]))
if "minExclusive" in facets and xValue <= facets["minExclusive"]:
raise ValueError(" <= minExclusive {0}".format(facets["minExclusive"]))
elif baseXsdType in {"integer",
"nonPositiveInteger","negativeInteger","nonNegativeInteger","positiveInteger",
"long","unsignedLong",
"int","unsignedInt",
"short","unsignedShort",
"byte","unsignedByte"}:
xValue = sValue = _INT(value)
if ((baseXsdType in {"nonNegativeInteger","unsignedLong","unsignedInt"}
and xValue < 0) or
(baseXsdType == "nonPositiveInteger" and xValue > 0) or
(baseXsdType == "positiveInteger" and xValue <= 0) or
(baseXsdType == "byte" and not -128 <= xValue < 127) or
(baseXsdType == "unsignedByte" and not 0 <= xValue < 255) or
(baseXsdType == "short" and not -32768 <= xValue < 32767) or
(baseXsdType == "unsignedShort" and not 0 <= xValue < 65535) or
(baseXsdType == "positiveInteger" and xValue <= 0)):
raise ValueError("{0} is not {1}".format(value, baseXsdType))
if facets:
if "totalDigits" in facets and len(value.replace(".","")) > facets["totalDigits"]:
raise ValueError("totalDigits facet {0}".format(facets["totalDigits"]))
if "fractionDigits" in facets and ( '.' in value and
len(value[value.index('.') + 1:]) > facets["fractionDigits"]):
raise ValueError("fraction digits facet {0}".format(facets["fractionDigits"]))
if "maxInclusive" in facets and xValue > facets["maxInclusive"]:
raise ValueError(" > maxInclusive {0}".format(facets["maxInclusive"]))
if "maxExclusive" in facets and xValue >= facets["maxExclusive"]:
raise ValueError(" >= maxInclusive {0}".format(facets["maxExclusive"]))
if "minInclusive" in facets and xValue < facets["minInclusive"]:
raise ValueError(" < minInclusive {0}".format(facets["minInclusive"]))
if "minExclusive" in facets and xValue <= facets["minExclusive"]:
raise ValueError(" <= minExclusive {0}".format(facets["minExclusive"]))
elif baseXsdType == "boolean":
if value in ("true", "1"):
xValue = sValue = True
elif value in ("false", "0"):
xValue = sValue = False
else: raise ValueError
elif baseXsdType == "QName":
xValue = qnameEltPfxName(elt, value, prefixException=ValueError)
#xValue = qname(elt, value, castException=ValueError, prefixException=ValueError)
sValue = value
''' not sure here, how are explicitDimensions validated, but bad units not?
if xValue.namespaceURI in modelXbrl.namespaceDocs:
if (xValue not in modelXbrl.qnameConcepts and
xValue not in modelXbrl.qnameTypes and
xValue not in modelXbrl.qnameAttributes and
xValue not in modelXbrl.qnameAttributeGroups):
raise ValueError("qname not defined " + str(xValue))
'''
elif baseXsdType in ("XBRLI_DECIMALSUNION", "XBRLI_PRECISIONUNION"):
xValue = sValue = value if value == "INF" else _INT(value)
elif baseXsdType in ("XBRLI_NONZERODECIMAL"):
xValue = sValue = _INT(value)
if xValue == 0:
raise ValueError("invalid value")
elif baseXsdType == "XBRLI_DATEUNION":
xValue = dateTime(value, type=DATEUNION, castException=ValueError)
sValue = value
elif baseXsdType == "dateTime":
xValue = dateTime(value, type=DATETIME, castException=ValueError)
sValue = value
elif baseXsdType == "date":
xValue = dateTime(value, type=DATE, castException=ValueError)
sValue = value
elif baseXsdType == "regex-pattern":
# for facet compiling
try:
sValue = value
if value in xmlSchemaPatterns:
xValue = xmlSchemaPatterns[value]
else:
if r"\i" in value or r"\c" in value:
value = value.replace(r"\i", iNameChar).replace(r"\c", cNameChar)
xValue = re_compile(value + "$") # must match whole string
except Exception as err:
raise ValueError(err)
elif baseXsdType == "fraction":
sValue = value
xValue = Fraction("/".join(elt.fractionValue))
else:
if baseXsdType in lexicalPatterns:
match = lexicalPatterns[baseXsdType].match(value)
if match is None:
raise ValueError("lexical pattern mismatch")
if baseXsdType == "gMonthDay":
month, day, zSign, zHrMin, zHr, zMin = match.groups()
if int(day) > {2:29, 4:30, 6:30, 9:30, 11:30, 1:31, 3:31, 5:31, 7:31, 8:31, 10:31, 12:31}[int(month)]:
raise ValueError("invalid day {0} for month {1}".format(day, month))
xValue = gMonthDay(month, day)
elif baseXsdType == "gYearMonth":
year, month, zSign, zHrMin, zHr, zMin = match.groups()
xValue = gYearMonth(year, month)
elif baseXsdType == "gYear":
year, zSign, zHrMin, zHr, zMin = match.groups()
xValue = gYear(year)
elif baseXsdType == "gMonth":
month, zSign, zHrMin, zHr, zMin = match.groups()
xValue = gMonth(month)
elif baseXsdType == "gDay":
day, zSign, zHrMin, zHr, zMin = match.groups()
xValue = gDay(day)
else:
xValue = value
else: # no lexical pattern, forget compiling value
xValue = value
sValue = value
except (ValueError, InvalidOperation) as err:
if ModelInlineValueObject is not None and isinstance(elt, ModelInlineValueObject):
errElt = "{0} fact {1}".format(elt.elementQname, elt.qname)
else:
errElt = elt.elementQname
if attrTag:
modelXbrl.error("xmlSchema:valueError",
_("Element %(element)s attribute %(attribute)s type %(typeName)s value error: %(value)s, %(error)s"),
modelObject=elt,
element=errElt,
attribute=XmlUtil.clarkNotationToPrefixedName(elt,attrTag,isAttribute=True),
typeName=baseXsdType,
value=value if len(value) < 31 else value[:30] + '...',
error=err)
else:
modelXbrl.error("xmlSchema:valueError",
_("Element %(element)s type %(typeName)s value error: %(value)s, %(error)s"),
modelObject=elt,
element=errElt,
typeName=baseXsdType,
value=value if len(value) < 31 else value[:30] + '...',
error=err)
xValue = None
sValue = value
xValid = INVALID
else:
xValue = sValue = None
xValid = UNKNOWN
if attrTag:
try: # dynamically allocate attributes (otherwise given shared empty set)
xAttributes = elt.xAttributes
except AttributeError:
elt.xAttributes = xAttributes = {}
xAttributes[attrTag] = ModelAttribute(elt, attrTag, xValid, xValue, sValue, value)
else:
elt.xValid = xValid
elt.xValue = xValue
elt.sValue = sValue
def validateValue(modelXbrl, elt, attrTag, baseXsdType, value, isNillable=False, facets=None):
if baseXsdType:
try:
if (len(value) == 0 and
not attrTag is None and
not isNillable and
baseXsdType not in ("anyType", "string", "normalizedString", "token", "NMTOKEN", "anyURI", "noContent")):
raise ValueError("missing value for not nillable element")
xValid = VALID
whitespaceReplace = (baseXsdType == "normalizedString")
whitespaceCollapse = (not whitespaceReplace and baseXsdType != "string")
pattern = baseXsdTypePatterns.get(baseXsdType)
if facets:
if "pattern" in facets:
pattern = facets["pattern"]
# note multiple patterns are or'ed togetner, which isn't yet implemented!
if "whiteSpace" in facets:
whitespaceReplace, whitespaceCollapse = {"preserve":(False,False), "replace":(True,False), "collapse":(False,True)}
if whitespaceReplace:
value = normalizeWhitespacePattern.sub(' ', value)
elif whitespaceCollapse:
value = collapseWhitespacePattern.sub(' ', value.strip())
if pattern is not None and pattern.match(value) is None:
raise ValueError("pattern facet " + facets["pattern"].pattern if facets and "pattern" in facets else "pattern mismatch")
if facets:
if "enumeration" in facets and value not in facets["enumeration"]:
raise ValueError("{0} is not in {1}".format(value, facets["enumeration"]))
if "length" in facets and len(value) != facets["length"]:
raise ValueError("length {0}, expected {1}".format(len(value), facets["length"]))
if "minLength" in facets and len(value) < facets["minLength"]:
raise ValueError("length {0}, minLength {1}".format(len(value), facets["minLength"]))
if "maxLength" in facets and len(value) > facets["maxLength"]:
raise ValueError("length {0}, maxLength {1}".format(len(value), facets["maxLength"]))
if baseXsdType == "noContent":
if len(value) > 0 and not value.isspace():
raise ValueError("value content not permitted")
xValue = sValue = None
elif baseXsdType in {"string", "normalizedString", "language", "token", "NMTOKEN","Name","NCName","IDREF","ENTITY"}:
xValue = sValue = value
elif baseXsdType == "ID":
xValue = sValue = value
xValid = VALID_ID
elif baseXsdType == "anyURI":
xValue = anyURI(value)
sValue = value
if xValue and not UrlUtil.isValid(xValue): # allow empty strings to be valid anyURIs
raise ValueError("invalid anyURI value")
elif not value: # rest of types get None if nil/empty value
xValue = sValue = None
elif baseXsdType in ("decimal", "float", "double"):
xValue = sValue = float(value)
if facets:
if "totalDigits" in facets and len(value.replace(".","")) > facets["totalDigits"]:
raise ValueError("totalDigits facet {0}".format(facets["totalDigits"]))
if "fractionDigits" in facets and ( '.' in value and
len(value[value.index('.') + 1:]) > facets["fractionDigits"]):
raise ValueError("fraction digits facet {0}".format(facets["fractionDigits"]))
elif baseXsdType in {"integer",
"nonPositiveInteger","negativeInteger","nonNegativeInteger","positiveInteger",
"long","unsignedLong",
"int","unsignedInt",
"short","unsignedShort",
"byte","unsignedByte"}:
xValue = sValue = int(value)
if ((baseXsdType in {"nonNegativeInteger","unsignedLong","unsignedInt"}
and xValue < 0) or
(baseXsdType == "nonPositiveInteger" and xValue > 0) or
(baseXsdType == "positiveInteger" and xValue <= 0) or
(baseXsdType == "byte" and not -128 <= xValue < 127) or
(baseXsdType == "unsignedByte" and not 0 <= xValue < 255) or
(baseXsdType == "short" and not -32768 <= xValue < 32767) or
(baseXsdType == "unsignedShort" and not 0 <= xValue < 65535) or
(baseXsdType == "positiveInteger" and xValue <= 0)):
raise ValueError("{0} is not {1}".format(value, baseXsdType))
elif baseXsdType == "boolean":
if value in ("true", "1"):
xValue = sValue = True
elif value in ("false", "0"):
xValue = sValue = False
else: raise ValueError
elif baseXsdType == "QName":
xValue = qname(elt, value, castException=ValueError)
sValue = value
''' not sure here, how are explicitDimensions validated, but bad units not?
if xValue.namespaceURI in modelXbrl.namespaceDocs:
if (xValue not in modelXbrl.qnameConcepts and
xValue not in modelXbrl.qnameTypes and
xValue not in modelXbrl.qnameAttributes and
xValue not in modelXbrl.qnameAttributeGroups):
raise ValueError("qname not defined " + str(xValue))
'''
elif baseXsdType in ("XBRLI_DECIMALSUNION", "XBRLI_PRECISIONUNION"):
xValue = sValue = value if value == "INF" else int(value)
elif baseXsdType in ("XBRLI_NONZERODECIMAL"):
xValue = sValue = int(value)
if xValue == 0:
raise ValueError("invalid value")
elif baseXsdType == "XBRLI_DATEUNION":
xValue = dateTime(value, type=DATEUNION, castException=ValueError)
sValue = value
elif baseXsdType == "dateTime":
xValue = dateTime(value, type=DATETIME, castException=ValueError)
sValue = value
elif baseXsdType == "date":
xValue = dateTime(value, type=DATE, castException=ValueError)
sValue = value
elif baseXsdType == "regex-pattern":
# for facet compiling
try:
xValue = re.compile(value + "$") # must match whole string
sValue = value
except Exception as err:
raise ValueError(err)
else:
if baseXsdType in lexicalPatterns:
match = lexicalPatterns[baseXsdType].match(value)
if match is None:
raise ValueError("lexical pattern mismatch")
if baseXsdType == "gMonthDay":
month, day, zSign, zHrMin, zHr, zMin = match.groups()
if int(day) > {2:29, 4:30, 6:30, 9:30, 11:30, 1:31, 3:31, 5:31, 7:31, 8:31, 10:31, 12:31}[int(month)]:
raise ValueError("invalid day {0} for month {1}".format(day, month))
xValue = value
sValue = value
except ValueError as err:
if ModelInlineFact is not None and isinstance(elt, ModelInlineFact):
errElt = "{0} fact {1}".format(elt.elementQname, elt.qname)
else:
errElt = elt.elementQname
if attrTag:
modelXbrl.error("xmlSchema:valueError",
_("Element %(element)s attribute %(attribute)s type %(typeName)s value error: %(value)s, %(error)s"),
modelObject=elt,
element=errElt,
attribute=XmlUtil.clarkNotationToPrefixedName(elt,attrTag,isAttribute=True),
typeName=baseXsdType,
value=value,
error=err)
else:
modelXbrl.error("xmlSchema:valueError",
_("Element %(element)s type %(typeName)s value error: %(value)s, %(error)s"),
modelObject=elt,
element=errElt,
typeName=baseXsdType,
value=value,
error=err)
xValue = None
sValue = value
xValid = INVALID
else:
xValue = sValue = None
xValid = UNKNOWN
if attrTag:
try: # dynamically allocate attributes (otherwise given shared empty set)
xAttributes = elt.xAttributes
except AttributeError:
elt.xAttributes = xAttributes = {}
xAttributes[attrTag] = ModelAttribute(elt, attrTag, xValid, xValue, sValue, value)
else:
elt.xValid = xValid
elt.xValue = xValue
elt.sValue = sValue
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri):
from openpyxl import load_workbook
from arelle import ModelDocument, ModelXbrl, XmlUtil
from arelle.ModelDocument import ModelDocumentReference
from arelle.ModelValue import qname
try:
currentAction = "initializing"
startingErrorCount = len(modelXbrl.errors)
startedAt = time.time()
if os.path.isabs(oimFile):
# allow relative filenames to loading directory
priorCWD = os.getcwd()
os.chdir(os.path.dirname(oimFile))
else:
priorCWD = None
currentAction = "determining file type"
isJSON = oimFile.endswith(".json") and not oimFile.endswith("-metadata.json")
isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json")
isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls")
isCSVorXL = isCSV or isXL
instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl"
if isJSON:
currentAction = "loading and parsing JSON OIM file"
def loadDict(keyValuePairs):
_dict = OrderedDict() # preserve fact order in resulting instance
for key, value in keyValuePairs:
if isinstance(value, dict):
if DUPJSONKEY in value:
for _errKey, _errValue, _otherValue in value[DUPJSONKEY]:
if key == "prefixes":
modelXbrl.error("oime:duplicatedPrefix",
_("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"),
modelObject=modelXbrl, prefix=_errKey, uri1=_errValue, uri2=_otherValue)
del value[DUPJSONKEY]
if key in _dict:
if DUPJSONKEY not in _dict:
_dict[DUPJSONKEY] = []
_dict[DUPJSONKEY].append((key, value, _dict[key]))
else:
_dict[key] = value
return _dict
with io.open(oimFile, 'rt', encoding='utf-8') as f:
oimObject = json.load(f, object_pairs_hook=loadDict)
missing = [t for t in ("dtsReferences", "prefixes", "facts") if t not in oimObject]
if missing:
raise OIMException("oime:missingJSONElements",
_("Required element(s) are missing from JSON input: %(missing)s"),
missing = ", ".join(missing))
currentAction = "identifying JSON objects"
dtsReferences = oimObject["dtsReferences"]
prefixesList = oimObject["prefixes"].items()
facts = oimObject["facts"]
footnotes = oimObject["facts"] # shares this object
elif isCSV:
currentAction = "identifying CSV input tables"
if sys.version[0] >= '3':
csvOpenMode = 'w'
csvOpenNewline = ''
else:
csvOpenMode = 'wb' # for 2.7
csvOpenNewline = None
oimFileBase = None
if "-facts" in oimFile:
oimFileBase = oimFile.partition("-facts")[0]
else:
for suffix in ("-dtsReferences.csv", "-defaults.csv", "-prefixes.csv", "-footnotes.csv", "-metadata.json"):
if oimFile.endswith(suffix):
oimFileBase = oimFile[:-len(suffix)]
break
if oimFileBase is None:
raise OIMException("oime:missingCSVTables",
_("Unable to identify CSV tables file name pattern"))
if (not os.path.exists(oimFileBase + "-dtsReferences.csv") or
not os.path.exists(oimFileBase + "-prefixes.csv")):
raise OIMException("oime:missingCSVTables",
_("Unable to identify CSV tables for dtsReferences or prefixes"))
instanceFileName = oimFileBase + ".xbrl"
currentAction = "loading CSV dtsReferences table"
dtsReferences = []
with io.open(oimFileBase + "-dtsReferences.csv", 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
else:
dtsReferences.append(dict((header[j], col) for j, col in enumerate(row)))
currentAction = "loading CSV prefixes table"
prefixesList = []
with io.open(oimFileBase + "-prefixes.csv", 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = dict((col,i) for i,col in enumerate(row))
else:
prefixesList.append((row[header["prefix"]], row[header["URI"]]))
defaults = {}
if os.path.exists(oimFileBase + "-defaults.csv"):
currentAction = "loading CSV defaults table"
with io.open(oimFileBase + "-defaults.csv", 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
fileCol = row.index("file")
else:
defaults[row[fileCol]] = dict((header[j], col) for j, col in enumerate(row) if j != fileCol)
currentAction = "loading CSV facts tables"
facts = []
_dir = os.path.dirname(oimFileBase)
factsFileBasename = os.path.basename(oimFileBase) + "-facts"
for filename in os.listdir(_dir):
filepath = os.path.join(_dir, filename)
if filename.startswith(factsFileBasename):
currentAction = "loading CSV facts table {}".format(filename)
tableDefaults = defaults.get(filename, {})
with io.open(filepath, 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
else:
fact = {}
fact.update(tableDefaults)
for j, col in enumerate(row):
if col is not None:
if header[j]: # skip cols with no header
if header[j].endswith("Value"):
if col: # ignore empty columns (= null CSV value)
fact["value"] = col
else:
fact[header[j]] = col
facts.append(fact)
footnotes = []
if os.path.exists(oimFileBase + "-footnotes.csv"):
currentAction = "loading CSV footnotes table"
with io.open(oimFileBase + "-footnotes.csv", 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
else:
footnotes.append(dict((header[j], col) for j, col in enumerate(row) if col))
elif isXL:
currentAction = "identifying workbook input worksheets"
oimWb = load_workbook(oimFile, read_only=True, data_only=True)
sheetNames = oimWb.get_sheet_names()
if (not any(sheetName == "prefixes" for sheetName in sheetNames) or
not any(sheetName == "dtsReferences" for sheetName in sheetNames) or
not any("facts" in sheetName for sheetName in sheetNames)):
raise OIMException("oime:missingWorkbookWorksheets",
_("Unable to identify worksheet tabs for dtsReferences, prefixes or facts"))
currentAction = "loading worksheet: dtsReferences"
dtsReferences = []
for i, row in enumerate(oimWb["dtsReferences"]):
if i == 0:
header = [col.value for col in row]
else:
dtsReferences.append(dict((header[j], col.value) for j, col in enumerate(row)))
currentAction = "loading worksheet: prefixes"
prefixesList = []
for i, row in enumerate(oimWb["prefixes"]):
if i == 0:
header = dict((col.value,i) for i,col in enumerate(row))
else:
prefixesList.append((row[header["prefix"]].value, row[header["URI"]].value))
defaults = {}
if "defaults" in sheetNames:
currentAction = "loading worksheet: defaults"
for i, row in enumerate(oimWb["defaults"]):
if i == 0:
header = dict((col.value,i) for i,col in enumerate(row))
fileCol = header["file"]
else:
defaults[row[fileCol].value] = dict((header[j], col.value) for j, col in enumerate(row) if j != fileCol)
facts = []
for sheetName in sheetNames:
if sheetName == "facts" or "-facts" in sheetName:
currentAction = "loading worksheet: {}".format(sheetName)
tableDefaults = defaults.get(sheetName, {})
for i, row in enumerate(oimWb[sheetName]):
if i == 0:
header = [col.value for col in row]
else:
fact = {}
fact.update(tableDefaults)
for j, col in enumerate(row):
if col.value is not None:
if header[j]: # skip cols with no header
if header[j].endswith("Value"):
fact["value"] = str(col.value)
else:
fact[header[j]] = str(col.value)
facts.append(fact)
footnotes = []
if "footnotes" in sheetNames:
currentAction = "loading worksheet: footnotes"
for i, row in enumerate(oimWb["footnotes"]):
if i == 0:
header = dict((col.value,i) for i,col in enumerate(row) if col.value)
else:
footnotes.append(dict((header[j], col.value) for j, col in enumerate(row) if col.value))
currentAction = "identifying default dimensions"
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults
currentAction = "validating OIM"
prefixes = {}
prefixedUris = {}
for _prefix, _uri in prefixesList:
if not _prefix:
modelXbrl.error("oime:emptyPrefix",
_("The empty string must not be used as a prefix, uri %(uri)s"),
modelObject=modelXbrl, uri=_uri)
elif not NCNamePattern.match(_prefix):
modelXbrl.error("oime:prefixPattern",
_("The prefix %(prefix)s must match the NCName lexical pattern, uri %(uri)s"),
modelObject=modelXbrl, prefix=_prefix, uri=_uri)
elif _prefix in prefixes:
modelXbrl.error("oime:duplicatedPrefix",
_("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"),
modelObject=modelXbrl, prefix=_prefix, uri1=prefixes[_prefix], uri2=_uri)
elif _uri in prefixedUris:
modelXbrl.error("oime:duplicatedUri",
_("The uri %(uri)s is used on prefix %(prefix1)s and prefix %(prefix2)s"),
modelObject=modelXbrl, uri=_uri, prefix1=prefixedUris[_uri], prefix2=_prefix)
else:
prefixes[_prefix] = _uri
prefixedUris[_uri] = _prefix
oimPrefix = None
for _nsOim in nsOim:
if _nsOim in prefixedUris:
oimPrefix = prefixedUris[_nsOim]
if not oimPrefix:
raise OIMException("oime:noOimPrefix",
_("The oim namespace must have a declared prefix"))
oimConcept = "{}:concept".format(oimPrefix)
oimEntity = "{}:entity".format(oimPrefix)
oimPeriod = "{}:period".format(oimPrefix)
oimPeriodStart = "{}:periodStart".format(oimPrefix)
oimPeriodEnd = "{}:periodEnd".format(oimPrefix)
oimUnit = "{}:unit".format(oimPrefix)
oimPrefix = "{}:".format(oimPrefix)
# create the instance document
currentAction = "creating instance document"
modelXbrl.blockDpmDBrecursion = True
modelXbrl.modelDocument = createModelDocument(
modelXbrl,
Type.INSTANCE,
instanceFileName,
schemaRefs=[dtsRef["href"] for dtsRef in dtsReferences if dtsRef["type"] == "schema"],
isEntry=True,
initialComment="extracted from OIM {}".format(mappedUri),
documentEncoding="utf-8")
cntxTbl = {}
unitTbl = {}
currentAction = "creating facts"
for fact in facts:
conceptQn = qname(fact[oimConcept], prefixes)
concept = modelXbrl.qnameConcepts.get(conceptQn)
entityAsQn = qname(fact[oimEntity], prefixes)
if oimPeriod in fact:
periodStart = fact[oimPeriod]["start"]
periodEnd = fact[oimPeriod]["end"]
else:
periodStart = fact[oimPeriodStart]
periodEnd = fact[oimPeriodEnd]
cntxKey = ( # hashable context key
("periodType", concept.periodType),
("entity", entityAsQn),
("periodStart", periodStart),
("periodEnd", periodEnd)) + tuple(sorted(
(dimName, dimVal)
for dimName, dimVal in fact.items()
if ":" in dimName and not dimName.startswith(oimPrefix)))
if cntxKey in cntxTbl:
_cntx = cntxTbl[cntxKey]
else:
cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
qnameDims = {}
for dimName, dimVal in fact.items():
if ":" in dimName and not dimName.startswith(oimPrefix) and dimVal:
dimQname = qname(dimName, prefixes)
dimConcept = modelXbrl.qnameConcepts.get(dimQname)
if ":" in dimVal and dimVal.partition(':')[0] in prefixes:
mem = qname(dimVal, prefixes) # explicit dim
elif dimConcept.isTypedDimension:
# a modelObject xml element is needed for all of the instance functions to manage the typed dim
mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False)
qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment")
_cntx = modelXbrl.createContext(
entityAsQn.namespaceURI,
entityAsQn.localName,
concept.periodType,
None if concept.periodType == "instant" else dateTime(periodStart, type=DATETIME),
dateTime(periodEnd, type=DATETIME),
None, # no dimensional validity checking (like formula does)
qnameDims, [], [],
id=cntxId)
cntxTbl[cntxKey] = _cntx
if oimUnit in fact:
unitKey = fact[oimUnit]
if unitKey in unitTbl:
_unit = unitTbl[unitKey]
else:
_unit = None
# validate unit
unitKeySub = PrefixedQName.sub(UnitPrefixedQNameSubstitutionChar, unitKey)
if not UnitPattern.match(unitKeySub):
modelXbrl.error("oime:unitStringRepresentation",
_("Unit string representation is lexically invalid, %(unit)s"),
modelObject=modelXbrl, unit=unitKey)
else:
_mul, _sep, _div = unitKey.partition('/')
if _mul.startswith('('):
_mul = _mul[1:-1]
_muls = [u for u in _mul.split('*') if u]
if _div.startswith('('):
_div = _div[1:-1]
_divs = [u for u in _div.split('*') if u]
if _muls != sorted(_muls) or _divs != sorted(_divs):
modelXbrl.error("oime:unitStringRepresentation",
_("Unit string representation measures are not in alphabetical order, %(unit)s"),
modelObject=modelXbrl, unit=unitKey)
try:
mulQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix",
_("Unit prefix is not declared: %(unit)s"),
unit=u))
for u in _muls]
divQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix",
_("Unit prefix is not declared: %(unit)s"),
unit=u))
for u in _divs]
unitId = 'u-{:02}'.format(len(unitTbl) + 1)
for _measures in mulQns, divQns:
for _measure in _measures:
addQnameValue(modelXbrl.modelDocument, _measure)
_unit = modelXbrl.createUnit(mulQns, divQns, id=unitId)
except OIMException as ex:
modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs)
unitTbl[unitKey] = _unit
else:
_unit = None
attrs = {"contextRef": _cntx.id}
if fact.get("value") is None:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact["value"]
if fact.get("id"):
attrs["id"] = fact["id"]
if concept.isNumeric:
if _unit is None:
continue # skip creating fact because unit was invalid
attrs["unitRef"] = _unit.id
if "accuracy" in fact:
attrs["decimals"] = fact["accuracy"]
# is value a QName?
if concept.baseXbrliType == "QName":
addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes))
f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
currentAction = "creating footnotes"
footnoteLinks = {} # ELR elements
factLocs = {} # index by (linkrole, factId)
footnoteNbr = 0
locNbr = 0
for factOrFootnote in footnotes:
if "factId" in factOrFootnote:
factId = factOrFootnote["factId"]
factFootnotes = (factOrFootnote,) # CSV or XL
elif "id" in factOrFootnote and "footnotes" in factOrFootnote:
factId = factOrFootnote["id"]
factFootnotes = factOrFootnote["footnotes"]
else:
factFootnotes = ()
for footnote in factFootnotes:
linkrole = footnote.get("group")
arcrole = footnote.get("footnoteType")
if not factId or not linkrole or not arcrole or not (
footnote.get("factRef") or footnote.get("footnote")):
# invalid footnote
continue
if linkrole not in footnoteLinks:
footnoteLinks[linkrole] = addChild(modelXbrl.modelDocument.xmlRootElement,
XbrlConst.qnLinkFootnoteLink,
attributes={"{http://www.w3.org/1999/xlink}type": "extended",
"{http://www.w3.org/1999/xlink}role": linkrole})
footnoteLink = footnoteLinks[linkrole]
if (linkrole, factId) not in factLocs:
locNbr += 1
locLabel = "l_{:02}".format(locNbr)
factLocs[(linkrole, factId)] = locLabel
addChild(footnoteLink, XbrlConst.qnLinkLoc,
attributes={XLINKTYPE: "locator",
XLINKHREF: "#" + factId,
XLINKLABEL: locLabel})
locLabel = factLocs[(linkrole, factId)]
if footnote.get("footnote"):
footnoteNbr += 1
footnoteLabel = "f_{:02}".format(footnoteNbr)
attrs = {XLINKTYPE: "resource",
XLINKLABEL: footnoteLabel}
if footnote.get("language"):
attrs[XMLLANG] = footnote["language"]
# note, for HTML will need to build an element structure
addChild(footnoteLink, XbrlConst.qnLinkFootnote, attributes=attrs, text=footnote["footnote"])
elif footnote.get("factRef"):
factRef = footnote.get("factRef")
if (linkrole, factRef) not in factLocs:
locNbr += 1
locLabel = "f_{:02}".format(footnoteNbr)
factLoc[(linkrole, factRef)] = locLabel
addChild(footnoteLink, XbrlConst.qnLinkLoc,
attributes={XLINKTYPE: "locator",
XLINKHREF: "#" + factRef,
XLINKLABEL: locLabel})
footnoteLabel = factLoc[(linkrole, factId)]
footnoteArc = addChild(footnoteLink,
XbrlConst.qnLinkFootnoteArc,
attributes={XLINKTYPE: "arc",
XLINKARCROLE: arcrole,
XLINKFROM: locLabel,
XLINKTO: footnoteLabel})
currentAction = "done loading facts and footnotes"
#cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt),
# messageCode="loadFromExcel:info")
except Exception as ex:
if isinstance(ex, OIMException):
modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs)
else:
modelXbrl.error("arelleOIMloader:error",
"Error while %(action)s, error %(error)s\ntraceback %(traceback)s",
modelObject=modelXbrl, action=currentAction, error=ex,
traceback=traceback.format_tb(sys.exc_info()[2]))
if priorCWD:
os.chdir(priorCWD) # restore prior current working directory startingErrorCount = len(modelXbrl.errors)
if startingErrorCount < len(modelXbrl.errors):
# had errors, don't allow ModelDocument.load to continue
return OIMException("arelleOIMloader:unableToLoad", "Unable to load due to reported errors")
return getattr(modelXbrl, "modelDocument", None) # none if returning from exception
def current_date(xc, p, contextItem, args):
from datetime import date
return dateTime(date.today(), type=DATE)
def createModelFact(fact, parentModelFact, topTupleFact):
aspects = fact.get("aspects", EMPTYDICT)
if oimConcept not in aspects:
modelXbrl.error("{}:conceptQName".format(errPrefix),
_("The concept QName could not be determined"),
modelObject=modelXbrl)
return
conceptQn = qname(aspects[oimConcept], prefixes)
concept = modelXbrl.qnameConcepts.get(conceptQn)
attrs = {}
if concept.isItem:
missingAspects = []
if oimEntity not in aspects:
missingAspects.append(oimEntity)
if oimPeriod not in aspects and (oimPeriodStart not in aspects or oimPeriodEnd not in aspects):
missingAspects.append(oimPeriod)
if missingAspects:
modelXbrl.error("{}:missingAspects".format(errPrefix),
_("The concept %(element)s is missing aspects %(missingAspects)s"),
modelObject=modelXbrl, element=conceptQn, missingAspects=", ".join(missingAspects))
return
entityAsQn = qname(aspects[oimEntity], prefixes) or qname("error",fact[oimEntity])
if oimPeriod in aspects:
periodStart = periodEnd = aspects[oimPeriod]
if oimPeriodStart in aspects and oimPeriodEnd in aspects:
periodStart = aspects[oimPeriodStart]
periodEnd = aspects[oimPeriodEnd]
cntxKey = ( # hashable context key
("periodType", concept.periodType),
("entity", entityAsQn),
("periodStart", periodStart),
("periodEnd", periodEnd)) + tuple(sorted(
(dimName, dimVal["value"] if isinstance(dimVal,dict) else dimVal)
for dimName, dimVal in aspects.items()
if ":" in dimName and not dimName.startswith(oimPrefix)))
if cntxKey in cntxTbl:
_cntx = cntxTbl[cntxKey]
else:
cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
qnameDims = {}
for dimName, dimVal in aspects.items():
if ":" in dimName and not dimName.startswith(oimPrefix) and dimVal:
dimQname = qname(dimName, prefixes)
dimConcept = modelXbrl.qnameConcepts.get(dimQname)
if dimConcept is None:
modelXbrl.error("{}:taxonomyDefinedAspectQName".format(errPrefix),
_("The taxonomy defined aspect concept QName %(qname)s could not be determined"),
modelObject=modelXbrl, qname=dimQname)
continue
if isinstance(dimVal, dict):
dimVal = dimVal["value"]
else:
dimVal = str(dimVal) # may be int or boolean
if isinstance(dimVal,str) and ":" in dimVal and dimVal.partition(':')[0] in prefixes:
mem = qname(dimVal, prefixes) # explicit dim
elif dimConcept.isTypedDimension:
# a modelObject xml element is needed for all of the instance functions to manage the typed dim
mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False)
qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment")
_cntx = modelXbrl.createContext(
entityAsQn.namespaceURI,
entityAsQn.localName,
concept.periodType,
None if concept.periodType == "instant" else dateTime(periodStart, type=DATETIME),
dateTime(periodEnd, type=DATETIME),
None, # no dimensional validity checking (like formula does)
qnameDims, [], [],
id=cntxId,
beforeSibling=topTupleFact)
cntxTbl[cntxKey] = _cntx
if oimUnit in aspects and concept.isNumeric:
unitKey = aspects[oimUnit]
if unitKey in unitTbl:
_unit = unitTbl[unitKey]
else:
_unit = None
# validate unit
unitKeySub = PrefixedQName.sub(UnitPrefixedQNameSubstitutionChar, unitKey)
if not UnitPattern.match(unitKeySub):
modelXbrl.error("oime:unitStringRepresentation",
_("Unit string representation is lexically invalid, %(unit)s"),
modelObject=modelXbrl, unit=unitKey)
else:
_mul, _sep, _div = unitKey.partition('/')
if _mul.startswith('('):
_mul = _mul[1:-1]
_muls = [u for u in _mul.split('*') if u]
if _div.startswith('('):
_div = _div[1:-1]
_divs = [u for u in _div.split('*') if u]
if _muls != sorted(_muls) or _divs != sorted(_divs):
modelXbrl.error("oime:unitStringRepresentation",
_("Unit string representation measures are not in alphabetical order, %(unit)s"),
modelObject=modelXbrl, unit=unitKey)
try:
mulQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix",
_("Unit prefix is not declared: %(unit)s"),
unit=u))
for u in _muls]
divQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix",
_("Unit prefix is not declared: %(unit)s"),
unit=u))
for u in _divs]
unitId = 'u-{:02}'.format(len(unitTbl) + 1)
for _measures in mulQns, divQns:
for _measure in _measures:
addQnameValue(modelXbrl.modelDocument, _measure)
_unit = modelXbrl.createUnit(mulQns, divQns, id=unitId, beforeSibling=topTupleFact)
except OIMException as ex:
modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs)
unitTbl[unitKey] = _unit
else:
_unit = None
attrs["contextRef"] = _cntx.id
if fact.get("value") is None:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact["value"]
if concept.isNumeric:
if _unit is None:
return # skip creating fact because unit was invalid
attrs["unitRef"] = _unit.id
if "accuracy" in attrs or attrs.get(XbrlConst.qnXsiNil, "false") != "true":
attrs["decimals"] = fact.get("accuracy", "INF")
else:
text = None #tuple
id = fact.get("id")
if id is not None:
attrs["id"] = fact["id"]
# is value a QName?
if concept.baseXbrliType == "QName":
addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes))
f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text, parent=parentModelFact, validate=False)
if id is not None and id in parentedFacts:
# create child facts
for i in sorted(parentedFacts[id],
key=lambda j: facts[j].get("aspects", EMPTYDICT).get(oimTupleOrder,0)):
createModelFact(facts[i], f, f if topTupleFact is None else topTupleFact)
# validate after creating tuple contents
xmlValidate(modelXbrl, f)
from arelle import ModelDocument, XmlUtil
from arelle.ModelValue import qname, dateTime, DATE
from arelle.ValidateXbrlCalcs import insignificantDigits
from arelle.XbrlConst import xbrli
try:
import regex as re
except ImportError:
import re
from collections import defaultdict
memNameNumPattern = re.compile(r"^([A-Za-z-]+)([0-9]+)$")
compTxmyNamespacePattern = re.compile(r"http://www.govtalk.gov.uk/uk/fr/tax/uk-hmrc-ct/[0-9-]{10}")
# capture background-image or list-style-image with URL function
styleImgUrlPattern = re.compile(r"[a-z]+-image:\s*url[(][^)]+[)]")
EMPTYDICT = {}
_6_APR_2008 = dateTime("2008-04-06", type=DATE)
commonMandatoryItems = {
"EntityCurrentLegalOrRegisteredName", "StartDateForPeriodCoveredByReport",
"EndDateForPeriodCoveredByReport", "BalanceSheetDate"}
mandatoryItems = {
"ukGAAP": commonMandatoryItems | {
"DateApprovalAccounts", "NameDirectorSigningAccounts", "EntityDormant", "EntityTrading",
"DateSigningDirectorsReport", "DirectorSigningReport"},
"charities": commonMandatoryItems | {
"DateApprovalAccounts", "NameTrusteeSigningAccounts", "EntityDormant", "EntityTrading",
"DateSigningTrusteesReport", "TrusteeSigningReport"},
"ukIFRS": commonMandatoryItems | {
"DateAuthorisationFinancialStatementsForIssue", "ExplanationOfBodyOfAuthorisation",
"EntityDormant", "EntityTrading", "DateSigningDirectorsReport", "DirectorSigningReport"},
"FRS": commonMandatoryItems | {
def atomize(self, p, x):
# sequence
if isinstance(x, (tuple,list,set)):
sequence = []
for item in self.flattenSequence(x):
atomizedItem = self.atomize(p, item)
if atomizedItem != []:
sequence.append(atomizedItem)
return sequence
# individual items
if isinstance(x, range):
return x
baseXsdType = None
e = None
if isinstance(x, ModelFact):
if x.isTuple:
raise XPathException(p, 'err:FOTY0012', _('Atomizing tuple {0} that does not have a typed value').format(x))
if x.isNil:
return []
baseXsdType = x.concept.baseXsdType
v = x.value # resolves default value
e = x
elif isinstance(x, ModelAttribute): # ModelAttribute is a tuple (below), check this first!
return x.xValue
else:
if isinstance(x, ModelObject):
e = x
if e is not None:
if e.get("{http://www.w3.org/2001/XMLSchema-instance}nil") == "true":
return []
try:
if e.xValid >= VALID:
return e.xValue
except AttributeError:
pass
modelXbrl = x.modelXbrl
modelConcept = modelXbrl.qnameConcepts.get(qname(x))
if modelConcept is not None:
baseXsdType = modelConcept.baseXsdType
v = XmlUtil.text(x)
if baseXsdType in ("decimal", "float", "double"):
try:
x = float(v)
except ValueError:
raise XPathException(p, 'err:FORG0001', _('Atomizing {0} to a {1} does not have a proper value').format(x,baseXsdType))
elif baseXsdType in ("integer",):
try:
x = int(v)
except ValueError:
raise XPathException(p, 'err:FORG0001', _('Atomizing {0} to an integer does not have a proper value').format(x))
elif baseXsdType == "boolean":
x = (v == "true" or v == "1")
elif baseXsdType == "QName" and e is not None:
x = qname(e, v)
elif baseXsdType == "anyURI":
x = anyURI(v.strip())
elif baseXsdType in ("normalizedString","token","language","NMTOKEN","Name","NCName","ID","IDREF","ENTITY"):
x = v.strip()
elif baseXsdType == "XBRLI_DATEUNION":
x = dateTime(v, type=DATEUNION)
elif baseXsdType == "date":
x = dateTime(v, type=DATE)
elif baseXsdType == "dateTime":
x = dateTime(v, type=DATETIME)
elif baseXsdType:
x = str(v)
return x
def atomize(self, p, x):
# sequence
if isinstance(x, SEQUENCE_TYPES):
sequence = []
for item in self.flattenSequence(x):
atomizedItem = self.atomize(p, item)
if atomizedItem != []:
sequence.append(atomizedItem)
return sequence
# individual items
if isinstance(x, _RANGE):
return x
baseXsdType = None
e = None
if isinstance(x, ModelFact):
if x.isTuple:
raise XPathException(
p, 'err:FOTY0012',
_('Atomizing tuple {0} that does not have a typed value').
format(x))
if x.isNil:
return []
baseXsdType = x.concept.baseXsdType
v = x.value # resolves default value
e = x
elif isinstance(
x, ModelAttribute
): # ModelAttribute is a tuple (below), check this first!
return x.xValue
else:
if isinstance(x, ModelObject):
e = x
if e is not None:
if getattr(e, "xValid", 0) == VALID_NO_CONTENT:
raise XPathException(
p, 'err:FOTY0012',
_('Atomizing element {0} that does not have a typed value'
).format(x))
if e.get("{http://www.w3.org/2001/XMLSchema-instance}nil"
) == "true":
return []
try:
if e.xValid >= VALID:
return e.xValue
except AttributeError:
pass
modelXbrl = x.modelXbrl
modelConcept = modelXbrl.qnameConcepts.get(qname(x))
if modelConcept is not None:
baseXsdType = modelConcept.baseXsdType
v = x.stringValue
if baseXsdType in ("float", "double"):
try:
x = float(v)
except ValueError:
raise XPathException(
p, 'err:FORG0001',
_('Atomizing {0} to a {1} does not have a proper value').
format(x, baseXsdType))
elif baseXsdType == "decimal":
try:
x = Decimal(v)
except InvalidOperation:
raise XPathException(
p, 'err:FORG0001',
_('Atomizing {0} to decimal does not have a proper value'))
elif baseXsdType in ("integer", "nonPositiveInteger",
"negativeInteger", "nonNegativeInteger",
"positiveInteger", "long", "unsignedLong", "int",
"unsignedInt", "short", "unsignedShort", "byte",
"unsignedByte"):
try:
x = _INT(v)
except ValueError:
raise XPathException(
p, 'err:FORG0001',
_('Atomizing {0} to an integer does not have a proper value'
).format(x))
elif baseXsdType == "boolean":
x = (v == "true" or v == "1")
elif baseXsdType == "QName" and e is not None:
x = qname(e, v)
elif baseXsdType == "anyURI":
x = anyURI(v.strip())
elif baseXsdType in ("normalizedString", "token", "language",
"NMTOKEN", "Name", "NCName", "ID", "IDREF",
"ENTITY"):
x = v.strip()
elif baseXsdType == "XBRLI_DATEUNION":
x = dateTime(v, type=DATEUNION)
elif baseXsdType == "date":
x = dateTime(v, type=DATE)
elif baseXsdType == "dateTime":
x = dateTime(v, type=DATETIME)
elif baseXsdType in GREGORIAN_TYPES and isinstance(v, GREGORIAN_TYPES):
x = v
elif baseXsdType == "noContent":
x = None # can't be atomized
elif baseXsdType:
x = str(v)
return x
def atomize(self, p, x):
# sequence
if hasattr(x, '__iter__') and not isinstance(x, str):
sequence = []
for item in self.flattenSequence(x):
atomizedItem = self.atomize(p, item)
if atomizedItem != []:
sequence.append(atomizedItem)
return sequence
# individual items
if isinstance(x, range):
return x
baseXsdType = None
e = None
if isinstance(x, ModelObject.ModelFact):
if x.isTuple:
raise XPathException(
p, 'err:FOTY0012',
_('Atomizing tuple {0} that does not have a typed value').
format(x))
if x.isNil:
return []
baseXsdType = x.concept.baseXsdType
v = x.value # resolves default value
e = x.element
else:
if isinstance(x, ModelObject.ModelObject):
e = x.element
elif isinstance(x, xml.dom.Node):
e = x
if e:
if x.nodeType == xml.dom.Node.ELEMENT_NODE:
if e.getAttributeNS(XbrlConst.xsi, "nil") == "true":
return []
modelXbrl = x.ownerDocument.modelDocument.modelXbrl
modelConcept = modelXbrl.qnameConcepts.get(qname(x))
if modelConcept:
baseXsdType = modelConcept.baseXsdType
v = XmlUtil.text(x)
elif x.nodeType == xml.dom.Node.ATTRIBUTE_NODE:
e = x.ownerElement
modelXbrl = e.ownerDocument.modelDocument.modelXbrl
if x.namespaceURI:
attrQname = qname(x.namespaceURI, x.localName)
else:
attrQname = qname(x.localName)
modelConcept = modelXbrl.qnameConcepts.get(qname(e))
if modelConcept:
baseXsdType = modelConcept.baseXsdAttrType(
attrQname) if modelConcept else None
if baseXsdType is None:
attrObject = modelXbrl.qnameAttributes.get(attrQname)
if attrObject:
baseXsdType = attrObject.baseXsdType
v = x.value
if baseXsdType in ("decimal", "float", "double"):
try:
x = float(v)
except ValueError:
raise XPathException(
p, 'err:FORG0001',
_('Atomizing {0} to a {1} does not have a proper value').
format(x, baseXsdType))
elif baseXsdType in ("integer", ):
try:
x = int(v)
except ValueError:
raise XPathException(
p, 'err:FORG0001',
_('Atomizing {0} to an integer does not have a proper value'
).format(x))
elif baseXsdType == "boolean":
x = (v == "true" or v == "1")
elif baseXsdType == "QName" and e:
x = qname(e, v)
elif baseXsdType == "anyURI":
x = anyURI(v.strip())
elif baseXsdType in ("normalizedString", "token", "language",
"NMTOKEN", "Name", "NCName", "ID", "IDREF",
"ENTITY"):
x = v.strip()
elif baseXsdType == "XBRLI_DATEUNION":
x = dateTime(v, type=DATEUNION)
elif baseXsdType == "date":
x = dateTime(v, type=DATE)
elif baseXsdType == "dateTime":
x = dateTime(v, type=DATETIME)
elif baseXsdType:
x = str(v)
return x
def validateValue(modelXbrl,
elt,
attrTag,
baseXsdType,
value,
isNillable=False,
facets=None):
if baseXsdType:
try:
if (len(value) == 0 and not attrTag is None and not isNillable
and baseXsdType
not in ("anyType", "string", "normalizedString", "token",
"NMTOKEN", "anyURI", "noContent")):
raise ValueError("missing value for not nillable element")
xValid = VALID
whitespaceReplace = (baseXsdType == "normalizedString")
whitespaceCollapse = (not whitespaceReplace
and baseXsdType != "string")
pattern = baseXsdTypePatterns.get(baseXsdType)
if facets:
if "pattern" in facets:
pattern = facets["pattern"]
# note multiple patterns are or'ed togetner, which isn't yet implemented!
if "whiteSpace" in facets:
whitespaceReplace, whitespaceCollapse = {
"preserve": (False, False),
"replace": (True, False),
"collapse": (False, True)
}
if whitespaceReplace:
value = normalizeWhitespacePattern.sub(' ', value)
elif whitespaceCollapse:
value = collapseWhitespacePattern.sub(' ', value.strip())
if pattern is not None and pattern.match(value) is None:
raise ValueError("pattern facet " +
facets["pattern"].pattern if facets and
"pattern" in facets else "pattern mismatch")
if facets:
if "enumeration" in facets and value not in facets[
"enumeration"]:
raise ValueError("is not in {1}".format(
value, facets["enumeration"]))
if "length" in facets and len(value) != facets["length"]:
raise ValueError("length {0}, expected {1}".format(
len(value), facets["length"]))
if "minLength" in facets and len(value) < facets["minLength"]:
raise ValueError("length {0}, minLength {1}".format(
len(value), facets["minLength"]))
if "maxLength" in facets and len(value) > facets["maxLength"]:
raise ValueError("length {0}, maxLength {1}".format(
len(value), facets["maxLength"]))
if baseXsdType == "noContent":
if len(value) > 0 and not value.isspace():
raise ValueError("value content not permitted")
xValue = sValue = None
elif baseXsdType in {
"string", "normalizedString", "language", "token",
"NMTOKEN", "Name", "NCName", "IDREF", "ENTITY"
}:
xValue = sValue = value
elif baseXsdType == "ID":
xValue = sValue = value
xValid = VALID_ID
elif baseXsdType == "anyURI":
xValue = anyURI(value)
sValue = value
if xValue and not UrlUtil.isValid(
xValue): # allow empty strings to be valid anyURIs
raise ValueError("invalid anyURI value")
elif not value: # rest of types get None if nil/empty value
xValue = sValue = None
elif baseXsdType in ("decimal", "float", "double"):
xValue = sValue = float(value)
if facets:
if "totalDigits" in facets and len(value.replace(
".", "")) > facets["totalDigits"]:
raise ValueError("totalDigits facet {0}".format(
facets["totalDigits"]))
if "fractionDigits" in facets and (
'.' in value and len(value[value.index('.') + 1:])
> facets["fractionDigits"]):
raise ValueError("fraction digits facet {0}".format(
facets["fractionDigits"]))
elif baseXsdType in ("integer", ):
xValue = sValue = int(value)
elif baseXsdType == "boolean":
if value in ("true", "1"):
xValue = sValue = True
elif value in ("false", "0"):
xValue = sValue = False
else:
raise ValueError
elif baseXsdType == "QName":
xValue = qname(elt, value, castException=ValueError)
sValue = value
''' not sure here, how are explicitDimensions validated, but bad units not?
if xValue.namespaceURI in modelXbrl.namespaceDocs:
if (xValue not in modelXbrl.qnameConcepts and
xValue not in modelXbrl.qnameTypes and
xValue not in modelXbrl.qnameAttributes and
xValue not in modelXbrl.qnameAttributeGroups):
raise ValueError("qname not defined " + str(xValue))
'''
elif baseXsdType in ("XBRLI_DECIMALSUNION",
"XBRLI_PRECISIONUNION"):
xValue = sValue = value if value == "INF" else int(value)
elif baseXsdType in ("XBRLI_NONZERODECIMAL"):
xValue = sValue = int(value)
if xValue == 0:
raise ValueError("invalid value")
elif baseXsdType == "XBRLI_DATEUNION":
xValue = dateTime(value,
type=DATEUNION,
castException=ValueError)
sValue = value
elif baseXsdType == "dateTime":
xValue = dateTime(value,
type=DATETIME,
castException=ValueError)
sValue = value
elif baseXsdType == "date":
xValue = dateTime(value, type=DATE, castException=ValueError)
sValue = value
elif baseXsdType == "regex-pattern":
# for facet compiling
try:
xValue = re.compile(value + "$") # must match whole string
sValue = value
except Exception as err:
raise ValueError(err)
else:
if baseXsdType in lexicalPatterns:
match = lexicalPatterns[baseXsdType].match(value)
if match is None:
raise ValueError("lexical pattern mismatch")
if baseXsdType == "gMonthDay":
month, day, zSign, zHrMin, zHr, zMin = match.groups()
if int(day) > {
2: 29,
4: 30,
6: 30,
9: 30,
11: 30,
1: 31,
3: 31,
5: 31,
7: 31,
8: 31,
10: 31,
12: 31
}[int(month)]:
raise ValueError(
"invalid day {0} for month {1}".format(
day, month))
xValue = value
sValue = value
except ValueError as err:
if ModelInlineFact is not None and isinstance(
elt, ModelInlineFact):
errElt = "{0} fact {1}".format(elt.elementQname, elt.qname)
else:
errElt = elt.elementQname
if attrTag:
modelXbrl.error(
"xmlSchema:valueError",
_("Element %(element)s attribute %(attribute)s type %(typeName)s value error: %(value)s, %(error)s"
),
modelObject=elt,
element=errElt,
attribute=XmlUtil.clarkNotationToPrefixedName(
elt, attrTag, isAttribute=True),
typeName=baseXsdType,
value=value,
error=err)
else:
modelXbrl.error(
"xmlSchema:valueError",
_("Element %(element)s type %(typeName)s value error: %(value)s, %(error)s"
),
modelObject=elt,
element=errElt,
typeName=baseXsdType,
value=value,
error=err)
xValue = None
sValue = value
xValid = INVALID
else:
xValue = sValue = None
xValid = UNKNOWN
if attrTag:
elt.xAttributes[attrTag] = ModelAttribute(elt, attrTag, xValid, xValue,
sValue, value)
else:
elt.xValid = xValid
elt.xValue = xValue
elt.sValue = sValue
def atomize(self, p, x):
# sequence
if hasattr(x, '__iter__') and not isinstance(x, str):
sequence = []
for item in self.flattenSequence(x):
atomizedItem = self.atomize(p, item)
if atomizedItem != []:
sequence.append(atomizedItem)
return sequence
# individual items
if isinstance(x, range):
return x
baseXsdType = None
e = None
if isinstance(x, ModelObject.ModelFact):
if x.isTuple:
raise XPathException(p, 'err:FOTY0012', _('Atomizing tuple {0} that does not have a typed value').format(x))
if x.isNil:
return []
baseXsdType = x.concept.baseXsdType
v = x.value # resolves default value
e = x.element
else:
if isinstance(x, ModelObject.ModelObject):
e = x.element
elif isinstance(x, xml.dom.Node):
e = x
if e:
if x.nodeType == xml.dom.Node.ELEMENT_NODE:
if e.getAttributeNS(XbrlConst.xsi,"nil") == "true":
return []
modelXbrl = x.ownerDocument.modelDocument.modelXbrl
modelConcept = modelXbrl.qnameConcepts.get(qname(x))
if modelConcept:
baseXsdType = modelConcept.baseXsdType
v = XmlUtil.text(x)
elif x.nodeType == xml.dom.Node.ATTRIBUTE_NODE:
e = x.ownerElement
modelXbrl = e.ownerDocument.modelDocument.modelXbrl
if x.namespaceURI:
attrQname = qname(x.namespaceURI, x.localName)
else:
attrQname = qname(x.localName)
modelConcept = modelXbrl.qnameConcepts.get(qname(e))
if modelConcept:
baseXsdType = modelConcept.baseXsdAttrType(attrQname) if modelConcept else None
if baseXsdType is None:
attrObject = modelXbrl.qnameAttributes.get(attrQname)
if attrObject:
baseXsdType = attrObject.baseXsdType
v = x.value
if baseXsdType in ("decimal", "float", "double"):
try:
x = float(v)
except ValueError:
raise XPathException(p, 'err:FORG0001', _('Atomizing {0} to a {1} does not have a proper value').format(x,baseXsdType))
elif baseXsdType in ("integer",):
try:
x = int(v)
except ValueError:
raise XPathException(p, 'err:FORG0001', _('Atomizing {0} to an integer does not have a proper value').format(x))
elif baseXsdType == "boolean":
x = (v == "true" or v == "1")
elif baseXsdType == "QName" and e:
x = qname(e, v)
elif baseXsdType == "anyURI":
x = anyURI(v.strip())
elif baseXsdType in ("normalizedString","token","language","NMTOKEN","Name","NCName","ID","IDREF","ENTITY"):
x = v.strip()
elif baseXsdType == "XBRLI_DATEUNION":
x = dateTime(v, type=DATEUNION)
elif baseXsdType == "date":
x = dateTime(v, type=DATE)
elif baseXsdType == "dateTime":
x = dateTime(v, type=DATETIME)
elif baseXsdType:
x = str(v)
return x