Example #1
0
def quickbooksGLrequest(qbReport=None, file=None):
    from arelle.CntlrQuickBooks import supportedQbReports, qbRequest 
    from arelle.ModelValue import dateTime
    errors = []
    requestPathParts = request.urlparts[2].split('/')
    viewRequested = "view" == requestPathParts[-1]
    media = request.query.media or 'html'
    fromDate = request.query.fromDate
    toDate = request.query.toDate
    if qbReport not in supportedQbReports:
        errors.append(_("QuickBooks report '{0}' is not supported (please select from: {1})").format(
                          qbReport, ', '.join(supportedQbReports)))
    if media not in ('xml', 'xhtml', 'html'):
        errors.append(_("Media '{0}' is not supported for xbrl-gl (please select xhtml, html or xml)").format(media))
    if not fromDate or dateTime(fromDate) is None:
        errors.append(_("FromDate '{0}' missing or not valid").format(fromDate))
    if not toDate or dateTime(toDate) is None:
        errors.append(_("ToDate '{0}' missing or not valid").format(toDate))
    if errors:
        return errorReport(errors, media)
    ticket = qbRequest(qbReport, fromDate, toDate, file)
    result = htmlBody(tableRows([_("Request queued for QuickBooks...")], header=_("Quickbooks Request")), script='''
<script type="text/javascript">
<!-- 
var timer = setInterval("autoRefresh()", 1000 * 10);
function autoRefresh(){{location.href = "/rest/quickbooks/response?ticket={0}&media={1}&view={2}";}}
//--> 
</script>
'''.format(ticket, media, viewRequested))
    return result
Example #2
0
def fn_dateTime(xc, p, contextItem, args):
    if len(args) != 2: raise XPathContext.FunctionNumArgs()
    date = anytypeArg(xc, args, 0, "xs:date", missingArgFallback=())
    time = anytypeArg(xc, args, 1, "xs:time", missingArgFallback=())
    if date is None or time is None:
        return ()
    return dateTime(date) + dayTimeDuration(time)
Example #3
0
def parsed_value(fact: ModelFact):
    if fact is None:
        return None
    concept = fact.concept  # type: ModelConcept
    if concept is None or concept.isTuple or fact.isNil:
        return None
    if concept.isFraction:
        num, den = map(fractions.Fraction, fact.fractionValue)
        return num / den
    val = fact.value.strip()
    if concept.isInteger:
        return int(val)
    elif concept.isNumeric:
        dec = fact.decimals
        if dec is None or dec == "INF":  # show using decimals or reported format
            dec = len(val.partition(".")[2])
        else:  # max decimals at 28
            dec = max(
                min(int(dec), 28), -28
            )  # 2.7 wants short int, 3.2 takes regular int, don't use _INT here
        num = roundValue(val, fact.precision,
                         dec)  # round using reported decimals
        return num
    elif concept.baseXbrliType == 'dateItemType':
        return dateTime(val)
    elif concept.baseXbrliType == 'booleanItemType':
        return val.lower() in ('1', 'true')
    elif concept.isTextBlock:
        return ' '.join(val.split())
    return val
Example #4
0
 def checkEntries(self):
     errors = []
     if not self.cellFeed.value in rssFeeds and not isValidAbsolute(self.cellFeed.value):
         errors.append(_("RSS feed field contents invalid"))
     try:
         if self.cellMatchText.value:
             re.compile(self.cellMatchText.value)
     except Exception as err:
         errors.append(_("Match text field contents error {0}").format(err))
     if self.cellFormulaFile.value and not os.path.exists(self.cellFormulaFile.value):
         errors.append(_("Formula file not found {0}").format(self.cellFormulaFile.value))
     if self.cellLogFile.value and not os.path.exists(os.path.dirname(self.cellLogFile.value)):
         errors.append(_("Log file directory not found {0}").format(self.cellLogFile.value))
     if self.cellEmailAddress.value:
         if not emailPattern.match(self.cellEmailAddress.value):
             errors.append(_("E-mail address format error").format(self.cellLogFile.value))
         if not self.options.get("smtpEmailSettings"):
             errors.append(_("Please enter E-mail server settings (SMTP)"))
     if self.cellLatestPubDate.value and dateTime(self.cellLatestPubDate.value) is None:
         errors.append(_("Latest pub date field contents invalid"))
     if errors:
         messagebox.showwarning(_("Dialog validation error(s)"),
                             "\n ".join(errors), parent=self)
         return False
     return True
Example #5
0
def validateNode(modelXbrl, elt, node, baseXsdType, value):
    if baseXsdType:
        try:
            if baseXsdType in ("decimal", "float", "double"):
                node.xValue = float(value)
            elif baseXsdType in ("integer", ):
                node.xValue = int(value)
            elif baseXsdType == "boolean":
                if value in ("true", "1"): node.xValue = True
                elif value in ("false", "0"): node.xValue = False
                else: raise ValueError
            elif baseXsdType == "QName":
                node.xValue = qname(elt, value, castException=ValueError)
            elif baseXsdType in ("normalizedString", "token", "language",
                                 "NMTOKEN", "Name", "NCName", "ID", "IDREF",
                                 "ENTITY"):
                node.xValue = value.strip()
            elif baseXsdType == "dateTime":
                node.xValue = dateTime(value,
                                       type=DATETIME,
                                       castException=ValueError)
            elif baseXsdType == "date":
                node.xValue = dateTime(value,
                                       type=DATE,
                                       castException=ValueError)
            else:
                node.xValue = value
            node.xValid = VALID
        except ValueError:
            if node.nodeType == 1:
                modelXbrl.error(
                    _("Element {0} type {1} value error: {2}").format(
                        elt.tagName, baseXsdType, value), "err",
                    "xmlSchema:valueError")
            else:
                modelXbrl.error(
                    _("Element {0} attribute {1} type {2} value error: {3}"
                      ).format(elt.tagName, node.name, baseXsdType, value),
                    "err", "xmlSchema:valueError")
            node.xValue = None
            node.xValid = INVALID
    else:
        node.xValue = None
        node.xValid = UNKNOWN
Example #6
0
def validateNode(modelXbrl, elt, node, baseXsdType, value):
    if baseXsdType:
        try:
            if baseXsdType in ("decimal", "float", "double"):
                node.xValue = float(value)
            elif baseXsdType in ("integer",):
                node.xValue = int(value)
            elif baseXsdType == "boolean":
                if value in ("true", "1"): node.xValue = True
                elif value in ("false", "0"): node.xValue = False
                else: raise ValueError
            elif baseXsdType == "QName":
                node.xValue = qname(elt, value, castException=ValueError)
            elif baseXsdType in ("normalizedString","token","language","NMTOKEN","Name","NCName","ID","IDREF","ENTITY"):
                node.xValue = value.strip()
            elif baseXsdType == "dateTime":
                node.xValue = dateTime(value, type=DATETIME, castException=ValueError)
            elif baseXsdType == "date":
                node.xValue = dateTime(value, type=DATE, castException=ValueError)
            else:
                node.xValue = value
            node.xValid = VALID
        except ValueError:
            if node.nodeType == 1:
                modelXbrl.error(
                    _("Element {0} type {1} value error: {2}").format(
                    elt.tagName,
                    baseXsdType,
                    value),
                    "err", "xmlSchema:valueError")
            else:
                modelXbrl.error(
                    _("Element {0} attribute {1} type {2} value error: {3}").format(
                    elt.tagName,
                    node.name,
                    baseXsdType,
                    value),
                    "err", "xmlSchema:valueError")
            node.xValue = None
            node.xValid = INVALID
    else:
        node.xValue = None
        node.xValid = UNKNOWN
Example #7
0
def quickbooksGLrequest(qbReport=None, file=None):
    """Initiate request to QuickBooks server for *get* requests to */rest/quickbooks/<qbReport>/xbrl-gl/...*.
    
    :returns: html, xml, csv, text -- Return per media type argument and request arguments
    """
    from arelle.CntlrQuickBooks import supportedQbReports, qbRequest
    from arelle.ModelValue import dateTime

    errors = []
    requestPathParts = request.urlparts[2].split("/")
    viewRequested = "view" == requestPathParts[-1]
    media = request.query.media or "html"
    fromDate = request.query.fromDate
    toDate = request.query.toDate
    if qbReport not in supportedQbReports:
        errors.append(
            _("QuickBooks report '{0}' is not supported (please select from: {1})").format(
                qbReport, ", ".join(supportedQbReports)
            )
        )
    if media not in ("xml", "xhtml", "html"):
        errors.append(_("Media '{0}' is not supported for xbrl-gl (please select xhtml, html or xml)").format(media))
    if not fromDate or dateTime(fromDate) is None:
        errors.append(_("FromDate '{0}' missing or not valid").format(fromDate))
    if not toDate or dateTime(toDate) is None:
        errors.append(_("ToDate '{0}' missing or not valid").format(toDate))
    if errors:
        return errorReport(errors, media)
    ticket = qbRequest(qbReport, fromDate, toDate, file)
    result = htmlBody(
        tableRows([_("Request queued for QuickBooks...")], header=_("Quickbooks Request")),
        script="""
<script type="text/javascript">
<!-- 
var timer = setInterval("autoRefresh()", 1000 * 10);
function autoRefresh(){{location.href = "/rest/quickbooks/response?ticket={0}&media={1}&view={2}";}}
//--> 
</script>
""".format(
            ticket, media, viewRequested
        ),
    )
    return result
 def checkEntries(self):
     errors = []
     if not self.cellEntityIdentScheme.value:
         errors.append(_("Entity scheme invalid"))
     if not self.cellEntityIdentValue.value:
         errors.append(_("Entity identifier value invalid"))
     if not self.cellStartDate.value or dateTime(self.cellStartDate.value) is None:
         errors.append(_("Start date invalid"))
     if not self.cellEndDate.value or dateTime(self.cellEndDate.value) is None:
         errors.append(_("End date invalid"))
     if self.cellMonetaryUnit.value not in monetaryUnits:
         errors.append(_("Monetary unit invalid"))
     if not decimalsPattern.match(self.cellMonetaryDecimals.value):
         errors.append(_("Monetary decimals invalid"))
     if not decimalsPattern.match(self.cellNonMonetaryDecimals.value):
         errors.append(_("Non-monetary decimals invalid"))
     if errors:
         messagebox.showwarning(_("Dialog validation error(s)"),
                             "\n ".join(errors), parent=self)
         return False
     return True
Example #9
0
def period_datetime(p, args, periodElement, addOneDay):
    if len(args) != 1: raise XPathContext.FunctionNumArgs()
    if len(args[0]) != 1: raise XPathContext.FunctionArgType(1,"xbrl:period")
    period = args[0][0]
    if (isinstance(period,xml.dom.Node) and period.nodeType == 1 and 
        period.localName == "period" and period.namespaceURI == XbrlConst.xbrli):
        child = XmlUtil.child(period, XbrlConst.xbrli, periodElement)
        if child:
            return dateTime( child, addOneDay=addOneDay, type=DATETIME)
        elif periodElement == "instant":
            raise XPathContext.XPathException(p, 'xfie:PeriodIsNotInstant', _('Period is not instant'))
        else:
            raise XPathContext.XPathException(p, 'xfie:PeriodIsForever', _('Period is forever'))
    raise XPathContext.FunctionArgType(1,"xbrl:period")
Example #10
0
def period_datetime(p, args, periodElement, addOneDay):
    if len(args) != 1: raise XPathContext.FunctionNumArgs()
    if len(args[0]) != 1: raise XPathContext.FunctionArgType(1, "xbrl:period")
    period = args[0][0]
    if (isinstance(period, xml.dom.Node) and period.nodeType == 1
            and period.localName == "period"
            and period.namespaceURI == XbrlConst.xbrli):
        child = XmlUtil.child(period, XbrlConst.xbrli, periodElement)
        if child:
            return dateTime(child, addOneDay=addOneDay, type=DATETIME)
        elif periodElement == "instant":
            raise XPathContext.XPathException(p, 'xfie:PeriodIsNotInstant',
                                              _('Period is not instant'))
        else:
            raise XPathContext.XPathException(p, 'xfie:PeriodIsForever',
                                              _('Period is forever'))
    raise XPathContext.FunctionArgType(1, "xbrl:period")
Example #11
0
def validateValue(modelXbrl, elt, attrTag, baseXsdType, value, isNillable=False, isNil=False, facets=None):
    if baseXsdType:
        try:
            u'''
            if (len(value) == 0 and attrTag is None and not isNillable and 
                baseXsdType not in ("anyType", "string", "normalizedString", "token", "NMTOKEN", "anyURI", "noContent")):
                raise ValueError("missing value for not nillable element")
            '''
            xValid = VALID
            whitespaceReplace = (baseXsdType == u"normalizedString")
            whitespaceCollapse = (not whitespaceReplace and baseXsdType != u"string")
            isList = baseXsdType in set([u"IDREFS", u"ENTITIES", u"NMTOKENS"])
            if isList:
                baseXsdType = baseXsdType[:-1] # remove plural
            pattern = baseXsdTypePatterns.get(baseXsdType)
            if facets:
                if u"pattern" in facets:
                    pattern = facets[u"pattern"]
                    # note multiple patterns are or'ed togetner, which isn't yet implemented!
                if u"whiteSpace" in facets:
                    whitespaceReplace, whitespaceCollapse = {u"preserve":(False,False), u"replace":(True,False), u"collapse":(False,True)}[facets[u"whiteSpace"]]
            if whitespaceReplace:
                value = normalizeWhitespacePattern.sub(u' ', value)
            elif whitespaceCollapse:
                value = collapseWhitespacePattern.sub(u' ', value.strip())
            if baseXsdType == u"noContent":
                if len(value) > 0 and not value.isspace():
                    raise ValueError(u"value content not permitted")
                # note that sValue and xValue are not innerText but only text elements on specific element (or attribute)
                xValue = sValue = None
                xValid = VALID_NO_CONTENT # notify others that element may contain subelements (for stringValue needs)
            elif not value and isNil and isNillable: # rest of types get None if nil/empty value
                xValue = sValue = None
            else:
                if pattern is not None:
                    if ((isList and any(pattern.match(v) is None for v in value.split())) or
                        (not isList and pattern.match(value) is None)):
                        raise ValueError(u"pattern facet " + facets[u"pattern"].pattern if facets and u"pattern" in facets else u"pattern mismatch")
                if facets:
                    if u"enumeration" in facets and value not in facets[u"enumeration"]:
                        raise ValueError(u"{0} is not in {1}".format(value, facets[u"enumeration"]))
                    if u"length" in facets and len(value) != facets[u"length"]:
                        raise ValueError(u"length {0}, expected {1}".format(len(value), facets[u"length"]))
                    if u"minLength" in facets and len(value) < facets[u"minLength"]:
                        raise ValueError(u"length {0}, minLength {1}".format(len(value), facets[u"minLength"]))
                    if u"maxLength" in facets and len(value) > facets[u"maxLength"]:
                        raise ValueError(u"length {0}, maxLength {1}".format(len(value), facets[u"maxLength"]))
                if baseXsdType in set([u"string", u"normalizedString", u"language", u"token", u"NMTOKEN",u"Name",u"NCName",u"IDREF",u"ENTITY"]):
                    xValue = sValue = value
                elif baseXsdType == u"ID":
                    xValue = sValue = value
                    xValid = VALID_ID
                elif baseXsdType == u"anyURI":
                    if value:  # allow empty strings to be valid anyURIs
                        if UrlUtil.relativeUrlPattern.match(value) is None:
                            raise ValueError(u"IETF RFC 2396 4.3 syntax")
                    # encode PSVI xValue similarly to Xerces and other implementations
                    xValue = anyURI(UrlUtil.anyUriQuoteForPSVI(value))
                    sValue = value
                elif baseXsdType in (u"decimal", u"float", u"double"):
                    if baseXsdType == u"decimal":
                        if decimalPattern.match(value) is None:
                            raise ValueError(u"lexical pattern mismatch")
                        xValue = Decimal(value)
                        sValue = float(value) # s-value uses Number (float) representation
                    else:
                        if floatPattern.match(value) is None:
                            raise ValueError(u"lexical pattern mismatch")
                        xValue = sValue = float(value)
                    if facets:
                        if u"totalDigits" in facets and len(value.replace(u".",u"")) > facets[u"totalDigits"]:
                            raise ValueError(u"totalDigits facet {0}".format(facets[u"totalDigits"]))
                        if u"fractionDigits" in facets and ( u'.' in value and
                            len(value[value.index(u'.') + 1:]) > facets[u"fractionDigits"]):
                            raise ValueError(u"fraction digits facet {0}".format(facets[u"fractionDigits"]))
                        if u"maxInclusive" in facets and xValue > facets[u"maxInclusive"]:
                            raise ValueError(u" > maxInclusive {0}".format(facets[u"maxInclusive"]))
                        if u"maxExclusive" in facets and xValue >= facets[u"maxExclusive"]:
                            raise ValueError(u" >= maxInclusive {0}".format(facets[u"maxExclusive"]))
                        if u"minInclusive" in facets and xValue < facets[u"minInclusive"]:
                            raise ValueError(u" < minInclusive {0}".format(facets[u"minInclusive"]))
                        if u"minExclusive" in facets and xValue <= facets[u"minExclusive"]:
                            raise ValueError(u" <= minExclusive {0}".format(facets[u"minExclusive"]))
                elif baseXsdType in set([u"integer",
                                     u"nonPositiveInteger",u"negativeInteger",u"nonNegativeInteger",u"positiveInteger",
                                     u"long",u"unsignedLong",
                                     u"int",u"unsignedInt",
                                     u"short",u"unsignedShort",
                                     u"byte",u"unsignedByte"]):
                    xValue = sValue = _INT(value)
                    if ((baseXsdType in set([u"nonNegativeInteger",u"unsignedLong",u"unsignedInt"]) 
                         and xValue < 0) or
                        (baseXsdType == u"nonPositiveInteger" and xValue > 0) or
                        (baseXsdType == u"positiveInteger" and xValue <= 0) or
                        (baseXsdType == u"byte" and not -128 <= xValue < 127) or
                        (baseXsdType == u"unsignedByte" and not 0 <= xValue < 255) or
                        (baseXsdType == u"short" and not -32768 <= xValue < 32767) or
                        (baseXsdType == u"unsignedShort" and not 0 <= xValue < 65535) or
                        (baseXsdType == u"positiveInteger" and xValue <= 0)):
                        raise ValueError(u"{0} is not {1}".format(value, baseXsdType))
                    if facets:
                        if u"totalDigits" in facets and len(value.replace(u".",u"")) > facets[u"totalDigits"]:
                            raise ValueError(u"totalDigits facet {0}".format(facets[u"totalDigits"]))
                        if u"fractionDigits" in facets and ( u'.' in value and
                            len(value[value.index(u'.') + 1:]) > facets[u"fractionDigits"]):
                            raise ValueError(u"fraction digits facet {0}".format(facets[u"fractionDigits"]))
                        if u"maxInclusive" in facets and xValue > facets[u"maxInclusive"]:
                            raise ValueError(u" > maxInclusive {0}".format(facets[u"maxInclusive"]))
                        if u"maxExclusive" in facets and xValue >= facets[u"maxExclusive"]:
                            raise ValueError(u" >= maxInclusive {0}".format(facets[u"maxExclusive"]))
                        if u"minInclusive" in facets and xValue < facets[u"minInclusive"]:
                            raise ValueError(u" < minInclusive {0}".format(facets[u"minInclusive"]))
                        if u"minExclusive" in facets and xValue <= facets[u"minExclusive"]:
                            raise ValueError(u" <= minExclusive {0}".format(facets[u"minExclusive"]))
                elif baseXsdType == u"boolean":
                    if value in (u"true", u"1"):  
                        xValue = sValue = True
                    elif value in (u"false", u"0"): 
                        xValue = sValue = False
                    else: raise ValueError
                elif baseXsdType == u"QName":
                    xValue = qnameEltPfxName(elt, value, prefixException=ValueError)
                    #xValue = qname(elt, value, castException=ValueError, prefixException=ValueError)
                    sValue = value
                    u''' not sure here, how are explicitDimensions validated, but bad units not?
                    if xValue.namespaceURI in modelXbrl.namespaceDocs:
                        if (xValue not in modelXbrl.qnameConcepts and 
                            xValue not in modelXbrl.qnameTypes and
                            xValue not in modelXbrl.qnameAttributes and
                            xValue not in modelXbrl.qnameAttributeGroups):
                            raise ValueError("qname not defined " + str(xValue))
                    '''
                elif baseXsdType in (u"XBRLI_DECIMALSUNION", u"XBRLI_PRECISIONUNION"):
                    xValue = sValue = value if value == u"INF" else _INT(value)
                elif baseXsdType in (u"XBRLI_NONZERODECIMAL"):
                    xValue = sValue = _INT(value)
                    if xValue == 0:
                        raise ValueError(u"invalid value")
                elif baseXsdType == u"XBRLI_DATEUNION":
                    xValue = dateTime(value, type=DATEUNION, castException=ValueError)
                    sValue = value
                elif baseXsdType == u"dateTime":
                    xValue = dateTime(value, type=DATETIME, castException=ValueError)
                    sValue = value
                elif baseXsdType == u"date":
                    xValue = dateTime(value, type=DATE, castException=ValueError)
                    sValue = value
                elif baseXsdType == u"regex-pattern":
                    # for facet compiling
                    try:
                        sValue = value
                        if value in xmlSchemaPatterns:
                            xValue = xmlSchemaPatterns[value]
                        else:
                            if ur"\i" in value or ur"\c" in value:
                                value = value.replace(ur"\i", iNameChar).replace(ur"\c", cNameChar)
                            xValue = re_compile(value + u"$") # must match whole string
                    except Exception, err:
                        raise ValueError(err)
                else:
                    if baseXsdType in lexicalPatterns:
                        match = lexicalPatterns[baseXsdType].match(value)
                        if match is None:
                            raise ValueError(u"lexical pattern mismatch")
                        if baseXsdType == u"gMonthDay":
                            month, day, zSign, zHrMin, zHr, zMin = match.groups()
                            if int(day) > {2:29, 4:30, 6:30, 9:30, 11:30, 1:31, 3:31, 5:31, 7:31, 8:31, 10:31, 12:31}[int(month)]:
                                raise ValueError(u"invalid day {0} for month {1}".format(day, month))
                            xValue = gMonthDay(month, day)
                        elif baseXsdType == u"gYearMonth":
                            year, month, zSign, zHrMin, zHr, zMin = match.groups()
                            xValue = gYearMonth(year, month)
                        elif baseXsdType == u"gYear":
                            year, zSign, zHrMin, zHr, zMin = match.groups()
                            xValue = gYear(year)
                        elif baseXsdType == u"gMonth":
                            month, zSign, zHrMin, zHr, zMin = match.groups()
                            xValue = gMonth(month)
                        elif baseXsdType == u"gDay":
                            day, zSign, zHrMin, zHr, zMin = match.groups()
                            xValue = gDay(day)
                        else:
                            xValue = value
                    else: # no lexical pattern, forget compiling value
                        xValue = value
                    sValue = value
Example #12
0
def validateValue(modelXbrl,
                  elt,
                  attrTag,
                  baseXsdType,
                  value,
                  isNillable=False,
                  isNil=False,
                  facets=None):
    if baseXsdType:
        try:
            '''
            if (len(value) == 0 and attrTag is None and not isNillable and 
                baseXsdType not in ("anyType", "string", "normalizedString", "token", "NMTOKEN", "anyURI", "noContent")):
                raise ValueError("missing value for not nillable element")
            '''
            xValid = VALID
            whitespaceReplace = (baseXsdType == "normalizedString")
            whitespaceCollapse = (not whitespaceReplace
                                  and baseXsdType != "string")
            isList = baseXsdType in {"IDREFS", "ENTITIES", "NMTOKENS"}
            if isList:
                baseXsdType = baseXsdType[:-1]  # remove plural
                if facets:
                    if "minLength" not in facets:
                        facets = facets.copy()
                        facets["minLength"] = 1
                else:
                    facets = {"minLength": 1}
            pattern = baseXsdTypePatterns.get(baseXsdType)
            if facets:
                if "pattern" in facets:
                    pattern = facets["pattern"]
                    # note multiple patterns are or'ed togetner, which isn't yet implemented!
                if "whiteSpace" in facets:
                    whitespaceReplace, whitespaceCollapse = {
                        "preserve": (False, False),
                        "replace": (True, False),
                        "collapse": (False, True)
                    }[facets["whiteSpace"]]
            if whitespaceReplace:
                value = normalizeWhitespacePattern.sub(
                    ' ', value)  # replace tab, line feed, return with space
            elif whitespaceCollapse:
                value = collapseWhitespacePattern.sub(' ', value).strip(
                    ' '
                )  # collapse multiple spaces, tabs, line feeds and returns to single space
            if baseXsdType == "noContent":
                if len(value) > 0 and not entirelyWhitespacePattern.match(
                        value):  # only xml schema pattern whitespaces removed
                    raise ValueError("value content not permitted")
                # note that sValue and xValue are not innerText but only text elements on specific element (or attribute)
                xValue = sValue = None
                xValid = VALID_NO_CONTENT  # notify others that element may contain subelements (for stringValue needs)
            elif not value and isNil and isNillable:  # rest of types get None if nil/empty value
                xValue = sValue = None
            else:
                if pattern is not None:
                    if ((isList and any(
                            pattern.match(v) is None for v in value.split()))
                            or (not isList and pattern.match(value) is None)):
                        raise ValueError(
                            "pattern facet " +
                            facets["pattern"].pattern if facets
                            and "pattern" in facets else "pattern mismatch")
                if facets:
                    if "enumeration" in facets and value not in facets[
                            "enumeration"]:
                        raise ValueError("{0} is not in {1}".format(
                            value, facets["enumeration"].keys()))
                    if "length" in facets and len(value) != facets["length"]:
                        raise ValueError("length {0}, expected {1}".format(
                            len(value), facets["length"]))
                    if "minLength" in facets and len(
                            value) < facets["minLength"]:
                        raise ValueError("length {0}, minLength {1}".format(
                            len(value), facets["minLength"]))
                    if "maxLength" in facets and len(
                            value) > facets["maxLength"]:
                        raise ValueError("length {0}, maxLength {1}".format(
                            len(value), facets["maxLength"]))
                if baseXsdType in {
                        "string", "normalizedString", "language",
                        "languageOrEmpty", "token", "NMTOKEN", "Name",
                        "NCName", "IDREF", "ENTITY"
                }:
                    xValue = sValue = value
                elif baseXsdType == "ID":
                    xValue = sValue = value
                    xValid = VALID_ID
                elif baseXsdType == "anyURI":
                    if value:  # allow empty strings to be valid anyURIs
                        if UrlUtil.relativeUrlPattern.match(value) is None:
                            raise ValueError("IETF RFC 2396 4.3 syntax")
                    # encode PSVI xValue similarly to Xerces and other implementations
                    xValue = anyURI(UrlUtil.anyUriQuoteForPSVI(value))
                    sValue = value
                elif baseXsdType in ("decimal", "float", "double",
                                     "XBRLI_NONZERODECIMAL"):
                    if baseXsdType in ("decimal", "XBRLI_NONZERODECIMAL"):
                        if decimalPattern.match(value) is None:
                            raise ValueError("lexical pattern mismatch")
                        xValue = Decimal(value)
                        sValue = float(
                            value
                        )  # s-value uses Number (float) representation
                        if sValue == 0 and baseXsdType == "XBRLI_NONZERODECIMAL":
                            raise ValueError("zero is not allowed")
                    else:
                        if floatPattern.match(value) is None:
                            raise ValueError("lexical pattern mismatch")
                        xValue = sValue = float(value)
                    if facets:
                        if "totalDigits" in facets and len(
                                value.replace(".",
                                              "")) > facets["totalDigits"]:
                            raise ValueError("totalDigits facet {0}".format(
                                facets["totalDigits"]))
                        if "fractionDigits" in facets and (
                                '.' in value
                                and len(value[value.index('.') + 1:]) >
                                facets["fractionDigits"]):
                            raise ValueError(
                                "fraction digits facet {0}".format(
                                    facets["fractionDigits"]))
                        if "maxInclusive" in facets and xValue > facets[
                                "maxInclusive"]:
                            raise ValueError(" > maxInclusive {0}".format(
                                facets["maxInclusive"]))
                        if "maxExclusive" in facets and xValue >= facets[
                                "maxExclusive"]:
                            raise ValueError(" >= maxInclusive {0}".format(
                                facets["maxExclusive"]))
                        if "minInclusive" in facets and xValue < facets[
                                "minInclusive"]:
                            raise ValueError(" < minInclusive {0}".format(
                                facets["minInclusive"]))
                        if "minExclusive" in facets and xValue <= facets[
                                "minExclusive"]:
                            raise ValueError(" <= minExclusive {0}".format(
                                facets["minExclusive"]))
                elif baseXsdType in {
                        "integer", "nonPositiveInteger", "negativeInteger",
                        "nonNegativeInteger", "positiveInteger", "long",
                        "unsignedLong", "int", "unsignedInt", "short",
                        "unsignedShort", "byte", "unsignedByte"
                }:
                    xValue = sValue = _INT(value)
                    if ((baseXsdType in {
                            "nonNegativeInteger", "unsignedLong", "unsignedInt"
                    } and xValue < 0) or
                        (baseXsdType == "nonPositiveInteger" and xValue > 0) or
                        (baseXsdType == "positiveInteger" and xValue <= 0) or
                        (baseXsdType == "byte" and not -128 <= xValue < 127)
                            or (baseXsdType == "unsignedByte"
                                and not 0 <= xValue < 255)
                            or (baseXsdType == "short"
                                and not -32768 <= xValue < 32767)
                            or (baseXsdType == "unsignedShort"
                                and not 0 <= xValue < 65535) or
                        (baseXsdType == "positiveInteger" and xValue <= 0)):
                        raise ValueError("{0} is not {1}".format(
                            value, baseXsdType))
                    if facets:
                        if "totalDigits" in facets and len(
                                value.replace(".",
                                              "")) > facets["totalDigits"]:
                            raise ValueError("totalDigits facet {0}".format(
                                facets["totalDigits"]))
                        if "fractionDigits" in facets and (
                                '.' in value
                                and len(value[value.index('.') + 1:]) >
                                facets["fractionDigits"]):
                            raise ValueError(
                                "fraction digits facet {0}".format(
                                    facets["fractionDigits"]))
                        if "maxInclusive" in facets and xValue > facets[
                                "maxInclusive"]:
                            raise ValueError(" > maxInclusive {0}".format(
                                facets["maxInclusive"]))
                        if "maxExclusive" in facets and xValue >= facets[
                                "maxExclusive"]:
                            raise ValueError(" >= maxInclusive {0}".format(
                                facets["maxExclusive"]))
                        if "minInclusive" in facets and xValue < facets[
                                "minInclusive"]:
                            raise ValueError(" < minInclusive {0}".format(
                                facets["minInclusive"]))
                        if "minExclusive" in facets and xValue <= facets[
                                "minExclusive"]:
                            raise ValueError(" <= minExclusive {0}".format(
                                facets["minExclusive"]))
                elif baseXsdType == "boolean":
                    if value in ("true", "1"):
                        xValue = sValue = True
                    elif value in ("false", "0"):
                        xValue = sValue = False
                    else:
                        raise ValueError
                elif baseXsdType == "QName":
                    xValue = qnameEltPfxName(elt,
                                             value,
                                             prefixException=ValueError)
                    #xValue = qname(elt, value, castException=ValueError, prefixException=ValueError)
                    sValue = value
                    ''' not sure here, how are explicitDimensions validated, but bad units not?
                    if xValue.namespaceURI in modelXbrl.namespaceDocs:
                        if (xValue not in modelXbrl.qnameConcepts and 
                            xValue not in modelXbrl.qnameTypes and
                            xValue not in modelXbrl.qnameAttributes and
                            xValue not in modelXbrl.qnameAttributeGroups):
                            raise ValueError("qname not defined " + str(xValue))
                    '''
                elif baseXsdType == "enumerationHrefs":
                    xValue = [qnameHref(href) for href in value.split()]
                    sValue = value
                elif baseXsdType == "enumerationQNames":
                    xValue = [
                        qnameEltPfxName(elt, qn, prefixException=ValueError)
                        for qn in value.split()
                    ]
                    sValue = value
                elif baseXsdType in ("XBRLI_DECIMALSUNION",
                                     "XBRLI_PRECISIONUNION"):
                    xValue = sValue = value if value == "INF" else _INT(value)
                elif baseXsdType in ("XBRLI_NONZERODECIMAL"):
                    xValue = sValue = _INT(value)
                    if xValue == 0:
                        raise ValueError("invalid value")
                elif baseXsdType == "regex-pattern":
                    # for facet compiling
                    try:
                        sValue = value
                        if value in xmlSchemaPatterns:
                            xValue = xmlSchemaPatterns[value]
                        else:
                            if r"\i" in value or r"\c" in value:
                                value = value.replace(r"[\i-[:]]", iNameChar).replace(r"\i", iNameChar) \
                                              .replace(r"[\c-[:]]", cMinusCNameChar).replace(r"\c", cNameChar)
                            xValue = re_compile(value +
                                                "$")  # must match whole string
                    except Exception as err:
                        raise ValueError(err)
                elif baseXsdType == "fraction":
                    numeratorStr, denominatorStr = elt.fractionValue
                    if numeratorStr == INVALIDixVALUE or denominatorStr == INVALIDixVALUE:
                        sValue = xValue = INVALIDixVALUE
                        xValid = INVALID
                    else:
                        sValue = value
                        numeratorNum = float(numeratorStr)
                        denominatorNum = float(denominatorStr)
                        if numeratorNum.is_integer(
                        ) and denominatorNum.is_integer():
                            xValue = Fraction(int(numeratorNum),
                                              int(denominatorNum))
                        else:
                            xValue = Fraction(numeratorNum / denominatorNum)
                else:
                    if baseXsdType in lexicalPatterns:
                        match = lexicalPatterns[baseXsdType].match(value)
                        if match is None:
                            raise ValueError("lexical pattern mismatch")
                        if baseXsdType == "XBRLI_DATEUNION":
                            xValue = dateTime(value,
                                              type=DATEUNION,
                                              castException=ValueError)
                            sValue = value
                        elif baseXsdType == "dateTime":
                            xValue = dateTime(value,
                                              type=DATETIME,
                                              castException=ValueError)
                            sValue = value
                        elif baseXsdType == "date":
                            xValue = dateTime(value,
                                              type=DATE,
                                              castException=ValueError)
                            sValue = value
                        elif baseXsdType == "gMonthDay":
                            month, day, zSign, zHrMin, zHr, zMin = match.groups(
                            )
                            if int(day) > {
                                    2: 29,
                                    4: 30,
                                    6: 30,
                                    9: 30,
                                    11: 30,
                                    1: 31,
                                    3: 31,
                                    5: 31,
                                    7: 31,
                                    8: 31,
                                    10: 31,
                                    12: 31
                            }[int(month)]:
                                raise ValueError(
                                    "invalid day {0} for month {1}".format(
                                        day, month))
                            xValue = gMonthDay(month, day)
                        elif baseXsdType == "gYearMonth":
                            year, month, zSign, zHrMin, zHr, zMin = match.groups(
                            )
                            xValue = gYearMonth(year, month)
                        elif baseXsdType == "gYear":
                            year, zSign, zHrMin, zHr, zMin = match.groups()
                            xValue = gYear(year)
                        elif baseXsdType == "gMonth":
                            month, zSign, zHrMin, zHr, zMin = match.groups()
                            xValue = gMonth(month)
                        elif baseXsdType == "gDay":
                            day, zSign, zHrMin, zHr, zMin = match.groups()
                            xValue = gDay(day)
                        elif baseXsdType == "duration":
                            xValue = isoDuration(value)
                        else:
                            xValue = value
                    else:  # no lexical pattern, forget compiling value
                        xValue = value
                    sValue = value
        except (ValueError, InvalidOperation) as err:
            if ModelInlineValueObject is not None and isinstance(
                    elt, ModelInlineValueObject):
                errElt = "{0} fact {1}".format(elt.elementQname, elt.qname)
            else:
                errElt = elt.elementQname
            if attrTag:
                modelXbrl.error(
                    "xmlSchema:valueError",
                    _("Element %(element)s attribute %(attribute)s type %(typeName)s value error: %(value)s, %(error)s"
                      ),
                    modelObject=elt,
                    element=errElt,
                    attribute=XmlUtil.clarkNotationToPrefixedName(
                        elt, attrTag, isAttribute=True),
                    typeName=baseXsdType,
                    value=strTruncate(value, 30),
                    error=err)
            else:
                modelXbrl.error(
                    "xmlSchema:valueError",
                    _("Element %(element)s type %(typeName)s value error: %(value)s, %(error)s"
                      ),
                    modelObject=elt,
                    element=errElt,
                    typeName=baseXsdType,
                    value=strTruncate(value, 30),
                    error=err)
            xValue = None
            sValue = value
            xValid = INVALID
    else:
        xValue = sValue = None
        xValid = UNKNOWN
    if attrTag:
        try:  # dynamically allocate attributes (otherwise given shared empty set)
            xAttributes = elt.xAttributes
        except AttributeError:
            elt.xAttributes = xAttributes = {}
        xAttributes[attrTag] = ModelAttribute(elt, attrTag, xValid, xValue,
                                              sValue, value)
    else:
        elt.xValid = xValid
        elt.xValue = xValue
        elt.sValue = sValue
Example #13
0
from arelle.ModelValue import qname, dateTime, DATE
from arelle.ValidateXbrlCalcs import inferredDecimals, rangeValue, insignificantDigits
from arelle.XbrlConst import xbrli, qnXbrliXbrl
try:
    import regex as re
except ImportError:
    import re
from collections import defaultdict

memNameNumPattern = re.compile(r"^([A-Za-z-]+)([0-9]+)$")
compTxmyNamespacePattern = re.compile(
    r"http://www.govtalk.gov.uk/uk/fr/tax/uk-hmrc-ct/[0-9-]{10}")
# capture background-image or list-style-image with URL function
styleImgUrlPattern = re.compile(r"[a-z]+-image:\s*url[(][^)]+[)]")
EMPTYDICT = {}
_6_APR_2008 = dateTime("2008-04-06", type=DATE)

commonMandatoryItems = {
    "EntityCurrentLegalOrRegisteredName", "StartDateForPeriodCoveredByReport",
    "EndDateForPeriodCoveredByReport", "BalanceSheetDate"
}
mandatoryItems = {
    "ukGAAP": commonMandatoryItems | {
        "DateApprovalAccounts", "NameDirectorSigningAccounts", "EntityDormant",
        "EntityTrading", "DateSigningDirectorsReport", "DirectorSigningReport"
    },
    "charities": commonMandatoryItems | {
        "DateAuthorisationFinancialStatementsForIssue",
        "DirectorSigningFinancialStatements", "EntityDormantTruefalse",
        "EntityTradingStatus", "AccountingStandardsApplied",
        "AccountingStandardsApplied"
Example #14
0
 def atomize(self, p, x):
     # sequence
     if isinstance(x, SEQUENCE_TYPES):
         sequence = []
         for item in self.flattenSequence(x):
             atomizedItem = self.atomize(p, item)
             if atomizedItem != []:
                 sequence.append(atomizedItem)
         return sequence
     # individual items
     if isinstance(x, _RANGE): 
         return x
     baseXsdType = None
     e = None
     if isinstance(x, ModelFact):
         if x.isTuple:
             raise XPathException(p, 'err:FOTY0012', _('Atomizing tuple {0} that does not have a typed value').format(x))
         if x.isNil:
             return []
         baseXsdType = x.concept.baseXsdType
         v = x.value # resolves default value
         e = x
     elif isinstance(x, ModelAttribute): # ModelAttribute is a tuple (below), check this first!
         return x.xValue
     else:
         if isinstance(x, ModelObject):
             e = x
         if e is not None:
             if getattr(e, "xValid", 0) == VALID_NO_CONTENT:
                 raise XPathException(p, 'err:FOTY0012', _('Atomizing element {0} that does not have a typed value').format(x))
             if e.get("{http://www.w3.org/2001/XMLSchema-instance}nil") == "true":
                 return []
             try:
                 if e.xValid >= VALID:
                     return e.xValue
             except AttributeError:
                 pass
             modelXbrl = x.modelXbrl
             modelConcept = modelXbrl.qnameConcepts.get(qname(x))
             if modelConcept is not None:
                 baseXsdType = modelConcept.baseXsdType
             v = x.stringValue
     if baseXsdType in ("float", "double"):
         try:
             x = float(v)
         except ValueError:
             raise XPathException(p, 'err:FORG0001', _('Atomizing {0} to a {1} does not have a proper value').format(x,baseXsdType))
     elif baseXsdType == "decimal":
         try:
             x = Decimal(v)
         except InvalidOperation:
             raise XPathException(p, 'err:FORG0001', _('Atomizing {0} to decimal does not have a proper value'))
     elif baseXsdType in ("integer",
                          "nonPositiveInteger","negativeInteger","nonNegativeInteger","positiveInteger",
                          "long","unsignedLong",
                          "int","unsignedInt",
                          "short","unsignedShort",
                          "byte","unsignedByte"):
         try:
             x = _INT(v)
         except ValueError:
             raise XPathException(p, 'err:FORG0001', _('Atomizing {0} to an integer does not have a proper value').format(x))
     elif baseXsdType == "boolean":
         x = (v == "true" or v == "1")
     elif baseXsdType == "QName" and e is not None:
         x = qname(e, v)
     elif baseXsdType == "anyURI":
         x = anyURI(v.strip())
     elif baseXsdType in ("normalizedString","token","language","NMTOKEN","Name","NCName","ID","IDREF","ENTITY"):
         x = v.strip()
     elif baseXsdType == "XBRLI_DATEUNION":
         x = dateTime(v, type=DATEUNION)
     elif baseXsdType == "date":
         x = dateTime(v, type=DATE)
     elif baseXsdType == "dateTime":
         x = dateTime(v, type=DATETIME)
     elif baseXsdType == "noContent":
         x = None # can't be atomized
     elif baseXsdType:
         x = str(v)
     return x
Example #15
0
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri):
    from openpyxl import load_workbook
    from arelle import ModelDocument, ModelXbrl, XmlUtil
    from arelle.ModelDocument import ModelDocumentReference
    from arelle.ModelValue import qname
    
    try:
        currentAction = "initializing"
        startedAt = time.time()
        
        if os.path.isabs(oimFile):
            # allow relative filenames to loading directory
            priorCWD = os.getcwd()
            os.chdir(os.path.dirname(oimFile))
        else:
            priorCWD = None
            
        currentAction = "determining file type"
        isJSON = oimFile.endswith(".json") and not oimFile.endswith("-metadata.json")
        isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json")
        isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls")
        isCSVorXL = isCSV or isXL
        instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl"
        
        if isJSON:
            currentAction = "loading and parsing JSON OIM file"
            with io.open(oimFile, 'rt', encoding='utf-8') as f:
                oimObject = json.load(f)
            missing = [t for t in ("dtsReferences", "prefixes", "facts") if t not in oimObject]
            if missing:
                raise OIMException("oime:missingJSONelements", 
                                   _("The required elements %(missing)s {} in JSON input"),
                                   missing = ", ".join(missing))
            currentAction = "identifying JSON objects"
            dtsReferences = oimObject["dtsReferences"]
            prefixes = oimObject["prefixes"]
            facts = oimObject["facts"]
            footnotes = oimOject["facts"] # shares this object
        elif isCSV:
            currentAction = "identifying CSV input tables"
            if sys.version[0] >= '3':
                csvOpenMode = 'w'
                csvOpenNewline = ''
            else:
                csvOpenMode = 'wb' # for 2.7
                csvOpenNewline = None
                
            oimFileBase = None
            if "-facts" in oimFile:
                oimFileBase = oimFile.partition("-facts")[0]
            else:
                for suffix in ("-dtsReferences.csv", "-defaults.csv", "-prefixes.csv", "-footnotes.csv", "-metadata.json"):
                    if oimFile.endswith(suffix):
                        oimFileBase = oimFile[:-length(suffix)]
                        break
            if oimFileBase is None:
                raise OIMException("oime:missingCSVtables", 
                                   _("Unable to identify CSV tables file name pattern"))
            if (not os.path.exists(oimFileBase + "-dtsReferences.csv") or
                not os.path.exists(oimFileBase + "-prefixes.csv")):
                raise OIMException("oime:missingCSVtables", 
                                   _("Unable to identify CSV tables for dtsReferences or prefixes"))
            instanceFileName = oimFileBase + ".xbrl"
            currentAction = "loading CSV dtsReferences table"
            dtsReferences = []
            with io.open(oimFileBase + "-dtsReferences.csv", 'rt', encoding='utf-8-sig') as f:
                csvReader = csv.reader(f)
                for i, row in enumerate(csvReader):
                    if i == 0:
                        header = row
                    else:
                        dtsReferences.append(dict((header[j], col) for j, col in enumerate(row)))
            currentAction = "loading CSV prefixes table"
            prefixes = {}
            with io.open(oimFileBase + "-prefixes.csv", 'rt', encoding='utf-8-sig') as f:
                csvReader = csv.reader(f)
                for i, row in enumerate(csvReader):
                    if i == 0:
                        header = dict((col,i) for i,col in enumerate(row))
                    else:
                        prefixes[row[header["prefix"]]] = row[header["URI"]]
            currentAction = "loading CSV defaults table"
            defaults = {}
            if os.path.exists(oimFileBase + "-defaults.csv"):
                with io.open(oimFileBase + "-defaults.csv", 'rt', encoding='utf-8-sig') as f:
                    csvReader = csv.reader(f)
                    for i, row in enumerate(csvReader):
                        if i == 0:
                            header = row
                            fileCol = row.index("file")
                        else:
                            defaults[row[fileCol]] = dict((header[j], col) for j, col in enumerate(row) if j != fileCol)
            currentAction = "loading CSV facts tables"
            facts = []
            _dir = os.path.dirname(oimFileBase)
            for filename in os.listdir(_dir):
                filepath = os.path.join(_dir, filename)
                if "-facts" in filename and filepath.startswith(oimFileBase):
                    tableDefaults = defaults.get(filename, {})
                    with io.open(filepath, 'rt', encoding='utf-8-sig') as f:
                        csvReader = csv.reader(f)
                        for i, row in enumerate(csvReader):
                            if i == 0:
                                header = row
                            else:
                                fact = {}
                                fact.update(tableDefaults)
                                for j, col in enumerate(row):
                                    if col is not None:
                                        if header[j].endswith("Value"):
                                            if col: # ignore empty columns (= null CSV value)
                                                fact["value"] = col
                                        else:
                                            fact[header[j]] = col
                                facts.append(fact)
            footnotes = []
            if os.path.exists(oimFileBase + "-footnotes.csv"):
                with io.open(oimFileBase + "-footnotes.csv", 'rt', encoding='utf-8-sig') as f:
                    csvReader = csv.reader(f)
                    for i, row in enumerate(csvReader):
                        if i == 0:
                            header = row
                        else:
                            footnotes.append(dict((header[j], col) for j, col in enumerate(row) if col))
        elif isXL:
            oimWb = load_workbook(oimFile, read_only=True, data_only=True)
            sheetNames = oimWb.get_sheet_names()
            if (not any(sheetName == "prefixes" for sheetName in sheetNames) or
                not any(sheetName == "dtsReferences" for sheetName in sheetNames) or
                not any("facts" in sheetName for sheetName in sheetNames)):
                if priorCWD: os.chdir(priorCWD)
                return None
            try:
                dtsReferences = []
                for i, row in oimWb["dtsReferences"]:
                    if i == 0:
                        header = [col.value for col in row]
                    else:
                        dtsReferences.append(dict((header[j], col.value) for j, col in enumerate(row)))
                prefixes = {}
                for i, row in oimWb["dtsReferences"]:
                    if i == 0:
                        header = dict((col.value,i) for i,col in enumerate(row))
                    else:
                        prefixes[row[header["prefix"]].value] = row[header["URI"].value]
                defaults = {}
                for i, row in oimW.get("defaults", ()):
                    if i == 0:
                        header = dict((col.value,i) for i,col in enumerate(row))
                        fileCol = header["file"]
                    else:
                        defaults[row[fileCol].value] = dict((header[j], col.value) for j, col in enumerate(row) if j != fileCol)
                facts = []
                _dir = os.path.dirpart(oimFileBase)
                for sheetName in sheetNames:
                    if sheetName == "facts" or "-facts" in sheetName:
                        for i, row in oimWb[sheetName]:
                            if i == 0:
                                header = [col.value for col in row]
                            else:
                                fact = {}
                                fact.update(tableDefaults)
                                for j, col in enumerate(row):
                                    if col.value is not None:
                                        if header[j].endswith("Value"):
                                            fact["value"] = str(col.value)
                                        else:
                                            fact[header[j]] = str(col.value)
                                facts.append(fact)
                footnotes = []
                for i, row in oimWb.get("footnotes", ()):
                    if i == 0:
                        header = dict((col.value,i) for i,col in enumerate(row) if col.value)
                    else:
                        footnotes.append(dict((header[j], col.value) for j, col in enumerate(row) if col.value))
            except Exception as ex:
                if priorCWD: os.chdir(priorCWD)
                return None
    
        ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults 
            
        # create the instance document
        modelXbrl.blockDpmDBrecursion = True
        modelXbrl.modelDocument = createModelDocument(
              modelXbrl, 
              Type.INSTANCE,
              instanceFileName,
              schemaRefs=[dtsRef["href"] for dtsRef in dtsReferences if dtsRef["type"] == "schema"],
              isEntry=True,
              initialComment="extracted from OIM {}".format(mappedUri),
              documentEncoding="utf-8")
        cntxTbl = {}
        unitTbl = {}
        for fact in facts:
            conceptQn = qname(fact["oim:concept"], prefixes)
            concept = modelXbrl.qnameConcepts.get(conceptQn)
            entityAsQn = qname(fact["oim:entity"], prefixes)
            if "oim:period" in fact:
                periodStart = fact["oim:period"]["start"]
                periodEnd = fact["oim:period"]["end"]
            else:
                periodStart = fact["oim:periodStart"]
                periodEnd = fact["oim:periodEnd"]
            cntxKey = ( # hashable context key
                ("periodType", concept.periodType),
                ("entity", entityAsQn),
                ("periodStart", periodStart),
                ("periodEnd", periodEnd)) + tuple(sorted(
                    (dimName, dimVal) 
                    for dimName, dimVal in fact.items()
                    if ":" in dimName and not dimName.startswith("oim:")))
            if cntxKey in cntxTbl:
                _cntx = cntxTbl[cntxKey]
            else:
                cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
                qnameDims = {}
                for dimName, dimVal in fact.items():
                    if ":" in dimName and not dimName.startswith("oim:"):
                        dimQname = qname(dimName, prefixes)
                        dimConcept = modelXbrl.qnameConcepts.get(dimQname)
                        if ":" in dimVal and dimVal.partition[':'][0] in prefixes:
                            mem = qname(dimVal, prefixes) # explicit dim
                        elif dimConcept.isTypedDimension:
                            # a modelObject xml element is needed for all of the instance functions to manage the typed dim
                            mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False)
                        qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment")
                _cntx = modelXbrl.createContext(
                                        entityAsQn.namespaceURI,
                                        entityAsQn.localName,
                                        concept.periodType,
                                        None if concept.periodType == "instnat" else dateTime(periodStart, type=DATETIME),
                                        dateTime(periodEnd, type=DATETIME),
                                        None, # no dimensional validity checking (like formula does)
                                        qnameDims, [], [],
                                        id=cntxId)
                cntxTbl[cntxKey] = _cntx
            if "oim:unit" in fact:
                unitKey = fact["oim:unit"]
                if unitKey in unitTbl:
                    _unit = unitTbl[unitKey]
                else:
                    _mul, _sep, _div = unitKey.partition('/')
                    if _mul.startswith('('):
                        _mul = _mul[1:-1]
                    mulQns = [qname(u, prefixes) for u in _mul.split('*') if u]
                    if _div.startswith('('):
                        _div = _div[1:-1]
                    divQns = [qname(u, prefixes) for u in _div.split('*') if u]
                    unitId = 'u-{:02}'.format(len(unitTbl) + 1)
                    for _measures in mulQns, divQns:
                        for _measure in _measures:
                            addQnameValue(modelXbrl.modelDocument, _measure)
                    _unit = modelXbrl.createUnit(mulQns, divQns, id=unitId)
                    unitTbl[unitKey] = _unit
            else:
                _unit = None
            
            attrs = {"contextRef": _cntx.id}
    
            if fact.get("value") is None:
                attrs[XbrlConst.qnXsiNil] = "true"
                text = None
            else:
                text = fact["value"]
                
            if fact.get("id"):
                attrs["id"] = fact["id"]
                
            if concept.isNumeric:
                if _unit is not None:
                    attrs["unitRef"] = _unit.id
                if "accuracy" in fact:
                    attrs["decimals"] = fact["accuracy"]
                    
            # is value a QName?
            if concept.baseXbrliType == "QName":
                addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes))
    
            f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
            
        footnoteLinks = {} # ELR elements
        factLocs = {} # index by (linkrole, factId)
        footnoteNbr = 0
        locNbr = 0
        for factOrFootnote in footnotes:
            if "factId" in factOrFootnote:
                factId = factOrFootnote["factId"]
                factFootnotes = (factOrFootnote,) # CSV or XL
            elif "id" in factOrFootnote and "footnotes" in factOrFootnote:
                factId = factOrFootnote["id"]
                factFootnotes = factOrFootnote["footnotes"]
            else:
                factFootnotes = ()
            for footnote in factFootnotes:
                linkrole = footnote.get("group")
                arcrole = footnote.get("footnoteType")
                if not factId or not linkrole or not arcrole or not (
                    footnote.get("factRef") or footnote.get("footnote")):
                    # invalid footnote
                    continue
                if linkrole not in footnoteLinks:
                    footnoteLinks[linkrole] = addChild(modelXbrl.modelDocument.xmlRootElement, 
                                                       XbrlConst.qnLinkFootnoteLink, 
                                                       attributes={"{http://www.w3.org/1999/xlink}type": "extended",
                                                                   "{http://www.w3.org/1999/xlink}role": linkrole})
                footnoteLink = footnoteLinks[linkrole]
                if (linkrole, factId) not in factLocs:
                    locNbr += 1
                    locLabel = "l_{:02}".format(locNbr)
                    factLocs[(linkrole, factId)] = locLabel
                    addChild(footnoteLink, XbrlConst.qnLinkLoc, 
                             attributes={XLINKTYPE: "locator",
                                         XLINKHREF: "#" + factId,
                                         XLINKLABEL: locLabel})
                locLabel = factLocs[(linkrole, factId)]
                if footnote.get("footnote"):
                    footnoteNbr += 1
                    footnoteLabel = "f_{:02}".format(footnoteNbr)
                    attrs = {XLINKTYPE: "resource",
                             XLINKLABEL: footnoteLabel}
                    if footnote.get("language"):
                        attrs[XMLLANG] = footnote["language"]
                    # note, for HTML will need to build an element structure
                    addChild(footnoteLink, XbrlConst.qnLinkFootnote, attributes=attrs, text=footnote["footnote"])
                elif footnote.get("factRef"):
                    factRef = footnote.get("factRef")
                    if (linkrole, factRef) not in factLocs:
                        locNbr += 1
                        locLabel = "f_{:02}".format(footnoteNbr)
                        factLoc[(linkrole, factRef)] = locLabel
                        addChild(footnoteLink, XbrlConst.qnLinkLoc, 
                                 attributes={XLINKTYPE: "locator",
                                             XLINKHREF: "#" + factRef,
                                             XLINKLABEL: locLabel})
                    footnoteLabel = factLoc[(linkrole, factId)]
                footnoteArc = addChild(footnoteLink, 
                                       XbrlConst.qnLinkFootnoteArc, 
                                       attributes={XLINKTYPE: "arc",
                                                   XLINKARCROLE: arcrole,
                                                   XLINKFROM: locLabel,
                                                   XLINKTO: footnoteLabel})
                    
        
        #cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt),
        #               messageCode="loadFromExcel:info")
    except Exception as ex:
        if ex is OIMException:
            modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs)
        else:
            modelXbrl.error("Error while %(action)s, error %(error)s",
                            modelObject=modelXbrl, action=currentAction, error=ex)
    
    if priorCWD:
        os.chdir(priorCWD) # restore prior current working directory
    return modelXbrl.modelDocument
Example #16
0
def validateValue(modelXbrl,
                  elt,
                  attrTag,
                  baseXsdType,
                  value,
                  isNillable=False,
                  isNil=False,
                  facets=None):
    if baseXsdType:
        try:
            '''
            if (len(value) == 0 and attrTag is None and not isNillable and 
                baseXsdType not in ("anyType", "string", "normalizedString", "token", "NMTOKEN", "anyURI", "noContent")):
                raise ValueError("missing value for not nillable element")
            '''
            xValid = VALID
            whitespaceReplace = (baseXsdType == "normalizedString")
            whitespaceCollapse = (not whitespaceReplace
                                  and baseXsdType != "string")
            pattern = baseXsdTypePatterns.get(baseXsdType)
            if facets:
                if "pattern" in facets:
                    pattern = facets["pattern"]
                    # note multiple patterns are or'ed togetner, which isn't yet implemented!
                if "whiteSpace" in facets:
                    whitespaceReplace, whitespaceCollapse = {
                        "preserve": (False, False),
                        "replace": (True, False),
                        "collapse": (False, True)
                    }[facets["whiteSpace"]]
            if whitespaceReplace:
                value = normalizeWhitespacePattern.sub(' ', value)
            elif whitespaceCollapse:
                value = collapseWhitespacePattern.sub(' ', value.strip())
            if pattern is not None and pattern.match(value) is None:
                raise ValueError("pattern facet " +
                                 facets["pattern"].pattern if facets and
                                 "pattern" in facets else "pattern mismatch")
            if facets:
                if "enumeration" in facets and value not in facets[
                        "enumeration"]:
                    raise ValueError("{0} is not in {1}".format(
                        value, facets["enumeration"]))
                if "length" in facets and len(value) != facets["length"]:
                    raise ValueError("length {0}, expected {1}".format(
                        len(value), facets["length"]))
                if "minLength" in facets and len(value) < facets["minLength"]:
                    raise ValueError("length {0}, minLength {1}".format(
                        len(value), facets["minLength"]))
                if "maxLength" in facets and len(value) > facets["maxLength"]:
                    raise ValueError("length {0}, maxLength {1}".format(
                        len(value), facets["maxLength"]))
            if baseXsdType == "noContent":
                if len(value) > 0 and not value.isspace():
                    raise ValueError("value content not permitted")
                xValue = sValue = None
            elif baseXsdType in {
                    "string", "normalizedString", "language", "token",
                    "NMTOKEN", "Name", "NCName", "IDREF", "ENTITY"
            }:
                xValue = sValue = value
            elif baseXsdType == "ID":
                xValue = sValue = value
                xValid = VALID_ID
            elif baseXsdType == "anyURI":
                if value:  # allow empty strings to be valid anyURIs
                    if UrlUtil.relativeUrlPattern.match(value) is None:
                        raise ValueError("IETF RFC 2396 4.3 syntax")
                # encode PSVI xValue similarly to Xerces and other implementations
                xValue = anyURI(UrlUtil.anyUriQuoteForPSVI(value))
                sValue = value
            elif not value and isNil and isNillable:  # rest of types get None if nil/empty value
                xValue = sValue = None
            elif baseXsdType in ("decimal", "float", "double"):
                xValue = sValue = float(value)
                if facets:
                    if "totalDigits" in facets and len(value.replace(
                            ".", "")) > facets["totalDigits"]:
                        raise ValueError("totalDigits facet {0}".format(
                            facets["totalDigits"]))
                    if "fractionDigits" in facets and (
                            '.' in value and len(value[value.index('.') + 1:])
                            > facets["fractionDigits"]):
                        raise ValueError("fraction digits facet {0}".format(
                            facets["fractionDigits"]))
                    if "maxInclusive" in facets and xValue > facets[
                            "maxInclusive"]:
                        raise ValueError(" > maxInclusive {0}".format(
                            facets["maxInclusive"]))
                    if "maxExclusive" in facets and xValue >= facets[
                            "maxExclusive"]:
                        raise ValueError(" >= maxInclusive {0}".format(
                            facets["maxExclusive"]))
                    if "minInclusive" in facets and xValue < facets[
                            "minInclusive"]:
                        raise ValueError(" < minInclusive {0}".format(
                            facets["minInclusive"]))
                    if "minExclusive" in facets and xValue <= facets[
                            "minExclusive"]:
                        raise ValueError(" <= minExclusive {0}".format(
                            facets["minExclusive"]))
            elif baseXsdType in {
                    "integer", "nonPositiveInteger", "negativeInteger",
                    "nonNegativeInteger", "positiveInteger", "long",
                    "unsignedLong", "int", "unsignedInt", "short",
                    "unsignedShort", "byte", "unsignedByte"
            }:
                xValue = sValue = _INT(value)
                if ((baseXsdType
                     in {"nonNegativeInteger", "unsignedLong", "unsignedInt"}
                     and xValue < 0)
                        or (baseXsdType == "nonPositiveInteger" and xValue > 0)
                        or (baseXsdType == "positiveInteger" and xValue <= 0)
                        or (baseXsdType == "byte" and not -128 <= xValue < 127)
                        or
                    (baseXsdType == "unsignedByte" and not 0 <= xValue < 255)
                        or
                    (baseXsdType == "short" and not -32768 <= xValue < 32767)
                        or (baseXsdType == "unsignedShort"
                            and not 0 <= xValue < 65535)
                        or (baseXsdType == "positiveInteger" and xValue <= 0)):
                    raise ValueError("{0} is not {1}".format(
                        value, baseXsdType))
                if facets:
                    if "totalDigits" in facets and len(value.replace(
                            ".", "")) > facets["totalDigits"]:
                        raise ValueError("totalDigits facet {0}".format(
                            facets["totalDigits"]))
                    if "fractionDigits" in facets and (
                            '.' in value and len(value[value.index('.') + 1:])
                            > facets["fractionDigits"]):
                        raise ValueError("fraction digits facet {0}".format(
                            facets["fractionDigits"]))
                    if "maxInclusive" in facets and xValue > facets[
                            "maxInclusive"]:
                        raise ValueError(" > maxInclusive {0}".format(
                            facets["maxInclusive"]))
                    if "maxExclusive" in facets and xValue >= facets[
                            "maxExclusive"]:
                        raise ValueError(" >= maxInclusive {0}".format(
                            facets["maxExclusive"]))
                    if "minInclusive" in facets and xValue < facets[
                            "minInclusive"]:
                        raise ValueError(" < minInclusive {0}".format(
                            facets["minInclusive"]))
                    if "minExclusive" in facets and xValue <= facets[
                            "minExclusive"]:
                        raise ValueError(" <= minExclusive {0}".format(
                            facets["minExclusive"]))
            elif baseXsdType == "boolean":
                if value in ("true", "1"):
                    xValue = sValue = True
                elif value in ("false", "0"):
                    xValue = sValue = False
                else:
                    raise ValueError
            elif baseXsdType == "QName":
                xValue = qname(elt,
                               value,
                               castException=ValueError,
                               prefixException=ValueError)
                sValue = value
                ''' not sure here, how are explicitDimensions validated, but bad units not?
                if xValue.namespaceURI in modelXbrl.namespaceDocs:
                    if (xValue not in modelXbrl.qnameConcepts and 
                        xValue not in modelXbrl.qnameTypes and
                        xValue not in modelXbrl.qnameAttributes and
                        xValue not in modelXbrl.qnameAttributeGroups):
                        raise ValueError("qname not defined " + str(xValue))
                '''
            elif baseXsdType in ("XBRLI_DECIMALSUNION",
                                 "XBRLI_PRECISIONUNION"):
                xValue = sValue = value if value == "INF" else _INT(value)
            elif baseXsdType in ("XBRLI_NONZERODECIMAL"):
                xValue = sValue = _INT(value)
                if xValue == 0:
                    raise ValueError("invalid value")
            elif baseXsdType == "XBRLI_DATEUNION":
                xValue = dateTime(value,
                                  type=DATEUNION,
                                  castException=ValueError)
                sValue = value
            elif baseXsdType == "dateTime":
                xValue = dateTime(value,
                                  type=DATETIME,
                                  castException=ValueError)
                sValue = value
            elif baseXsdType == "date":
                xValue = dateTime(value, type=DATE, castException=ValueError)
                sValue = value
            elif baseXsdType == "regex-pattern":
                # for facet compiling
                try:
                    sValue = value
                    if value in xmlSchemaPatterns:
                        xValue = xmlSchemaPatterns[value]
                    else:
                        if r"\i" in value or r"\c" in value:
                            value = value.replace(r"\i", iNameChar).replace(
                                r"\c", cNameChar)
                        xValue = re.compile(value +
                                            "$")  # must match whole string
                except Exception as err:
                    raise ValueError(err)
            else:
                if baseXsdType in lexicalPatterns:
                    match = lexicalPatterns[baseXsdType].match(value)
                    if match is None:
                        raise ValueError("lexical pattern mismatch")
                    if baseXsdType == "gMonthDay":
                        month, day, zSign, zHrMin, zHr, zMin = match.groups()
                        if int(day) > {
                                2: 29,
                                4: 30,
                                6: 30,
                                9: 30,
                                11: 30,
                                1: 31,
                                3: 31,
                                5: 31,
                                7: 31,
                                8: 31,
                                10: 31,
                                12: 31
                        }[int(month)]:
                            raise ValueError(
                                "invalid day {0} for month {1}".format(
                                    day, month))
                xValue = value
                sValue = value
        except ValueError as err:
            if ModelInlineFact is not None and isinstance(
                    elt, ModelInlineFact):
                errElt = "{0} fact {1}".format(elt.elementQname, elt.qname)
            else:
                errElt = elt.elementQname
            if attrTag:
                modelXbrl.error(
                    "xmlSchema:valueError",
                    _("Element %(element)s attribute %(attribute)s type %(typeName)s value error: %(value)s, %(error)s"
                      ),
                    modelObject=elt,
                    element=errElt,
                    attribute=XmlUtil.clarkNotationToPrefixedName(
                        elt, attrTag, isAttribute=True),
                    typeName=baseXsdType,
                    value=value,
                    error=err)
            else:
                modelXbrl.error(
                    "xmlSchema:valueError",
                    _("Element %(element)s type %(typeName)s value error: %(value)s, %(error)s"
                      ),
                    modelObject=elt,
                    element=errElt,
                    typeName=baseXsdType,
                    value=value,
                    error=err)
            xValue = None
            sValue = value
            xValid = INVALID
    else:
        xValue = sValue = None
        xValid = UNKNOWN
    if attrTag:
        try:  # dynamically allocate attributes (otherwise given shared empty set)
            xAttributes = elt.xAttributes
        except AttributeError:
            elt.xAttributes = xAttributes = {}
        xAttributes[attrTag] = ModelAttribute(elt, attrTag, xValid, xValue,
                                              sValue, value)
    else:
        elt.xValid = xValid
        elt.xValue = xValue
        elt.sValue = sValue
Example #17
0
 def atomize(self, p, x):
     # sequence
     if isinstance(x, (tuple, list, set)):
         sequence = []
         for item in self.flattenSequence(x):
             atomizedItem = self.atomize(p, item)
             if atomizedItem != []:
                 sequence.append(atomizedItem)
         return sequence
     # individual items
     if isinstance(x, _RANGE):
         return x
     baseXsdType = None
     e = None
     if isinstance(x, ModelFact):
         if x.isTuple:
             raise XPathException(
                 p, 'err:FOTY0012',
                 _('Atomizing tuple {0} that does not have a typed value').
                 format(x))
         if x.isNil:
             return []
         baseXsdType = x.concept.baseXsdType
         v = x.value  # resolves default value
         e = x
     elif isinstance(
             x, ModelAttribute
     ):  # ModelAttribute is a tuple (below), check this first!
         return x.xValue
     else:
         if isinstance(x, ModelObject):
             e = x
         if e is not None:
             if e.get("{http://www.w3.org/2001/XMLSchema-instance}nil"
                      ) == "true":
                 return []
             try:
                 if e.xValid >= VALID:
                     return e.xValue
             except AttributeError:
                 pass
             modelXbrl = x.modelXbrl
             modelConcept = modelXbrl.qnameConcepts.get(qname(x))
             if modelConcept is not None:
                 baseXsdType = modelConcept.baseXsdType
             v = XmlUtil.text(x)
     if baseXsdType in ("decimal", "float", "double"):
         try:
             x = float(v)
         except ValueError:
             raise XPathException(
                 p, 'err:FORG0001',
                 _('Atomizing {0} to a {1} does not have a proper value').
                 format(x, baseXsdType))
     elif baseXsdType in ("integer", ):
         try:
             x = _INT(v)
         except ValueError:
             raise XPathException(
                 p, 'err:FORG0001',
                 _('Atomizing {0} to an integer does not have a proper value'
                   ).format(x))
     elif baseXsdType == "boolean":
         x = (v == "true" or v == "1")
     elif baseXsdType == "QName" and e is not None:
         x = qname(e, v)
     elif baseXsdType == "anyURI":
         x = anyURI(v.strip())
     elif baseXsdType in ("normalizedString", "token", "language",
                          "NMTOKEN", "Name", "NCName", "ID", "IDREF",
                          "ENTITY"):
         x = v.strip()
     elif baseXsdType == "XBRLI_DATEUNION":
         x = dateTime(v, type=DATEUNION)
     elif baseXsdType == "date":
         x = dateTime(v, type=DATE)
     elif baseXsdType == "dateTime":
         x = dateTime(v, type=DATETIME)
     elif baseXsdType:
         x = str(v)
     return x
Example #18
0
    def insertDataPoints(self):
        instanceId = self.instanceId
        self.showStatus("deleting prior data points of this report")
        for tableName in ("dFact", "dFilingIndicator", "dProcessingContext", "dProcessingFact"):
            self.execute("DELETE FROM {0} WHERE {0}.InstanceID = {1}"
                         .format( self.dbTableName(tableName), instanceId), 
                         close=False, fetch=False)
        self.showStatus("insert data points")
        # contexts
        def dimValKey(cntx, typedDim=False):
            return '|'.join(sorted("{}({})".format(dim.dimensionQname,
                                                   dim.memberQname if dim.isExplicit 
                                                   else xmlstring(dim.typedMember, stripXmlns=True) if typedDim
                                                   else '*' )
                                   for dim in cntx.qnameDims.values()))
        def dimNameKey(cntx):
            return '|'.join(sorted("{}".format(dim.dimensionQname)
                                   for dim in cntx.qnameDims.values()))
        contextSortedAllDims = dict((cntx.id, dimValKey(cntx))  # has (*) for typed dim
                                    for cntx in self.modelXbrl.contexts.values()
                                    if cntx.qnameDims)
        contextSortedTypedDims = dict((cntx.id, dimValKey(cntx, typedDim=True)) # typed dims with value only if typed
                                      for cntx in self.modelXbrl.contexts.values()
                                      if any(dim.isTyped for dim in cntx.qnameDims.values()))
        contextSortedDimNames = dict((cntx.id, dimNameKey(cntx))
                                     for cntx in self.modelXbrl.contexts.values()
                                     if cntx.qnameDims)
        
        def met(fact):
            return "MET({})".format(fact.qname)
        
        def metDimAllKey(fact):
            key = met(fact)
            if fact.contextID in contextSortedAllDims:
                key += '|' + contextSortedAllDims[fact.contextID]
            return key
        
        def metDimTypedKey(fact):
            if fact.contextID in contextSortedTypedDims:
                key = met(fact)
                if fact.contextID in contextSortedTypedDims:
                    key += '|' + contextSortedTypedDims[fact.contextID]
                return key
            return None
                
        def metDimNameKey(fact):
            key = met(fact)
            if fact.contextID in contextSortedDimNames:
                key += '|' + contextSortedDimNames[fact.contextID]
            return key
        
            
        table = self.getTable("dProcessingContext", None,
                              ('InstanceID', 'ContextID', 'SortedDimensions', 'NotValid'),
                              ('InstanceID', 'ContextID'),
                              tuple((instanceId,
                                     cntxID,
                                     cntxDimKey,
                                     False
                                     )
                                    for cntxID, cntxDimKey in sorted(contextSortedAllDims.items())))
        
        # contexts with typed dimensions
        
        # dCloseFactTable
        dFilingIndicators = set()
        dProcessingFacts = []
        dFacts = []
        for f in self.modelXbrl.factsInInstance:
            cntx = f.context
            concept = f.concept
            isNumeric = isBool = isDateTime = isText = False
            if concept is not None:
                if concept.isNumeric:
                    isNumeric = True
                else:
                    baseXbrliType = concept.baseXbrliType
                    if baseXbrliType == "booleanItemType":
                        isBool = True
                    elif baseXbrliType == "dateTimeItemType": # also is dateItemType?
                        isDateTime = True
                xValue = f.xValue
            else:
                if f.isNil:
                    xValue = None
                else:
                    xValue = f.value
                    c = f.qname.localName[0]
                    if c == 'm':
                        isNumeric = True
                        # not validated, do own xValue
                        try:
                            xValue = Decimal(xValue)
                        except InvalidOperation:
                            xValue = Decimal('NaN')
                    elif c == 'd':
                        isDateTime = True
                        try:
                            xValue = dateTime(xValue, type=DATEUNION, castException=ValueError)
                        except ValueError:
                            pass
                    elif c == 'b':
                        isBool = True
                        xValue = xValue.strip()
                        if xValue in ("true", "1"):  
                            xValue = True
                        elif xValue in ("false", "0"): 
                            xValue = False
                
            isText = not (isNumeric or isBool or isDateTime)
            if f.qname == qnFindFilingIndicators:
                for filingIndicator in f.modelTupleFacts:
                    if filingIndicator.qname == qnFindFilingIndicator:
                        dFilingIndicators.add(filingIndicator.textValue.strip())
            elif cntx is not None:
                dFacts.append((instanceId,
                               metDimAllKey(f),
                               metDimTypedKey(f),
                               str(f.unit.measures[0][0]) if isNumeric and f.unit is not None and f.unit.isSingleMeasure else None,
                               f.decimals,
                               xValue if isNumeric else None,
                               xValue if isDateTime else None,
                               xValue if isBool else None,
                               xValue if isText else None
                            ))
                dProcessingFacts.append((instanceId,
                                         met(f),
                                         f.contextID if isNumeric else None,                                         
                                         xValue if isText or isBool else None,
                                         xValue if isNumeric else None,
                                         xValue if isDateTime else None,
                                         None))
        # get filing indicator template IDs
        results = self.execute("SELECT mToT.TemplateOrTableCode, mToT.TemplateOrTableID "
                               "  FROM mModuleBusinessTemplate mBT, mTemplateOrTable mToT "
                               "  WHERE mBT.ModuleID = {0} AND"
                               "        mToT.TemplateOrTableID = mBT.BusinessTemplateID AND"
                               "        mToT.TemplateOrTableCode in ({1})"
                               .format(self.moduleId,
                                       ', '.join("'{}'".format(filingIndicator)
                                                 for filingIndicator in dFilingIndicators)))
        filingIndicatorCodeIDs = dict((code, id) for code, id in results)
        
        if _DICT_SET(filingIndicatorCodeIDs.keys()) != dFilingIndicators:
            self.modelXbrl.error("sqlDB:MissingFilingIndicators",
                                 _("The filing indicator IDs not found for codes %(missingFilingIndicatorCodes)"),
                                 modelObject=self.modelXbrl,
                                 missingFilingIndicatorCodes=','.join(dFilingIndicators - _DICT_SET(filingIndicatorCodeIDs.keys()))) 

        table = self.getTable("dFilingIndicator", None,
                              ("InstanceID", "BusinessTemplateID"),
                              ("InstanceID", "BusinessTemplateID"),
                              ((instanceId,
                                filingIndicatorCodeId)
                               for filingIndicatorCodeId in sorted(filingIndicatorCodeIDs.values())))

        
        table = self.getTable("dFact", None,
                              ('InstanceID', 'DataPointSignature', 'DataPointSignatureWithValuesForWildcards', 
                               'Unit', 'Decimals',
                               'NumericValue', 'DateTimeValue', 'BooleanValue', 'TextValue'),
                              ('InstanceID', ),
                              dFacts)
        table = self.getTable("dProcessingFact", None,
                              ('InstanceID', 'Metric', 'ContextID', 
                               'ValueTxt', 'ValueDecimal', 'ValueDate',
                               'Error'),
                              ('InstanceID', ),
                              dProcessingFacts)
Example #19
0
    def insertDataPoints(self):
        instanceId = self.instanceId
        self.showStatus("deleting prior data points of this report")
        for tableName in ("dFact", "dFilingIndicator", "dAvailableTable"):
            self.execute("DELETE FROM {0} WHERE {0}.InstanceID = {1}"
                         .format( self.dbTableName(tableName), instanceId), 
                         close=False, fetch=False)
            
        self.showStatus("obtaining mapping table information")
        result = self.execute("SELECT MetricAndDimensions, TableID FROM mTableDimensionSet WHERE ModuleID = {}"
                              .format(self.moduleId))
        tableIDs = set()
        metricAndDimensionsTableId = defaultdict(set)
        for metricAndDimensions, tableID in result:
            tableIDs.add(tableID)
            metricAndDimensionsTableId[metricAndDimensions].add(tableID)
            
        result = self.execute("SELECT TableID, YDimVal, ZDimVal FROM mTable WHERE TableID in ({})"
                              .format(', '.join(str(i) for i in sorted(tableIDs))))
        yDimVal = defaultdict(dict)
        zDimVal = defaultdict(dict)
        for tableID, yDimVals, zDimVals in result:
            for tblDimVal, dimVals in ((yDimVal, yDimVals), (zDimVal, zDimVals)):
                if dimVals:
                    for dimVal in dimVals.split('|'):
                        dim, sep, val = dimVal.partition('(')
                        tblDimVal[tableID][dim] = val[:-1]

        availableTableRows = defaultdict(set) # index (tableID, zDimVal) = set of yDimVals 
               
        self.showStatus("insert data points")
        # contexts
        emptySet = set()
        def dimValKey(cntx, typedDim=False, behaveAsTypedDims=emptySet, restrictToDims=None):
            return '|'.join(sorted("{}({})".format(dim.dimensionQname,
                                                   dim.memberQname if dim.isExplicit and dim not in behaveAsTypedDims
                                                   else dim.memberQname if typedDim and not dim.isTyped
                                                   else xmlstring(dim.typedMember, stripXmlns=True) if typedDim
                                                   else '*' )
                                   for dim in cntx.qnameDims.values()
                                   if not restrictToDims or str(dim.dimensionQname) in restrictToDims))
        def dimNameKey(cntx):
            return '|'.join(sorted("{}".format(dim.dimensionQname)
                                   for dim in cntx.qnameDims.values()))
        contextSortedAllDims = dict((cntx.id, dimValKey(cntx))  # has (*) for typed dim
                                    for cntx in self.modelXbrl.contexts.values()
                                    if cntx.qnameDims)
        contextSortedTypedDims = dict((cntx.id, dimValKey(cntx, typedDim=True)) # typed dims with value only if typed
                                      for cntx in self.modelXbrl.contexts.values()
                                      if any(dim.isTyped for dim in cntx.qnameDims.values()))
        contextSortedDimNames = dict((cntx.id, dimNameKey(cntx))
                                     for cntx in self.modelXbrl.contexts.values()
                                     if cntx.qnameDims)
        
        def met(fact):
            return "MET({})".format(fact.qname)
        
        # key for use in dFact with * for dim that behaves as or is typed
        def metDimAllKey(fact, behaveAsTypedDims=emptySet):
            key = met(fact)
            cntx = fact.context
            if cntx.qnameDims:
                key += '|' + dimValKey(cntx, behaveAsTypedDims=behaveAsTypedDims)
            return key
        
        # key for use in dFact only when there's a dim that behaves as or is typed
        def metDimTypedKey(fact, behaveAsTypedDims=emptySet):
            cntx = fact.context
            if any(dimQname in behaveAsTypedDims for dimQname in cntx.qnameDims):
                key = met(fact) + '|' + dimValKey(cntx, typedDim=True, behaveAsTypedDims=behaveAsTypedDims)
                return key
            return None
        
        # key for use in dAvailable where mem and typed values show up
        def metDimValKey(cntx, typedDim=False, behaveAsTypedDims=emptySet, restrictToDims=emptySet):
            if "MET" in restrictToDims:
                key = "MET({})|".format(restrictToDims["MET"])
            else:
                key = ""
            key += dimValKey(cntx, typedDim=typedDim, behaveAsTypedDims=behaveAsTypedDims, restrictToDims=restrictToDims)
            return key
                
        def metDimNameKey(fact):
            key = met(fact)
            if fact.contextID in contextSortedDimNames:
                key += '|' + contextSortedDimNames[fact.contextID]
            return key
        
        ''' deprecated
        table = self.getTable("dProcessingContext", None,
                              ('InstanceID', 'ContextID', 'SortedDimensions', 'NotValid'),
                              ('InstanceID', 'ContextID'),
                              tuple((instanceId,
                                     cntxID,
                                     cntxDimKey,
                                     False
                                     )
                                    for cntxID, cntxDimKey in sorted(contextSortedAllDims.items())))
        '''
        
        # contexts with typed dimensions
        
        # dCloseFactTable
        dFilingIndicators = set()
        # dProcessingFacts = []
        dFacts = []
        for f in self.modelXbrl.facts:
            cntx = f.context
            concept = f.concept
            isNumeric = isBool = isDateTime = isText = False
            if concept is not None:
                if concept.isNumeric:
                    isNumeric = True
                else:
                    baseXbrliType = concept.baseXbrliType
                    if baseXbrliType == "booleanItemType":
                        isBool = True
                    elif baseXbrliType == "dateTimeItemType": # also is dateItemType?
                        isDateTime = True
                xValue = f.xValue
            else:
                if f.isNil:
                    xValue = None
                else:
                    xValue = f.value
                    c = f.qname.localName[0]
                    if c == 'm':
                        isNumeric = True
                        # not validated, do own xValue
                        try:
                            xValue = Decimal(xValue)
                        except InvalidOperation:
                            xValue = Decimal('NaN')
                    elif c == 'd':
                        isDateTime = True
                        try:
                            xValue = dateTime(xValue, type=DATEUNION, castException=ValueError)
                        except ValueError:
                            pass
                    elif c == 'b':
                        isBool = True
                        xValue = xValue.strip()
                        if xValue in ("true", "1"):  
                            xValue = True
                        elif xValue in ("false", "0"): 
                            xValue = False
                
            isText = not (isNumeric or isBool or isDateTime)
            if f.qname == qnFindFilingIndicators:
                for filingIndicator in f.modelTupleFacts:
                    if filingIndicator.qname == qnFindFilingIndicator:
                        dFilingIndicators.add(filingIndicator.textValue.strip())
            elif cntx is not None:
                # find which explicit dimensions act as typed
                behaveAsTypedDims = set()
                zDimValues = {}
                tableID = None
                for tableID in metricAndDimensionsTableId.get(metDimNameKey(f), ()):
                    yDimVals = yDimVal[tableID]
                    zDimVals = zDimVal[tableID]
                    for dimQname in cntx.qnameDims.keys():
                        dimStr = str(dimQname)
                        if (dimStr in zDimVals and zDimVals[dimStr] == "*" or
                            dimStr in yDimVals and yDimVals[dimStr] == "*"):
                            behaveAsTypedDims.add(dimQname)
                    zDimKey = (metDimValKey(cntx, typedDim=True, behaveAsTypedDims=behaveAsTypedDims, restrictToDims=zDimVals)
                               or None)  # want None if no dimVal Z key
                    yDimKey = metDimValKey(cntx, typedDim=True, behaveAsTypedDims=behaveAsTypedDims, restrictToDims=yDimVals)
                    availableTableRows[tableID,zDimKey].add(yDimKey)
                    break
                dFacts.append((instanceId,
                               metDimAllKey(f, behaveAsTypedDims),
                               metDimTypedKey(f, behaveAsTypedDims),
                               str(f.unit.measures[0][0]) if isNumeric and f.unit is not None and f.unit.isSingleMeasure else None,
                               f.decimals,
                               xValue if isNumeric else None,
                               xValue if isDateTime else None,
                               xValue if isBool else None,
                               xValue if isText else None
                            ))
                ''' deprecated
                dProcessingFacts.append((instanceId,
                                         met(f),
                                         f.contextID if isNumeric else None,                                         
                                         xValue if isText or isBool else None,
                                         xValue if isNumeric else None,
                                         xValue if isDateTime else None,
                                         None))
                '''
                
                # availableTable processing
        # get filing indicator template IDs
        results = self.execute("SELECT mToT.TemplateOrTableCode, mToT.TemplateOrTableID "
                               "  FROM mModuleBusinessTemplate mBT, mTemplateOrTable mToT "
                               "  WHERE mBT.ModuleID = {0} AND"
                               "        mToT.TemplateOrTableID = mBT.BusinessTemplateID AND"
                               "        mToT.TemplateOrTableCode in ({1})"
                               .format(self.moduleId,
                                       ', '.join("'{}'".format(filingIndicator)
                                                 for filingIndicator in dFilingIndicators)))
        filingIndicatorCodeIDs = dict((code, id) for code, id in results)
        
        if _DICT_SET(filingIndicatorCodeIDs.keys()) != dFilingIndicators:
            self.modelXbrl.error("sqlDB:MissingFilingIndicators",
                                 _("The filing indicator IDs not found for codes %(missingFilingIndicatorCodes)"),
                                 modelObject=self.modelXbrl,
                                 missingFilingIndicatorCodes=','.join(dFilingIndicators - _DICT_SET(filingIndicatorCodeIDs.keys()))) 

        self.getTable("dFilingIndicator", None,
                      ("InstanceID", "BusinessTemplateID"),
                      ("InstanceID", "BusinessTemplateID"),
                      ((instanceId,
                        filingIndicatorCodeId)
                       for filingIndicatorCodeId in sorted(filingIndicatorCodeIDs.values())))

        
        self.getTable("dFact", None,
                      ('InstanceID', 'DataPointSignature', 'DataPointSignatureWithValuesForWildcards', 
                       'Unit', 'Decimals',
                       'NumericValue', 'DateTimeValue', 'BooleanValue', 'TextValue'),
                      ('InstanceID', ),
                      dFacts)
        ''' deprecated
        table = self.getTable("dProcessingFact", None,
                              ('InstanceID', 'Metric', 'ContextID', 
                               'ValueTxt', 'ValueDecimal', 'ValueDate',
                               'Error'),
                              ('InstanceID', ),
                              dProcessingFacts)
        '''
        self.getTable("dAvailableTable", None,
                      ('InstanceID', 'TableID', 'ZDimVal', "NumberOfRows"), 
                      ('InstanceID', 'TableID', 'ZDimVal'),
                      ((instanceId,
                        availTableKey[0], # table Id
                        availTableKey[1], # zDimVal
                        len(setOfYDimVals))
                       for availTableKey, setOfYDimVals in availableTableRows.items()))
Example #20
0
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri):
    from openpyxl import load_workbook
    from arelle import ModelDocument, ModelXbrl, XmlUtil
    from arelle.ModelDocument import ModelDocumentReference
    from arelle.ModelValue import qname

    try:
        currentAction = "initializing"
        startingErrorCount = len(modelXbrl.errors)
        startedAt = time.time()

        if os.path.isabs(oimFile):
            # allow relative filenames to loading directory
            priorCWD = os.getcwd()
            os.chdir(os.path.dirname(oimFile))
        else:
            priorCWD = None

        currentAction = "determining file type"
        isJSON = oimFile.endswith(
            ".json") and not oimFile.endswith("-metadata.json")
        isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json")
        isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls")
        isCSVorXL = isCSV or isXL
        instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl"

        if isJSON:
            currentAction = "loading and parsing JSON OIM file"

            def loadDict(keyValuePairs):
                _dict = OrderedDict(
                )  # preserve fact order in resulting instance
                for key, value in keyValuePairs:
                    if isinstance(value, dict):
                        if DUPJSONKEY in value:
                            for _errKey, _errValue, _otherValue in value[
                                    DUPJSONKEY]:
                                if key == "prefixes":
                                    modelXbrl.error(
                                        "oime:duplicatedPrefix",
                                        _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"
                                          ),
                                        modelObject=modelXbrl,
                                        prefix=_errKey,
                                        uri1=_errValue,
                                        uri2=_otherValue)
                            del value[DUPJSONKEY]
                    if key in _dict:
                        if DUPJSONKEY not in _dict:
                            _dict[DUPJSONKEY] = []
                        _dict[DUPJSONKEY].append((key, value, _dict[key]))
                    else:
                        _dict[key] = value
                return _dict

            with io.open(oimFile, 'rt', encoding='utf-8') as f:
                oimObject = json.load(f, object_pairs_hook=loadDict)
            missing = [
                t for t in ("dtsReferences", "prefixes", "facts")
                if t not in oimObject
            ]
            if missing:
                raise OIMException(
                    "oime:missingJSONElements",
                    _("Required element(s) are missing from JSON input: %(missing)s"
                      ),
                    missing=", ".join(missing))
            currentAction = "identifying JSON objects"
            dtsReferences = oimObject["dtsReferences"]
            prefixesList = oimObject["prefixes"].items()
            facts = oimObject["facts"]
            footnotes = oimObject["facts"]  # shares this object
        elif isCSV:
            currentAction = "identifying CSV input tables"
            if sys.version[0] >= '3':
                csvOpenMode = 'w'
                csvOpenNewline = ''
            else:
                csvOpenMode = 'wb'  # for 2.7
                csvOpenNewline = None

            oimFileBase = None
            if "-facts" in oimFile:
                oimFileBase = oimFile.partition("-facts")[0]
            else:
                for suffix in ("-dtsReferences.csv", "-defaults.csv",
                               "-prefixes.csv", "-footnotes.csv",
                               "-metadata.json"):
                    if oimFile.endswith(suffix):
                        oimFileBase = oimFile[:-len(suffix)]
                        break
            if oimFileBase is None:
                raise OIMException(
                    "oime:missingCSVTables",
                    _("Unable to identify CSV tables file name pattern"))
            if (not os.path.exists(oimFileBase + "-dtsReferences.csv")
                    or not os.path.exists(oimFileBase + "-prefixes.csv")):
                raise OIMException(
                    "oime:missingCSVTables",
                    _("Unable to identify CSV tables for dtsReferences or prefixes"
                      ))
            instanceFileName = oimFileBase + ".xbrl"
            currentAction = "loading CSV dtsReferences table"
            dtsReferences = []
            with io.open(oimFileBase + "-dtsReferences.csv",
                         'rt',
                         encoding='utf-8-sig') as f:
                csvReader = csv.reader(f)
                for i, row in enumerate(csvReader):
                    if i == 0:
                        header = row
                    else:
                        dtsReferences.append(
                            dict(
                                (header[j], col) for j, col in enumerate(row)))
            currentAction = "loading CSV prefixes table"
            prefixesList = []
            with io.open(oimFileBase + "-prefixes.csv",
                         'rt',
                         encoding='utf-8-sig') as f:
                csvReader = csv.reader(f)
                for i, row in enumerate(csvReader):
                    if i == 0:
                        header = dict((col, i) for i, col in enumerate(row))
                    else:
                        prefixesList.append(
                            (row[header["prefix"]], row[header["URI"]]))
            defaults = {}
            if os.path.exists(oimFileBase + "-defaults.csv"):
                currentAction = "loading CSV defaults table"
                with io.open(oimFileBase + "-defaults.csv",
                             'rt',
                             encoding='utf-8-sig') as f:
                    csvReader = csv.reader(f)
                    for i, row in enumerate(csvReader):
                        if i == 0:
                            header = row
                            fileCol = row.index("file")
                        else:
                            defaults[row[fileCol]] = dict(
                                (header[j], col) for j, col in enumerate(row)
                                if j != fileCol)
            currentAction = "loading CSV facts tables"
            facts = []
            _dir = os.path.dirname(oimFileBase)
            factsFileBasename = os.path.basename(oimFileBase) + "-facts"
            for filename in os.listdir(_dir):
                filepath = os.path.join(_dir, filename)
                if filename.startswith(factsFileBasename):
                    currentAction = "loading CSV facts table {}".format(
                        filename)
                    tableDefaults = defaults.get(filename, {})
                    with io.open(filepath, 'rt', encoding='utf-8-sig') as f:
                        csvReader = csv.reader(f)
                        for i, row in enumerate(csvReader):
                            if i == 0:
                                header = row
                            else:
                                fact = {}
                                fact.update(tableDefaults)
                                for j, col in enumerate(row):
                                    if col is not None:
                                        if header[
                                                j]:  # skip cols with no header
                                            if header[j].endswith("Value"):
                                                if col:  # ignore empty columns (= null CSV value)
                                                    fact["value"] = col
                                            else:
                                                fact[header[j]] = col
                                facts.append(fact)
            footnotes = []
            if os.path.exists(oimFileBase + "-footnotes.csv"):
                currentAction = "loading CSV footnotes table"
                with io.open(oimFileBase + "-footnotes.csv",
                             'rt',
                             encoding='utf-8-sig') as f:
                    csvReader = csv.reader(f)
                    for i, row in enumerate(csvReader):
                        if i == 0:
                            header = row
                        else:
                            footnotes.append(
                                dict((header[j], col)
                                     for j, col in enumerate(row) if col))
        elif isXL:
            currentAction = "identifying workbook input worksheets"
            oimWb = load_workbook(oimFile, read_only=True, data_only=True)
            sheetNames = oimWb.get_sheet_names()
            if (not any(sheetName == "prefixes" for sheetName in sheetNames)
                    or not any(sheetName == "dtsReferences"
                               for sheetName in sheetNames)
                    or not any("facts" in sheetName
                               for sheetName in sheetNames)):
                raise OIMException(
                    "oime:missingWorkbookWorksheets",
                    _("Unable to identify worksheet tabs for dtsReferences, prefixes or facts"
                      ))
            currentAction = "loading worksheet: dtsReferences"
            dtsReferences = []
            for i, row in enumerate(oimWb["dtsReferences"]):
                if i == 0:
                    header = [col.value for col in row]
                else:
                    dtsReferences.append(
                        dict((header[j], col.value)
                             for j, col in enumerate(row)))
            currentAction = "loading worksheet: prefixes"
            prefixesList = []
            for i, row in enumerate(oimWb["prefixes"]):
                if i == 0:
                    header = dict((col.value, i) for i, col in enumerate(row))
                else:
                    prefixesList.append((row[header["prefix"]].value,
                                         row[header["URI"]].value))
            defaults = {}
            if "defaults" in sheetNames:
                currentAction = "loading worksheet: defaults"
                for i, row in enumerate(oimWb["defaults"]):
                    if i == 0:
                        header = dict(
                            (col.value, i) for i, col in enumerate(row))
                        fileCol = header["file"]
                    else:
                        defaults[row[fileCol].value] = dict(
                            (header[j], col.value) for j, col in enumerate(row)
                            if j != fileCol)
            facts = []
            for sheetName in sheetNames:
                if sheetName == "facts" or "-facts" in sheetName:
                    currentAction = "loading worksheet: {}".format(sheetName)
                    tableDefaults = defaults.get(sheetName, {})
                    for i, row in enumerate(oimWb[sheetName]):
                        if i == 0:
                            header = [col.value for col in row]
                        else:
                            fact = {}
                            fact.update(tableDefaults)
                            for j, col in enumerate(row):
                                if col.value is not None:
                                    if header[j]:  # skip cols with no header
                                        if header[j].endswith("Value"):
                                            fact["value"] = str(col.value)
                                        else:
                                            fact[header[j]] = str(col.value)
                            facts.append(fact)
            footnotes = []
            if "footnotes" in sheetNames:
                currentAction = "loading worksheet: footnotes"
                for i, row in enumerate(oimWb["footnotes"]):
                    if i == 0:
                        header = dict((col.value, i)
                                      for i, col in enumerate(row)
                                      if col.value)
                    else:
                        footnotes.append(
                            dict((header[j], col.value)
                                 for j, col in enumerate(row) if col.value))

        currentAction = "identifying default dimensions"
        ValidateXbrlDimensions.loadDimensionDefaults(
            modelXbrl)  # needs dimension defaults

        currentAction = "validating OIM"
        prefixes = {}
        prefixedUris = {}
        for _prefix, _uri in prefixesList:
            if not _prefix:
                modelXbrl.error(
                    "oime:emptyPrefix",
                    _("The empty string must not be used as a prefix, uri %(uri)s"
                      ),
                    modelObject=modelXbrl,
                    uri=_uri)
            elif not NCNamePattern.match(_prefix):
                modelXbrl.error(
                    "oime:prefixPattern",
                    _("The prefix %(prefix)s must match the NCName lexical pattern, uri %(uri)s"
                      ),
                    modelObject=modelXbrl,
                    prefix=_prefix,
                    uri=_uri)
            elif _prefix in prefixes:
                modelXbrl.error(
                    "oime:duplicatedPrefix",
                    _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"
                      ),
                    modelObject=modelXbrl,
                    prefix=_prefix,
                    uri1=prefixes[_prefix],
                    uri2=_uri)
            elif _uri in prefixedUris:
                modelXbrl.error(
                    "oime:duplicatedUri",
                    _("The uri %(uri)s is used on prefix %(prefix1)s and prefix %(prefix2)s"
                      ),
                    modelObject=modelXbrl,
                    uri=_uri,
                    prefix1=prefixedUris[_uri],
                    prefix2=_prefix)
            else:
                prefixes[_prefix] = _uri
                prefixedUris[_uri] = _prefix

        oimPrefix = None
        for _nsOim in nsOim:
            if _nsOim in prefixedUris:
                oimPrefix = prefixedUris[_nsOim]
        if not oimPrefix:
            raise OIMException(
                "oime:noOimPrefix",
                _("The oim namespace must have a declared prefix"))
        oimConcept = "{}:concept".format(oimPrefix)
        oimEntity = "{}:entity".format(oimPrefix)
        oimPeriod = "{}:period".format(oimPrefix)
        oimPeriodStart = "{}:periodStart".format(oimPrefix)
        oimPeriodEnd = "{}:periodEnd".format(oimPrefix)
        oimUnit = "{}:unit".format(oimPrefix)
        oimPrefix = "{}:".format(oimPrefix)

        # create the instance document
        currentAction = "creating instance document"
        modelXbrl.blockDpmDBrecursion = True
        modelXbrl.modelDocument = createModelDocument(
            modelXbrl,
            Type.INSTANCE,
            instanceFileName,
            schemaRefs=[
                dtsRef["href"] for dtsRef in dtsReferences
                if dtsRef["type"] == "schema"
            ],
            isEntry=True,
            initialComment="extracted from OIM {}".format(mappedUri),
            documentEncoding="utf-8")
        cntxTbl = {}
        unitTbl = {}
        currentAction = "creating facts"
        for fact in facts:
            conceptQn = qname(fact[oimConcept], prefixes)
            concept = modelXbrl.qnameConcepts.get(conceptQn)
            entityAsQn = qname(fact[oimEntity], prefixes)
            if oimPeriod in fact:
                periodStart = fact[oimPeriod]["start"]
                periodEnd = fact[oimPeriod]["end"]
            else:
                periodStart = fact[oimPeriodStart]
                periodEnd = fact[oimPeriodEnd]
            cntxKey = (  # hashable context key
                ("periodType", concept.periodType), ("entity", entityAsQn),
                ("periodStart", periodStart),
                ("periodEnd", periodEnd)) + tuple(
                    sorted(
                        (dimName, dimVal)
                        for dimName, dimVal in fact.items() if ":" in dimName
                        and not dimName.startswith(oimPrefix)))
            if cntxKey in cntxTbl:
                _cntx = cntxTbl[cntxKey]
            else:
                cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
                qnameDims = {}
                for dimName, dimVal in fact.items():
                    if ":" in dimName and not dimName.startswith(
                            oimPrefix) and dimVal:
                        dimQname = qname(dimName, prefixes)
                        dimConcept = modelXbrl.qnameConcepts.get(dimQname)
                        if ":" in dimVal and dimVal.partition(
                                ':')[0] in prefixes:
                            mem = qname(dimVal, prefixes)  # explicit dim
                        elif dimConcept.isTypedDimension:
                            # a modelObject xml element is needed for all of the instance functions to manage the typed dim
                            mem = addChild(modelXbrl.modelDocument,
                                           dimConcept.typedDomainElement.qname,
                                           text=dimVal,
                                           appendChild=False)
                        qnameDims[dimQname] = DimValuePrototype(
                            modelXbrl, None, dimQname, mem, "segment")
                _cntx = modelXbrl.createContext(
                    entityAsQn.namespaceURI,
                    entityAsQn.localName,
                    concept.periodType,
                    None if concept.periodType == "instant" else dateTime(
                        periodStart, type=DATETIME),
                    dateTime(periodEnd, type=DATETIME),
                    None,  # no dimensional validity checking (like formula does)
                    qnameDims,
                    [],
                    [],
                    id=cntxId)
                cntxTbl[cntxKey] = _cntx
            if oimUnit in fact:
                unitKey = fact[oimUnit]
                if unitKey in unitTbl:
                    _unit = unitTbl[unitKey]
                else:
                    _unit = None
                    # validate unit
                    unitKeySub = PrefixedQName.sub(
                        UnitPrefixedQNameSubstitutionChar, unitKey)
                    if not UnitPattern.match(unitKeySub):
                        modelXbrl.error(
                            "oime:unitStringRepresentation",
                            _("Unit string representation is lexically invalid, %(unit)s"
                              ),
                            modelObject=modelXbrl,
                            unit=unitKey)
                    else:
                        _mul, _sep, _div = unitKey.partition('/')
                        if _mul.startswith('('):
                            _mul = _mul[1:-1]
                        _muls = [u for u in _mul.split('*') if u]
                        if _div.startswith('('):
                            _div = _div[1:-1]
                        _divs = [u for u in _div.split('*') if u]
                        if _muls != sorted(_muls) or _divs != sorted(_divs):
                            modelXbrl.error(
                                "oime:unitStringRepresentation",
                                _("Unit string representation measures are not in alphabetical order, %(unit)s"
                                  ),
                                modelObject=modelXbrl,
                                unit=unitKey)
                        try:
                            mulQns = [
                                qname(
                                    u,
                                    prefixes,
                                    prefixException=OIMException(
                                        "oime:unitPrefix",
                                        _("Unit prefix is not declared: %(unit)s"
                                          ),
                                        unit=u)) for u in _muls
                            ]
                            divQns = [
                                qname(
                                    u,
                                    prefixes,
                                    prefixException=OIMException(
                                        "oime:unitPrefix",
                                        _("Unit prefix is not declared: %(unit)s"
                                          ),
                                        unit=u)) for u in _divs
                            ]
                            unitId = 'u-{:02}'.format(len(unitTbl) + 1)
                            for _measures in mulQns, divQns:
                                for _measure in _measures:
                                    addQnameValue(modelXbrl.modelDocument,
                                                  _measure)
                            _unit = modelXbrl.createUnit(mulQns,
                                                         divQns,
                                                         id=unitId)
                        except OIMException as ex:
                            modelXbrl.error(ex.code,
                                            ex.message,
                                            modelObject=modelXbrl,
                                            **ex.msgArgs)
                    unitTbl[unitKey] = _unit
            else:
                _unit = None

            attrs = {"contextRef": _cntx.id}

            if fact.get("value") is None:
                attrs[XbrlConst.qnXsiNil] = "true"
                text = None
            else:
                text = fact["value"]

            if fact.get("id"):
                attrs["id"] = fact["id"]

            if concept.isNumeric:
                if _unit is None:
                    continue  # skip creating fact because unit was invalid
                attrs["unitRef"] = _unit.id
                if "accuracy" in fact:
                    attrs["decimals"] = fact["accuracy"]

            # is value a QName?
            if concept.baseXbrliType == "QName":
                addQnameValue(modelXbrl.modelDocument,
                              qname(text.strip(), prefixes))

            f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text)

        currentAction = "creating footnotes"
        footnoteLinks = {}  # ELR elements
        factLocs = {}  # index by (linkrole, factId)
        footnoteNbr = 0
        locNbr = 0
        for factOrFootnote in footnotes:
            if "factId" in factOrFootnote:
                factId = factOrFootnote["factId"]
                factFootnotes = (factOrFootnote, )  # CSV or XL
            elif "id" in factOrFootnote and "footnotes" in factOrFootnote:
                factId = factOrFootnote["id"]
                factFootnotes = factOrFootnote["footnotes"]
            else:
                factFootnotes = ()
            for footnote in factFootnotes:
                linkrole = footnote.get("group")
                arcrole = footnote.get("footnoteType")
                if not factId or not linkrole or not arcrole or not (
                        footnote.get("factRef") or footnote.get("footnote")):
                    # invalid footnote
                    continue
                if linkrole not in footnoteLinks:
                    footnoteLinks[linkrole] = addChild(
                        modelXbrl.modelDocument.xmlRootElement,
                        XbrlConst.qnLinkFootnoteLink,
                        attributes={
                            "{http://www.w3.org/1999/xlink}type": "extended",
                            "{http://www.w3.org/1999/xlink}role": linkrole
                        })
                footnoteLink = footnoteLinks[linkrole]
                if (linkrole, factId) not in factLocs:
                    locNbr += 1
                    locLabel = "l_{:02}".format(locNbr)
                    factLocs[(linkrole, factId)] = locLabel
                    addChild(footnoteLink,
                             XbrlConst.qnLinkLoc,
                             attributes={
                                 XLINKTYPE: "locator",
                                 XLINKHREF: "#" + factId,
                                 XLINKLABEL: locLabel
                             })
                locLabel = factLocs[(linkrole, factId)]
                if footnote.get("footnote"):
                    footnoteNbr += 1
                    footnoteLabel = "f_{:02}".format(footnoteNbr)
                    attrs = {XLINKTYPE: "resource", XLINKLABEL: footnoteLabel}
                    if footnote.get("language"):
                        attrs[XMLLANG] = footnote["language"]
                    # note, for HTML will need to build an element structure
                    addChild(footnoteLink,
                             XbrlConst.qnLinkFootnote,
                             attributes=attrs,
                             text=footnote["footnote"])
                elif footnote.get("factRef"):
                    factRef = footnote.get("factRef")
                    if (linkrole, factRef) not in factLocs:
                        locNbr += 1
                        locLabel = "f_{:02}".format(footnoteNbr)
                        factLoc[(linkrole, factRef)] = locLabel
                        addChild(footnoteLink,
                                 XbrlConst.qnLinkLoc,
                                 attributes={
                                     XLINKTYPE: "locator",
                                     XLINKHREF: "#" + factRef,
                                     XLINKLABEL: locLabel
                                 })
                    footnoteLabel = factLoc[(linkrole, factId)]
                footnoteArc = addChild(footnoteLink,
                                       XbrlConst.qnLinkFootnoteArc,
                                       attributes={
                                           XLINKTYPE: "arc",
                                           XLINKARCROLE: arcrole,
                                           XLINKFROM: locLabel,
                                           XLINKTO: footnoteLabel
                                       })

        currentAction = "done loading facts and footnotes"

        #cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt),
        #               messageCode="loadFromExcel:info")
    except Exception as ex:
        if isinstance(ex, OIMException):
            modelXbrl.error(ex.code,
                            ex.message,
                            modelObject=modelXbrl,
                            **ex.msgArgs)
        else:
            modelXbrl.error(
                "arelleOIMloader:error",
                "Error while %(action)s, error %(error)s\ntraceback %(traceback)s",
                modelObject=modelXbrl,
                action=currentAction,
                error=ex,
                traceback=traceback.format_tb(sys.exc_info()[2]))

    if priorCWD:
        os.chdir(
            priorCWD
        )  # restore prior current working directory            startingErrorCount = len(modelXbrl.errors)

    if startingErrorCount < len(modelXbrl.errors):
        # had errors, don't allow ModelDocument.load to continue
        return OIMException("arelleOIMloader:unableToLoad",
                            "Unable to load due to reported errors")

    return getattr(modelXbrl, "modelDocument",
                   None)  # none if returning from exception
Example #21
0
def current_dateTime(xc, p, contextItem, args):
    from datetime import datetime
    return dateTime(datetime.now(), type=DATETIME)
Example #22
0
def validateValue(modelXbrl, elt, attrTag, baseXsdType, value, isNillable=False, facets=None):
    if baseXsdType:
        try:
            if (len(value) == 0 and
                not attrTag is None and 
                not isNillable and 
                baseXsdType not in ("anyType", "string", "normalizedString", "token", "NMTOKEN", "anyURI", "noContent")):
                raise ValueError("missing value for not nillable element")
            xValid = VALID
            whitespaceReplace = (baseXsdType == "normalizedString")
            whitespaceCollapse = (not whitespaceReplace and baseXsdType != "string")
            pattern = baseXsdTypePatterns.get(baseXsdType)
            if facets:
                if "pattern" in facets:
                    pattern = facets["pattern"]
                    # note multiple patterns are or'ed togetner, which isn't yet implemented!
                if "whiteSpace" in facets:
                    whitespaceReplace, whitespaceCollapse = {"preserve":(False,False), "replace":(True,False), "collapse":(False,True)}
            if whitespaceReplace:
                value = normalizeWhitespacePattern.sub(' ', value)
            elif whitespaceCollapse:
                value = collapseWhitespacePattern.sub(' ', value.strip())
            if pattern is not None and pattern.match(value) is None:
                    raise ValueError("pattern facet " + facets["pattern"].pattern if facets and "pattern" in facets else "pattern mismatch")
            if facets:
                if "enumeration" in facets and value not in facets["enumeration"]:
                    raise ValueError("is not in {1}".format(value, facets["enumeration"]))
                if "length" in facets and len(value) != facets["length"]:
                    raise ValueError("length facet")
                if "minLength" in facets and len(value) < facets["minLength"]:
                    raise ValueError("minLength facet")
                if "maxLength" in facets and len(value) > facets["maxLength"]:
                    raise ValueError("maxLength facet")
            if baseXsdType == "noContent":
                if len(value) > 0 and not value.isspace():
                    raise ValueError("value content not permitted")
                xValue = sValue = None
            elif baseXsdType in {"string", "normalizedString", "language", "token", "NMTOKEN","Name","NCName","IDREF","ENTITY"}:
                xValue = sValue = value
            elif baseXsdType == "ID":
                xValue = sValue = value
                xValid = VALID_ID
            elif baseXsdType == "anyURI":
                xValue = anyURI(value)
                sValue = value
                if xValue and not UrlUtil.isValid(xValue):  # allow empty strings to be valid anyURIs
                    raise ValueError("invalid anyURI value")
            elif not value: # rest of types get None if nil/empty value
                xValue = sValue = None
            elif baseXsdType in ("decimal", "float", "double"):
                xValue = sValue = float(value)
                if facets:
                    if "totalDigits" in facets and len(value.replace(".","")) != facets["totalDigits"]:
                        raise ValueError("total digits facet")
                    if "fractionDigits" in facets and ( '.' not in value or
                        len(value[value.index('.') + 1:]) != facets["fractionDigits"]):
                        raise ValueError("fraction digits facet")
            elif baseXsdType in ("integer",):
                xValue = sValue = int(value)
            elif baseXsdType == "boolean":
                if value in ("true", "1"):  
                    xValue = sValue = True
                elif value in ("false", "0"): 
                    xValue = sValue = False
                else: raise ValueError
            elif baseXsdType == "QName":
                xValue = qname(elt, value, castException=ValueError)
                sValue = value
                ''' not sure here, how are explicitDimensions validated, but bad units not?
                if xValue.namespaceURI in modelXbrl.namespaceDocs:
                    if (xValue not in modelXbrl.qnameConcepts and 
                        xValue not in modelXbrl.qnameTypes and
                        xValue not in modelXbrl.qnameAttributes and
                        xValue not in modelXbrl.qnameAttributeGroups):
                        raise ValueError("qname not defined " + str(xValue))
                '''
            elif baseXsdType in ("XBRLI_DECIMALSUNION", "XBRLI_PRECISIONUNION"):
                xValue = sValue = value if value == "INF" else int(value)
            elif baseXsdType in ("XBRLI_NONZERODECIMAL"):
                xValue = sValue = int(value)
                if xValue == 0:
                    raise ValueError("invalid value")
            elif baseXsdType == "XBRLI_DATEUNION":
                xValue = dateTime(value, type=DATEUNION, castException=ValueError)
                sValue = value
            elif baseXsdType == "dateTime":
                xValue = dateTime(value, type=DATETIME, castException=ValueError)
                sValue = value
            elif baseXsdType == "date":
                xValue = dateTime(value, type=DATE, castException=ValueError)
                sValue = value
            elif baseXsdType == "regex-pattern":
                # for facet compiling
                try:
                    xValue = re.compile(value + "$") # must match whole string
                    sValue = value
                except Exception as err:
                    raise ValueError(err)
            else:
                xValue = value
                sValue = value
        except ValueError as err:
            if attrTag:
                modelXbrl.error("xmlSchema:valueError",
                    _("Element %(element)s attribute %(attribute)s type %(typeName)s value error: %(value)s, %(error)s"),
                    modelObject=elt,
                    element=elt.elementQname,
                    attribute=XmlUtil.clarkNotationToPrefixedName(elt,attrTag,isAttribute=True),
                    typeName=baseXsdType,
                    value=value,
                    error=err)
            else:
                modelXbrl.error("xmlSchema:valueError",
                    _("Element %(element)s type %(typeName)s value error: %(value)s"),
                    modelObject=elt,
                    element=elt.elementQname,
                    typeName=baseXsdType,
                    value=value)
            xValue = None
            sValue = value
            xValid = INVALID
    else:
        xValue = sValue = None
        xValid = UNKNOWN
    if attrTag:
        elt.xAttributes[attrTag] = ModelAttribute(elt, attrTag, xValid, xValue, sValue, value)
    else:
        elt.xValid = xValid
        elt.xValue = xValue
        elt.sValue = sValue
Example #23
0
def validateValue(modelXbrl, elt, attrTag, baseXsdType, value, isNillable=False, isNil=False, facets=None):
    if baseXsdType:
        try:
            '''
            if (len(value) == 0 and attrTag is None and not isNillable and 
                baseXsdType not in ("anyType", "string", "normalizedString", "token", "NMTOKEN", "anyURI", "noContent")):
                raise ValueError("missing value for not nillable element")
            '''
            xValid = VALID
            whitespaceReplace = (baseXsdType == "normalizedString")
            whitespaceCollapse = (not whitespaceReplace and baseXsdType != "string")
            isList = baseXsdType in {"IDREFS", "ENTITIES", "NMTOKENS"}
            if isList:
                baseXsdType = baseXsdType[:-1] # remove plural
            pattern = baseXsdTypePatterns.get(baseXsdType)
            if facets:
                if "pattern" in facets:
                    pattern = facets["pattern"]
                    # note multiple patterns are or'ed togetner, which isn't yet implemented!
                if "whiteSpace" in facets:
                    whitespaceReplace, whitespaceCollapse = {"preserve":(False,False), "replace":(True,False), "collapse":(False,True)}[facets["whiteSpace"]]
            if whitespaceReplace:
                value = normalizeWhitespacePattern.sub(' ', value)
            elif whitespaceCollapse:
                value = collapseWhitespacePattern.sub(' ', value.strip())
            if baseXsdType == "noContent":
                if len(value) > 0 and not value.isspace():
                    raise ValueError("value content not permitted")
                # note that sValue and xValue are not innerText but only text elements on specific element (or attribute)
                xValue = sValue = None
                xValid = VALID_NO_CONTENT # notify others that element may contain subelements (for stringValue needs)
            elif not value and isNil and isNillable: # rest of types get None if nil/empty value
                xValue = sValue = None
            else:
                if pattern is not None:
                    if ((isList and any(pattern.match(v) is None for v in value.split())) or
                        (not isList and pattern.match(value) is None)):
                        raise ValueError("pattern facet " + facets["pattern"].pattern if facets and "pattern" in facets else "pattern mismatch")
                if facets:
                    if "enumeration" in facets and value not in facets["enumeration"]:
                        raise ValueError("{0} is not in {1}".format(value, facets["enumeration"]))
                    if "length" in facets and len(value) != facets["length"]:
                        raise ValueError("length {0}, expected {1}".format(len(value), facets["length"]))
                    if "minLength" in facets and len(value) < facets["minLength"]:
                        raise ValueError("length {0}, minLength {1}".format(len(value), facets["minLength"]))
                    if "maxLength" in facets and len(value) > facets["maxLength"]:
                        raise ValueError("length {0}, maxLength {1}".format(len(value), facets["maxLength"]))
                if baseXsdType in {"string", "normalizedString", "language", "token", "NMTOKEN","Name","NCName","IDREF","ENTITY"}:
                    xValue = sValue = value
                elif baseXsdType == "ID":
                    xValue = sValue = value
                    xValid = VALID_ID
                elif baseXsdType == "anyURI":
                    if value:  # allow empty strings to be valid anyURIs
                        if UrlUtil.relativeUrlPattern.match(value) is None:
                            raise ValueError("IETF RFC 2396 4.3 syntax")
                    # encode PSVI xValue similarly to Xerces and other implementations
                    xValue = anyURI(UrlUtil.anyUriQuoteForPSVI(value))
                    sValue = value
                elif baseXsdType in ("decimal", "float", "double"):
                    if baseXsdType == "decimal":
                        if decimalPattern.match(value) is None:
                            raise ValueError("lexical pattern mismatch")
                        xValue = Decimal(value)
                        sValue = float(value) # s-value uses Number (float) representation
                    else:
                        if floatPattern.match(value) is None:
                            raise ValueError("lexical pattern mismatch")
                        xValue = sValue = float(value)
                    if facets:
                        if "totalDigits" in facets and len(value.replace(".","")) > facets["totalDigits"]:
                            raise ValueError("totalDigits facet {0}".format(facets["totalDigits"]))
                        if "fractionDigits" in facets and ( '.' in value and
                            len(value[value.index('.') + 1:]) > facets["fractionDigits"]):
                            raise ValueError("fraction digits facet {0}".format(facets["fractionDigits"]))
                        if "maxInclusive" in facets and xValue > facets["maxInclusive"]:
                            raise ValueError(" > maxInclusive {0}".format(facets["maxInclusive"]))
                        if "maxExclusive" in facets and xValue >= facets["maxExclusive"]:
                            raise ValueError(" >= maxInclusive {0}".format(facets["maxExclusive"]))
                        if "minInclusive" in facets and xValue < facets["minInclusive"]:
                            raise ValueError(" < minInclusive {0}".format(facets["minInclusive"]))
                        if "minExclusive" in facets and xValue <= facets["minExclusive"]:
                            raise ValueError(" <= minExclusive {0}".format(facets["minExclusive"]))
                elif baseXsdType in {"integer",
                                     "nonPositiveInteger","negativeInteger","nonNegativeInteger","positiveInteger",
                                     "long","unsignedLong",
                                     "int","unsignedInt",
                                     "short","unsignedShort",
                                     "byte","unsignedByte"}:
                    xValue = sValue = _INT(value)
                    if ((baseXsdType in {"nonNegativeInteger","unsignedLong","unsignedInt"} 
                         and xValue < 0) or
                        (baseXsdType == "nonPositiveInteger" and xValue > 0) or
                        (baseXsdType == "positiveInteger" and xValue <= 0) or
                        (baseXsdType == "byte" and not -128 <= xValue < 127) or
                        (baseXsdType == "unsignedByte" and not 0 <= xValue < 255) or
                        (baseXsdType == "short" and not -32768 <= xValue < 32767) or
                        (baseXsdType == "unsignedShort" and not 0 <= xValue < 65535) or
                        (baseXsdType == "positiveInteger" and xValue <= 0)):
                        raise ValueError("{0} is not {1}".format(value, baseXsdType))
                    if facets:
                        if "totalDigits" in facets and len(value.replace(".","")) > facets["totalDigits"]:
                            raise ValueError("totalDigits facet {0}".format(facets["totalDigits"]))
                        if "fractionDigits" in facets and ( '.' in value and
                            len(value[value.index('.') + 1:]) > facets["fractionDigits"]):
                            raise ValueError("fraction digits facet {0}".format(facets["fractionDigits"]))
                        if "maxInclusive" in facets and xValue > facets["maxInclusive"]:
                            raise ValueError(" > maxInclusive {0}".format(facets["maxInclusive"]))
                        if "maxExclusive" in facets and xValue >= facets["maxExclusive"]:
                            raise ValueError(" >= maxInclusive {0}".format(facets["maxExclusive"]))
                        if "minInclusive" in facets and xValue < facets["minInclusive"]:
                            raise ValueError(" < minInclusive {0}".format(facets["minInclusive"]))
                        if "minExclusive" in facets and xValue <= facets["minExclusive"]:
                            raise ValueError(" <= minExclusive {0}".format(facets["minExclusive"]))
                elif baseXsdType == "boolean":
                    if value in ("true", "1"):  
                        xValue = sValue = True
                    elif value in ("false", "0"): 
                        xValue = sValue = False
                    else: raise ValueError
                elif baseXsdType == "QName":
                    xValue = qnameEltPfxName(elt, value, prefixException=ValueError)
                    #xValue = qname(elt, value, castException=ValueError, prefixException=ValueError)
                    sValue = value
                    ''' not sure here, how are explicitDimensions validated, but bad units not?
                    if xValue.namespaceURI in modelXbrl.namespaceDocs:
                        if (xValue not in modelXbrl.qnameConcepts and 
                            xValue not in modelXbrl.qnameTypes and
                            xValue not in modelXbrl.qnameAttributes and
                            xValue not in modelXbrl.qnameAttributeGroups):
                            raise ValueError("qname not defined " + str(xValue))
                    '''
                elif baseXsdType in ("XBRLI_DECIMALSUNION", "XBRLI_PRECISIONUNION"):
                    xValue = sValue = value if value == "INF" else _INT(value)
                elif baseXsdType in ("XBRLI_NONZERODECIMAL"):
                    xValue = sValue = _INT(value)
                    if xValue == 0:
                        raise ValueError("invalid value")
                elif baseXsdType == "XBRLI_DATEUNION":
                    xValue = dateTime(value, type=DATEUNION, castException=ValueError)
                    sValue = value
                elif baseXsdType == "dateTime":
                    xValue = dateTime(value, type=DATETIME, castException=ValueError)
                    sValue = value
                elif baseXsdType == "date":
                    xValue = dateTime(value, type=DATE, castException=ValueError)
                    sValue = value
                elif baseXsdType == "regex-pattern":
                    # for facet compiling
                    try:
                        sValue = value
                        if value in xmlSchemaPatterns:
                            xValue = xmlSchemaPatterns[value]
                        else:
                            if r"\i" in value or r"\c" in value:
                                value = value.replace(r"\i", iNameChar).replace(r"\c", cNameChar)
                            xValue = re_compile(value + "$") # must match whole string
                    except Exception as err:
                        raise ValueError(err)
                elif baseXsdType == "fraction":
                    sValue = value
                    xValue = Fraction("/".join(elt.fractionValue))
                else:
                    if baseXsdType in lexicalPatterns:
                        match = lexicalPatterns[baseXsdType].match(value)
                        if match is None:
                            raise ValueError("lexical pattern mismatch")
                        if baseXsdType == "gMonthDay":
                            month, day, zSign, zHrMin, zHr, zMin = match.groups()
                            if int(day) > {2:29, 4:30, 6:30, 9:30, 11:30, 1:31, 3:31, 5:31, 7:31, 8:31, 10:31, 12:31}[int(month)]:
                                raise ValueError("invalid day {0} for month {1}".format(day, month))
                            xValue = gMonthDay(month, day)
                        elif baseXsdType == "gYearMonth":
                            year, month, zSign, zHrMin, zHr, zMin = match.groups()
                            xValue = gYearMonth(year, month)
                        elif baseXsdType == "gYear":
                            year, zSign, zHrMin, zHr, zMin = match.groups()
                            xValue = gYear(year)
                        elif baseXsdType == "gMonth":
                            month, zSign, zHrMin, zHr, zMin = match.groups()
                            xValue = gMonth(month)
                        elif baseXsdType == "gDay":
                            day, zSign, zHrMin, zHr, zMin = match.groups()
                            xValue = gDay(day)
                        else:
                            xValue = value
                    else: # no lexical pattern, forget compiling value
                        xValue = value
                    sValue = value
        except (ValueError, InvalidOperation) as err:
            if ModelInlineValueObject is not None and isinstance(elt, ModelInlineValueObject):
                errElt = "{0} fact {1}".format(elt.elementQname, elt.qname)
            else:
                errElt = elt.elementQname
            if attrTag:
                modelXbrl.error("xmlSchema:valueError",
                    _("Element %(element)s attribute %(attribute)s type %(typeName)s value error: %(value)s, %(error)s"),
                    modelObject=elt,
                    element=errElt,
                    attribute=XmlUtil.clarkNotationToPrefixedName(elt,attrTag,isAttribute=True),
                    typeName=baseXsdType,
                    value=value if len(value) < 31 else value[:30] + '...',
                    error=err)
            else:
                modelXbrl.error("xmlSchema:valueError",
                    _("Element %(element)s type %(typeName)s value error: %(value)s, %(error)s"),
                    modelObject=elt,
                    element=errElt,
                    typeName=baseXsdType,
                    value=value if len(value) < 31 else value[:30] + '...',
                    error=err)
            xValue = None
            sValue = value
            xValid = INVALID
    else:
        xValue = sValue = None
        xValid = UNKNOWN
    if attrTag:
        try:  # dynamically allocate attributes (otherwise given shared empty set)
            xAttributes = elt.xAttributes
        except AttributeError:
            elt.xAttributes = xAttributes = {}
        xAttributes[attrTag] = ModelAttribute(elt, attrTag, xValid, xValue, sValue, value)
    else:
        elt.xValid = xValid
        elt.xValue = xValue
        elt.sValue = sValue
Example #24
0
def validateValue(modelXbrl, elt, attrTag, baseXsdType, value, isNillable=False, facets=None):
    if baseXsdType:
        try:
            if (len(value) == 0 and
                not attrTag is None and 
                not isNillable and 
                baseXsdType not in ("anyType", "string", "normalizedString", "token", "NMTOKEN", "anyURI", "noContent")):
                raise ValueError("missing value for not nillable element")
            xValid = VALID
            whitespaceReplace = (baseXsdType == "normalizedString")
            whitespaceCollapse = (not whitespaceReplace and baseXsdType != "string")
            pattern = baseXsdTypePatterns.get(baseXsdType)
            if facets:
                if "pattern" in facets:
                    pattern = facets["pattern"]
                    # note multiple patterns are or'ed togetner, which isn't yet implemented!
                if "whiteSpace" in facets:
                    whitespaceReplace, whitespaceCollapse = {"preserve":(False,False), "replace":(True,False), "collapse":(False,True)}
            if whitespaceReplace:
                value = normalizeWhitespacePattern.sub(' ', value)
            elif whitespaceCollapse:
                value = collapseWhitespacePattern.sub(' ', value.strip())
            if pattern is not None and pattern.match(value) is None:
                    raise ValueError("pattern facet " + facets["pattern"].pattern if facets and "pattern" in facets else "pattern mismatch")
            if facets:
                if "enumeration" in facets and value not in facets["enumeration"]:
                    raise ValueError("{0} is not in {1}".format(value, facets["enumeration"]))
                if "length" in facets and len(value) != facets["length"]:
                    raise ValueError("length {0}, expected {1}".format(len(value), facets["length"]))
                if "minLength" in facets and len(value) < facets["minLength"]:
                    raise ValueError("length {0}, minLength {1}".format(len(value), facets["minLength"]))
                if "maxLength" in facets and len(value) > facets["maxLength"]:
                    raise ValueError("length {0}, maxLength {1}".format(len(value), facets["maxLength"]))
            if baseXsdType == "noContent":
                if len(value) > 0 and not value.isspace():
                    raise ValueError("value content not permitted")
                xValue = sValue = None
            elif baseXsdType in {"string", "normalizedString", "language", "token", "NMTOKEN","Name","NCName","IDREF","ENTITY"}:
                xValue = sValue = value
            elif baseXsdType == "ID":
                xValue = sValue = value
                xValid = VALID_ID
            elif baseXsdType == "anyURI":
                xValue = anyURI(value)
                sValue = value
                if xValue and not UrlUtil.isValid(xValue):  # allow empty strings to be valid anyURIs
                    raise ValueError("invalid anyURI value")
            elif not value: # rest of types get None if nil/empty value
                xValue = sValue = None
            elif baseXsdType in ("decimal", "float", "double"):
                xValue = sValue = float(value)
                if facets:
                    if "totalDigits" in facets and len(value.replace(".","")) > facets["totalDigits"]:
                        raise ValueError("totalDigits facet {0}".format(facets["totalDigits"]))
                    if "fractionDigits" in facets and ( '.' in value and
                        len(value[value.index('.') + 1:]) > facets["fractionDigits"]):
                        raise ValueError("fraction digits facet {0}".format(facets["fractionDigits"]))
            elif baseXsdType in {"integer",
                                 "nonPositiveInteger","negativeInteger","nonNegativeInteger","positiveInteger",
                                 "long","unsignedLong",
                                 "int","unsignedInt",
                                 "short","unsignedShort",
                                 "byte","unsignedByte"}:
                xValue = sValue = int(value)
                if ((baseXsdType in {"nonNegativeInteger","unsignedLong","unsignedInt"} 
                     and xValue < 0) or
                    (baseXsdType == "nonPositiveInteger" and xValue > 0) or
                    (baseXsdType == "positiveInteger" and xValue <= 0) or
                    (baseXsdType == "byte" and not -128 <= xValue < 127) or
                    (baseXsdType == "unsignedByte" and not 0 <= xValue < 255) or
                    (baseXsdType == "short" and not -32768 <= xValue < 32767) or
                    (baseXsdType == "unsignedShort" and not 0 <= xValue < 65535) or
                    (baseXsdType == "positiveInteger" and xValue <= 0)):
                    raise ValueError("{0} is not {1}".format(value, baseXsdType))
            elif baseXsdType == "boolean":
                if value in ("true", "1"):  
                    xValue = sValue = True
                elif value in ("false", "0"): 
                    xValue = sValue = False
                else: raise ValueError
            elif baseXsdType == "QName":
                xValue = qname(elt, value, castException=ValueError)
                sValue = value
                ''' not sure here, how are explicitDimensions validated, but bad units not?
                if xValue.namespaceURI in modelXbrl.namespaceDocs:
                    if (xValue not in modelXbrl.qnameConcepts and 
                        xValue not in modelXbrl.qnameTypes and
                        xValue not in modelXbrl.qnameAttributes and
                        xValue not in modelXbrl.qnameAttributeGroups):
                        raise ValueError("qname not defined " + str(xValue))
                '''
            elif baseXsdType in ("XBRLI_DECIMALSUNION", "XBRLI_PRECISIONUNION"):
                xValue = sValue = value if value == "INF" else int(value)
            elif baseXsdType in ("XBRLI_NONZERODECIMAL"):
                xValue = sValue = int(value)
                if xValue == 0:
                    raise ValueError("invalid value")
            elif baseXsdType == "XBRLI_DATEUNION":
                xValue = dateTime(value, type=DATEUNION, castException=ValueError)
                sValue = value
            elif baseXsdType == "dateTime":
                xValue = dateTime(value, type=DATETIME, castException=ValueError)
                sValue = value
            elif baseXsdType == "date":
                xValue = dateTime(value, type=DATE, castException=ValueError)
                sValue = value
            elif baseXsdType == "regex-pattern":
                # for facet compiling
                try:
                    xValue = re.compile(value + "$") # must match whole string
                    sValue = value
                except Exception as err:
                    raise ValueError(err)
            else:
                if baseXsdType in lexicalPatterns:
                    match = lexicalPatterns[baseXsdType].match(value)
                    if match is None:
                        raise ValueError("lexical pattern mismatch")
                    if baseXsdType == "gMonthDay":
                        month, day, zSign, zHrMin, zHr, zMin = match.groups()
                        if int(day) > {2:29, 4:30, 6:30, 9:30, 11:30, 1:31, 3:31, 5:31, 7:31, 8:31, 10:31, 12:31}[int(month)]:
                            raise ValueError("invalid day {0} for month {1}".format(day, month))
                xValue = value
                sValue = value
        except ValueError as err:
            if ModelInlineFact is not None and isinstance(elt, ModelInlineFact):
                errElt = "{0} fact {1}".format(elt.elementQname, elt.qname)
            else:
                errElt = elt.elementQname
            if attrTag:
                modelXbrl.error("xmlSchema:valueError",
                    _("Element %(element)s attribute %(attribute)s type %(typeName)s value error: %(value)s, %(error)s"),
                    modelObject=elt,
                    element=errElt,
                    attribute=XmlUtil.clarkNotationToPrefixedName(elt,attrTag,isAttribute=True),
                    typeName=baseXsdType,
                    value=value,
                    error=err)
            else:
                modelXbrl.error("xmlSchema:valueError",
                    _("Element %(element)s type %(typeName)s value error: %(value)s, %(error)s"),
                    modelObject=elt,
                    element=errElt,
                    typeName=baseXsdType,
                    value=value,
                    error=err)
            xValue = None
            sValue = value
            xValid = INVALID
    else:
        xValue = sValue = None
        xValid = UNKNOWN
    if attrTag:
        try:  # dynamically allocate attributes (otherwise given shared empty set)
            xAttributes = elt.xAttributes
        except AttributeError:
            elt.xAttributes = xAttributes = {}
        xAttributes[attrTag] = ModelAttribute(elt, attrTag, xValid, xValue, sValue, value)
    else:
        elt.xValid = xValid
        elt.xValue = xValue
        elt.sValue = sValue
Example #25
0
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri):
    from openpyxl import load_workbook
    from arelle import ModelDocument, ModelXbrl, XmlUtil
    from arelle.ModelDocument import ModelDocumentReference
    from arelle.ModelValue import qname
    
    try:
        currentAction = "initializing"
        startingErrorCount = len(modelXbrl.errors)
        startedAt = time.time()
        
        if os.path.isabs(oimFile):
            # allow relative filenames to loading directory
            priorCWD = os.getcwd()
            os.chdir(os.path.dirname(oimFile))
        else:
            priorCWD = None
            
        currentAction = "determining file type"
        isJSON = oimFile.endswith(".json") and not oimFile.endswith("-metadata.json")
        isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json")
        isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls")
        isCSVorXL = isCSV or isXL
        instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl"
        
        if isJSON:
            currentAction = "loading and parsing JSON OIM file"
            def loadDict(keyValuePairs):
                _dict = OrderedDict() # preserve fact order in resulting instance
                for key, value in keyValuePairs:
                    if isinstance(value, dict):
                        if DUPJSONKEY in value:
                            for _errKey, _errValue, _otherValue in value[DUPJSONKEY]:
                                if key == "prefixes":
                                    modelXbrl.error("oime:duplicatedPrefix",
                                                    _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"),
                                                    modelObject=modelXbrl, prefix=_errKey, uri1=_errValue, uri2=_otherValue)
                            del value[DUPJSONKEY]
                    if key in _dict:
                        if DUPJSONKEY not in _dict:
                            _dict[DUPJSONKEY] = []
                        _dict[DUPJSONKEY].append((key, value, _dict[key]))
                    else:
                        _dict[key] = value
                return _dict
            with io.open(oimFile, 'rt', encoding='utf-8') as f:
                oimObject = json.load(f, object_pairs_hook=loadDict)
            missing = [t for t in ("dtsReferences", "prefixes", "facts") if t not in oimObject]
            if missing:
                raise OIMException("oime:missingJSONElements", 
                                   _("Required element(s) are missing from JSON input: %(missing)s"),
                                   missing = ", ".join(missing))
            currentAction = "identifying JSON objects"
            dtsReferences = oimObject["dtsReferences"]
            prefixesList = oimObject["prefixes"].items()
            facts = oimObject["facts"]
            footnotes = oimObject["facts"] # shares this object
        elif isCSV:
            currentAction = "identifying CSV input tables"
            if sys.version[0] >= '3':
                csvOpenMode = 'w'
                csvOpenNewline = ''
            else:
                csvOpenMode = 'wb' # for 2.7
                csvOpenNewline = None
                
            oimFileBase = None
            if "-facts" in oimFile:
                oimFileBase = oimFile.partition("-facts")[0]
            else:
                for suffix in ("-dtsReferences.csv", "-defaults.csv", "-prefixes.csv", "-footnotes.csv", "-metadata.json"):
                    if oimFile.endswith(suffix):
                        oimFileBase = oimFile[:-len(suffix)]
                        break
            if oimFileBase is None:
                raise OIMException("oime:missingCSVTables", 
                                   _("Unable to identify CSV tables file name pattern"))
            if (not os.path.exists(oimFileBase + "-dtsReferences.csv") or
                not os.path.exists(oimFileBase + "-prefixes.csv")):
                raise OIMException("oime:missingCSVTables", 
                                   _("Unable to identify CSV tables for dtsReferences or prefixes"))
            instanceFileName = oimFileBase + ".xbrl"
            currentAction = "loading CSV dtsReferences table"
            dtsReferences = []
            with io.open(oimFileBase + "-dtsReferences.csv", 'rt', encoding='utf-8-sig') as f:
                csvReader = csv.reader(f)
                for i, row in enumerate(csvReader):
                    if i == 0:
                        header = row
                    else:
                        dtsReferences.append(dict((header[j], col) for j, col in enumerate(row)))
            currentAction = "loading CSV prefixes table"
            prefixesList = []
            with io.open(oimFileBase + "-prefixes.csv", 'rt', encoding='utf-8-sig') as f:
                csvReader = csv.reader(f)
                for i, row in enumerate(csvReader):
                    if i == 0:
                        header = dict((col,i) for i,col in enumerate(row))
                    else:
                        prefixesList.append((row[header["prefix"]], row[header["URI"]]))
            defaults = {}
            if os.path.exists(oimFileBase + "-defaults.csv"):
                currentAction = "loading CSV defaults table"
                with io.open(oimFileBase + "-defaults.csv", 'rt', encoding='utf-8-sig') as f:
                    csvReader = csv.reader(f)
                    for i, row in enumerate(csvReader):
                        if i == 0:
                            header = row
                            fileCol = row.index("file")
                        else:
                            defaults[row[fileCol]] = dict((header[j], col) for j, col in enumerate(row) if j != fileCol)
            currentAction = "loading CSV facts tables"
            facts = []
            _dir = os.path.dirname(oimFileBase)
            factsFileBasename = os.path.basename(oimFileBase) + "-facts"
            for filename in os.listdir(_dir):
                filepath = os.path.join(_dir, filename)
                if filename.startswith(factsFileBasename):
                    currentAction = "loading CSV facts table {}".format(filename)
                    tableDefaults = defaults.get(filename, {})
                    with io.open(filepath, 'rt', encoding='utf-8-sig') as f:
                        csvReader = csv.reader(f)
                        for i, row in enumerate(csvReader):
                            if i == 0:
                                header = row
                            else:
                                fact = {}
                                fact.update(tableDefaults)
                                for j, col in enumerate(row):
                                    if col is not None:
                                        if header[j]: # skip cols with no header
                                            if header[j].endswith("Value"):
                                                if col: # ignore empty columns (= null CSV value)
                                                    fact["value"] = col
                                            else:
                                                fact[header[j]] = col
                                facts.append(fact)
            footnotes = []
            if os.path.exists(oimFileBase + "-footnotes.csv"):
                currentAction = "loading CSV footnotes table"
                with io.open(oimFileBase + "-footnotes.csv", 'rt', encoding='utf-8-sig') as f:
                    csvReader = csv.reader(f)
                    for i, row in enumerate(csvReader):
                        if i == 0:
                            header = row
                        else:
                            footnotes.append(dict((header[j], col) for j, col in enumerate(row) if col))
        elif isXL:
            currentAction = "identifying workbook input worksheets"
            oimWb = load_workbook(oimFile, read_only=True, data_only=True)
            sheetNames = oimWb.get_sheet_names()
            if (not any(sheetName == "prefixes" for sheetName in sheetNames) or
                not any(sheetName == "dtsReferences" for sheetName in sheetNames) or
                not any("facts" in sheetName for sheetName in sheetNames)):
                raise OIMException("oime:missingWorkbookWorksheets", 
                                   _("Unable to identify worksheet tabs for dtsReferences, prefixes or facts"))
            currentAction = "loading worksheet: dtsReferences"
            dtsReferences = []
            for i, row in enumerate(oimWb["dtsReferences"]):
                if i == 0:
                    header = [col.value for col in row]
                else:
                    dtsReferences.append(dict((header[j], col.value) for j, col in enumerate(row)))
            currentAction = "loading worksheet: prefixes"
            prefixesList = []
            for i, row in enumerate(oimWb["prefixes"]):
                if i == 0:
                    header = dict((col.value,i) for i,col in enumerate(row))
                else:
                    prefixesList.append((row[header["prefix"]].value, row[header["URI"]].value))
            defaults = {}
            if "defaults" in sheetNames:
                currentAction = "loading worksheet: defaults"
                for i, row in enumerate(oimWb["defaults"]):
                    if i == 0:
                        header = dict((col.value,i) for i,col in enumerate(row))
                        fileCol = header["file"]
                    else:
                        defaults[row[fileCol].value] = dict((header[j], col.value) for j, col in enumerate(row) if j != fileCol)
            facts = []
            for sheetName in sheetNames:
                if sheetName == "facts" or "-facts" in sheetName:
                    currentAction = "loading worksheet: {}".format(sheetName)
                    tableDefaults = defaults.get(sheetName, {})
                    for i, row in enumerate(oimWb[sheetName]):
                        if i == 0:
                            header = [col.value for col in row]
                        else:
                            fact = {}
                            fact.update(tableDefaults)
                            for j, col in enumerate(row):
                                if col.value is not None:
                                    if header[j]: # skip cols with no header
                                        if header[j].endswith("Value"):
                                            fact["value"] = str(col.value)
                                        else:
                                            fact[header[j]] = str(col.value)
                            facts.append(fact)
            footnotes = []
            if "footnotes" in sheetNames:
                currentAction = "loading worksheet: footnotes"
                for i, row in enumerate(oimWb["footnotes"]):
                    if i == 0:
                        header = dict((col.value,i) for i,col in enumerate(row) if col.value)
                    else:
                        footnotes.append(dict((header[j], col.value) for j, col in enumerate(row) if col.value))
    
        currentAction = "identifying default dimensions"
        ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults 
        
        currentAction = "validating OIM"
        prefixes = {}
        prefixedUris = {}
        for _prefix, _uri in prefixesList:
            if not _prefix:
                modelXbrl.error("oime:emptyPrefix",
                                _("The empty string must not be used as a prefix, uri %(uri)s"),
                                modelObject=modelXbrl, uri=_uri)
            elif not NCNamePattern.match(_prefix):
                modelXbrl.error("oime:prefixPattern",
                                _("The prefix %(prefix)s must match the NCName lexical pattern, uri %(uri)s"),
                                modelObject=modelXbrl, prefix=_prefix, uri=_uri)
            elif _prefix in prefixes:
                modelXbrl.error("oime:duplicatedPrefix",
                                _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"),
                                modelObject=modelXbrl, prefix=_prefix, uri1=prefixes[_prefix], uri2=_uri)
            elif _uri in prefixedUris:
                modelXbrl.error("oime:duplicatedUri",
                                _("The uri %(uri)s is used on prefix %(prefix1)s and prefix %(prefix2)s"),
                                modelObject=modelXbrl, uri=_uri, prefix1=prefixedUris[_uri], prefix2=_prefix)
            else:
                prefixes[_prefix] = _uri
                prefixedUris[_uri] = _prefix
                
        oimPrefix = None
        for _nsOim in nsOim:
            if _nsOim in prefixedUris:
                oimPrefix = prefixedUris[_nsOim]
        if not oimPrefix:
            raise OIMException("oime:noOimPrefix",
                               _("The oim namespace must have a declared prefix"))
        oimConcept = "{}:concept".format(oimPrefix)
        oimEntity = "{}:entity".format(oimPrefix)
        oimPeriod = "{}:period".format(oimPrefix)
        oimPeriodStart = "{}:periodStart".format(oimPrefix)
        oimPeriodEnd = "{}:periodEnd".format(oimPrefix)
        oimUnit = "{}:unit".format(oimPrefix)
        oimPrefix = "{}:".format(oimPrefix)
            
        # create the instance document
        currentAction = "creating instance document"
        modelXbrl.blockDpmDBrecursion = True
        modelXbrl.modelDocument = createModelDocument(
              modelXbrl, 
              Type.INSTANCE,
              instanceFileName,
              schemaRefs=[dtsRef["href"] for dtsRef in dtsReferences if dtsRef["type"] == "schema"],
              isEntry=True,
              initialComment="extracted from OIM {}".format(mappedUri),
              documentEncoding="utf-8")
        cntxTbl = {}
        unitTbl = {}
        currentAction = "creating facts"
        for fact in facts:
            conceptQn = qname(fact[oimConcept], prefixes)
            concept = modelXbrl.qnameConcepts.get(conceptQn)
            entityAsQn = qname(fact[oimEntity], prefixes)
            if oimPeriod in fact:
                periodStart = fact[oimPeriod]["start"]
                periodEnd = fact[oimPeriod]["end"]
            else:
                periodStart = fact[oimPeriodStart]
                periodEnd = fact[oimPeriodEnd]
            cntxKey = ( # hashable context key
                ("periodType", concept.periodType),
                ("entity", entityAsQn),
                ("periodStart", periodStart),
                ("periodEnd", periodEnd)) + tuple(sorted(
                    (dimName, dimVal) 
                    for dimName, dimVal in fact.items()
                    if ":" in dimName and not dimName.startswith(oimPrefix)))
            if cntxKey in cntxTbl:
                _cntx = cntxTbl[cntxKey]
            else:
                cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
                qnameDims = {}
                for dimName, dimVal in fact.items():
                    if ":" in dimName and not dimName.startswith(oimPrefix) and dimVal:
                        dimQname = qname(dimName, prefixes)
                        dimConcept = modelXbrl.qnameConcepts.get(dimQname)
                        if ":" in dimVal and dimVal.partition(':')[0] in prefixes:
                            mem = qname(dimVal, prefixes) # explicit dim
                        elif dimConcept.isTypedDimension:
                            # a modelObject xml element is needed for all of the instance functions to manage the typed dim
                            mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False)
                        qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment")
                _cntx = modelXbrl.createContext(
                                        entityAsQn.namespaceURI,
                                        entityAsQn.localName,
                                        concept.periodType,
                                        None if concept.periodType == "instant" else dateTime(periodStart, type=DATETIME),
                                        dateTime(periodEnd, type=DATETIME),
                                        None, # no dimensional validity checking (like formula does)
                                        qnameDims, [], [],
                                        id=cntxId)
                cntxTbl[cntxKey] = _cntx
            if oimUnit in fact:
                unitKey = fact[oimUnit]
                if unitKey in unitTbl:
                    _unit = unitTbl[unitKey]
                else:
                    _unit = None
                    # validate unit
                    unitKeySub = PrefixedQName.sub(UnitPrefixedQNameSubstitutionChar, unitKey)
                    if not UnitPattern.match(unitKeySub):
                        modelXbrl.error("oime:unitStringRepresentation",
                                        _("Unit string representation is lexically invalid, %(unit)s"),
                                        modelObject=modelXbrl, unit=unitKey)
                    else:
                        _mul, _sep, _div = unitKey.partition('/')
                        if _mul.startswith('('):
                            _mul = _mul[1:-1]
                        _muls = [u for u in _mul.split('*') if u]
                        if _div.startswith('('):
                            _div = _div[1:-1]
                        _divs = [u for u in _div.split('*') if u]
                        if _muls != sorted(_muls) or _divs != sorted(_divs):
                            modelXbrl.error("oime:unitStringRepresentation",
                                            _("Unit string representation measures are not in alphabetical order, %(unit)s"),
                                            modelObject=modelXbrl, unit=unitKey)
                        try:
                            mulQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix",
                                                                                      _("Unit prefix is not declared: %(unit)s"),
                                                                                      unit=u)) 
                                      for u in _muls]
                            divQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix",
                                                                                      _("Unit prefix is not declared: %(unit)s"),
                                                                                      unit=u))
                                      for u in _divs]
                            unitId = 'u-{:02}'.format(len(unitTbl) + 1)
                            for _measures in mulQns, divQns:
                                for _measure in _measures:
                                    addQnameValue(modelXbrl.modelDocument, _measure)
                            _unit = modelXbrl.createUnit(mulQns, divQns, id=unitId)
                        except OIMException as ex:
                            modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs)
                    unitTbl[unitKey] = _unit
            else:
                _unit = None
            
            attrs = {"contextRef": _cntx.id}
    
            if fact.get("value") is None:
                attrs[XbrlConst.qnXsiNil] = "true"
                text = None
            else:
                text = fact["value"]
                
            if fact.get("id"):
                attrs["id"] = fact["id"]
                
            if concept.isNumeric:
                if _unit is None:
                    continue # skip creating fact because unit was invalid
                attrs["unitRef"] = _unit.id
                if "accuracy" in fact:
                    attrs["decimals"] = fact["accuracy"]
                    
            # is value a QName?
            if concept.baseXbrliType == "QName":
                addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes))
    
            f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
            
        currentAction = "creating footnotes"
        footnoteLinks = {} # ELR elements
        factLocs = {} # index by (linkrole, factId)
        footnoteNbr = 0
        locNbr = 0
        for factOrFootnote in footnotes:
            if "factId" in factOrFootnote:
                factId = factOrFootnote["factId"]
                factFootnotes = (factOrFootnote,) # CSV or XL
            elif "id" in factOrFootnote and "footnotes" in factOrFootnote:
                factId = factOrFootnote["id"]
                factFootnotes = factOrFootnote["footnotes"]
            else:
                factFootnotes = ()
            for footnote in factFootnotes:
                linkrole = footnote.get("group")
                arcrole = footnote.get("footnoteType")
                if not factId or not linkrole or not arcrole or not (
                    footnote.get("factRef") or footnote.get("footnote")):
                    # invalid footnote
                    continue
                if linkrole not in footnoteLinks:
                    footnoteLinks[linkrole] = addChild(modelXbrl.modelDocument.xmlRootElement, 
                                                       XbrlConst.qnLinkFootnoteLink, 
                                                       attributes={"{http://www.w3.org/1999/xlink}type": "extended",
                                                                   "{http://www.w3.org/1999/xlink}role": linkrole})
                footnoteLink = footnoteLinks[linkrole]
                if (linkrole, factId) not in factLocs:
                    locNbr += 1
                    locLabel = "l_{:02}".format(locNbr)
                    factLocs[(linkrole, factId)] = locLabel
                    addChild(footnoteLink, XbrlConst.qnLinkLoc, 
                             attributes={XLINKTYPE: "locator",
                                         XLINKHREF: "#" + factId,
                                         XLINKLABEL: locLabel})
                locLabel = factLocs[(linkrole, factId)]
                if footnote.get("footnote"):
                    footnoteNbr += 1
                    footnoteLabel = "f_{:02}".format(footnoteNbr)
                    attrs = {XLINKTYPE: "resource",
                             XLINKLABEL: footnoteLabel}
                    if footnote.get("language"):
                        attrs[XMLLANG] = footnote["language"]
                    # note, for HTML will need to build an element structure
                    addChild(footnoteLink, XbrlConst.qnLinkFootnote, attributes=attrs, text=footnote["footnote"])
                elif footnote.get("factRef"):
                    factRef = footnote.get("factRef")
                    if (linkrole, factRef) not in factLocs:
                        locNbr += 1
                        locLabel = "f_{:02}".format(footnoteNbr)
                        factLoc[(linkrole, factRef)] = locLabel
                        addChild(footnoteLink, XbrlConst.qnLinkLoc, 
                                 attributes={XLINKTYPE: "locator",
                                             XLINKHREF: "#" + factRef,
                                             XLINKLABEL: locLabel})
                    footnoteLabel = factLoc[(linkrole, factId)]
                footnoteArc = addChild(footnoteLink, 
                                       XbrlConst.qnLinkFootnoteArc, 
                                       attributes={XLINKTYPE: "arc",
                                                   XLINKARCROLE: arcrole,
                                                   XLINKFROM: locLabel,
                                                   XLINKTO: footnoteLabel})
                    
        currentAction = "done loading facts and footnotes"
        
        #cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt),
        #               messageCode="loadFromExcel:info")
    except Exception as ex:
        if isinstance(ex, OIMException):
            modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs)
        else:
            modelXbrl.error("arelleOIMloader:error",
                            "Error while %(action)s, error %(error)s\ntraceback %(traceback)s",
                            modelObject=modelXbrl, action=currentAction, error=ex,
                            traceback=traceback.format_tb(sys.exc_info()[2]))
    
    if priorCWD:
        os.chdir(priorCWD) # restore prior current working directory            startingErrorCount = len(modelXbrl.errors)
        
    if startingErrorCount < len(modelXbrl.errors):
        # had errors, don't allow ModelDocument.load to continue
        return OIMException("arelleOIMloader:unableToLoad", "Unable to load due to reported errors")

    return getattr(modelXbrl, "modelDocument", None) # none if returning from exception
Example #26
0
def current_date(xc, p, contextItem, args):
    from datetime import date
    return dateTime(date.today(), type=DATE)
Example #27
0
     def createModelFact(fact, parentModelFact, topTupleFact):
         aspects = fact.get("aspects", EMPTYDICT)
         if oimConcept not in aspects:
             modelXbrl.error("{}:conceptQName".format(errPrefix),
                             _("The concept QName could not be determined"),
                             modelObject=modelXbrl)
             return
         conceptQn = qname(aspects[oimConcept], prefixes)
         concept = modelXbrl.qnameConcepts.get(conceptQn)
         attrs = {}
         if concept.isItem:
             missingAspects = []
             if oimEntity not in aspects: 
                 missingAspects.append(oimEntity)
             if oimPeriod not in aspects and (oimPeriodStart not in aspects or oimPeriodEnd not in aspects):
                 missingAspects.append(oimPeriod)
             if missingAspects:
                 modelXbrl.error("{}:missingAspects".format(errPrefix),
                                 _("The concept %(element)s is missing aspects %(missingAspects)s"),
                                 modelObject=modelXbrl, element=conceptQn, missingAspects=", ".join(missingAspects))
                 return
             entityAsQn = qname(aspects[oimEntity], prefixes) or qname("error",fact[oimEntity])
             if oimPeriod in aspects:
                 periodStart = periodEnd = aspects[oimPeriod]
             if oimPeriodStart in aspects and oimPeriodEnd in aspects:
                 periodStart = aspects[oimPeriodStart]
                 periodEnd = aspects[oimPeriodEnd]
             cntxKey = ( # hashable context key
                 ("periodType", concept.periodType),
                 ("entity", entityAsQn),
                 ("periodStart", periodStart),
                 ("periodEnd", periodEnd)) + tuple(sorted(
                     (dimName, dimVal["value"] if isinstance(dimVal,dict) else dimVal) 
                     for dimName, dimVal in aspects.items()
                     if ":" in dimName and not dimName.startswith(oimPrefix)))
             if cntxKey in cntxTbl:
                 _cntx = cntxTbl[cntxKey]
             else:
                 cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
                 qnameDims = {}
                 for dimName, dimVal in aspects.items():
                     if ":" in dimName and not dimName.startswith(oimPrefix) and dimVal:
                         dimQname = qname(dimName, prefixes)
                         dimConcept = modelXbrl.qnameConcepts.get(dimQname)
                         if dimConcept is None:
                             modelXbrl.error("{}:taxonomyDefinedAspectQName".format(errPrefix),
                                             _("The taxonomy defined aspect concept QName %(qname)s could not be determined"),
                                             modelObject=modelXbrl, qname=dimQname)
                             continue
                         if isinstance(dimVal, dict):
                             dimVal = dimVal["value"]
                         else:
                             dimVal = str(dimVal) # may be int or boolean
                         if isinstance(dimVal,str) and ":" in dimVal and dimVal.partition(':')[0] in prefixes:
                             mem = qname(dimVal, prefixes) # explicit dim
                         elif dimConcept.isTypedDimension:
                             # a modelObject xml element is needed for all of the instance functions to manage the typed dim
                             mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False)
                         qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment")
                 _cntx = modelXbrl.createContext(
                                         entityAsQn.namespaceURI,
                                         entityAsQn.localName,
                                         concept.periodType,
                                         None if concept.periodType == "instant" else dateTime(periodStart, type=DATETIME),
                                         dateTime(periodEnd, type=DATETIME),
                                         None, # no dimensional validity checking (like formula does)
                                         qnameDims, [], [],
                                         id=cntxId,
                                         beforeSibling=topTupleFact)
                 cntxTbl[cntxKey] = _cntx
             if oimUnit in aspects and concept.isNumeric:
                 unitKey = aspects[oimUnit]
                 if unitKey in unitTbl:
                     _unit = unitTbl[unitKey]
                 else:
                     _unit = None
                     # validate unit
                     unitKeySub = PrefixedQName.sub(UnitPrefixedQNameSubstitutionChar, unitKey)
                     if not UnitPattern.match(unitKeySub):
                         modelXbrl.error("oime:unitStringRepresentation",
                                         _("Unit string representation is lexically invalid, %(unit)s"),
                                         modelObject=modelXbrl, unit=unitKey)
                     else:
                         _mul, _sep, _div = unitKey.partition('/')
                         if _mul.startswith('('):
                             _mul = _mul[1:-1]
                         _muls = [u for u in _mul.split('*') if u]
                         if _div.startswith('('):
                             _div = _div[1:-1]
                         _divs = [u for u in _div.split('*') if u]
                         if _muls != sorted(_muls) or _divs != sorted(_divs):
                             modelXbrl.error("oime:unitStringRepresentation",
                                             _("Unit string representation measures are not in alphabetical order, %(unit)s"),
                                             modelObject=modelXbrl, unit=unitKey)
                         try:
                             mulQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix",
                                                                                       _("Unit prefix is not declared: %(unit)s"),
                                                                                       unit=u)) 
                                       for u in _muls]
                             divQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix",
                                                                                       _("Unit prefix is not declared: %(unit)s"),
                                                                                       unit=u))
                                       for u in _divs]
                             unitId = 'u-{:02}'.format(len(unitTbl) + 1)
                             for _measures in mulQns, divQns:
                                 for _measure in _measures:
                                     addQnameValue(modelXbrl.modelDocument, _measure)
                             _unit = modelXbrl.createUnit(mulQns, divQns, id=unitId, beforeSibling=topTupleFact)
                         except OIMException as ex:
                             modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs)
                     unitTbl[unitKey] = _unit
             else:
                 _unit = None
         
             attrs["contextRef"] = _cntx.id
     
             if fact.get("value") is None:
                 attrs[XbrlConst.qnXsiNil] = "true"
                 text = None
             else:
                 text = fact["value"]
                 
             if concept.isNumeric:
                 if _unit is None:
                     return # skip creating fact because unit was invalid
                 attrs["unitRef"] = _unit.id
                 if "accuracy" in attrs or attrs.get(XbrlConst.qnXsiNil, "false") != "true":
                     attrs["decimals"] = fact.get("accuracy", "INF")
         else:
             text = None #tuple
                 
         id = fact.get("id")
         if id is not None:
             attrs["id"] = fact["id"]
                 
         # is value a QName?
         if concept.baseXbrliType == "QName":
             addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes))
 
         f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text, parent=parentModelFact, validate=False)
         
         if id is not None and id in parentedFacts:
             # create child facts
             for i in sorted(parentedFacts[id],
                             key=lambda j: facts[j].get("aspects", EMPTYDICT).get(oimTupleOrder,0)):
                 createModelFact(facts[i], f, f if topTupleFact is None else topTupleFact)
                 
         # validate after creating tuple contents
         xmlValidate(modelXbrl, f)
Example #28
0
from arelle import ModelDocument, XmlUtil
from arelle.ModelValue import qname, dateTime, DATE
from arelle.ValidateXbrlCalcs import insignificantDigits
from arelle.XbrlConst import xbrli
try:
    import regex as re
except ImportError:
    import re
from collections import defaultdict

memNameNumPattern = re.compile(r"^([A-Za-z-]+)([0-9]+)$")
compTxmyNamespacePattern = re.compile(r"http://www.govtalk.gov.uk/uk/fr/tax/uk-hmrc-ct/[0-9-]{10}")
# capture background-image or list-style-image with URL function
styleImgUrlPattern = re.compile(r"[a-z]+-image:\s*url[(][^)]+[)]")
EMPTYDICT = {}
_6_APR_2008 = dateTime("2008-04-06", type=DATE)

commonMandatoryItems = {
    "EntityCurrentLegalOrRegisteredName", "StartDateForPeriodCoveredByReport", 
    "EndDateForPeriodCoveredByReport", "BalanceSheetDate"}
mandatoryItems = {
    "ukGAAP": commonMandatoryItems | {
        "DateApprovalAccounts", "NameDirectorSigningAccounts", "EntityDormant", "EntityTrading",
        "DateSigningDirectorsReport", "DirectorSigningReport"},
    "charities": commonMandatoryItems | {
        "DateApprovalAccounts", "NameTrusteeSigningAccounts", "EntityDormant", "EntityTrading",
        "DateSigningTrusteesReport", "TrusteeSigningReport"},
    "ukIFRS": commonMandatoryItems | {
        "DateAuthorisationFinancialStatementsForIssue", "ExplanationOfBodyOfAuthorisation", 
        "EntityDormant", "EntityTrading", "DateSigningDirectorsReport", "DirectorSigningReport"},
    "FRS": commonMandatoryItems | {
Example #29
0
 def atomize(self, p, x):
     # sequence
     if isinstance(x, (tuple,list,set)):
         sequence = []
         for item in self.flattenSequence(x):
             atomizedItem = self.atomize(p, item)
             if atomizedItem != []:
                 sequence.append(atomizedItem)
         return sequence
     # individual items
     if isinstance(x, range): 
         return x
     baseXsdType = None
     e = None
     if isinstance(x, ModelFact):
         if x.isTuple:
             raise XPathException(p, 'err:FOTY0012', _('Atomizing tuple {0} that does not have a typed value').format(x))
         if x.isNil:
             return []
         baseXsdType = x.concept.baseXsdType
         v = x.value # resolves default value
         e = x
     elif isinstance(x, ModelAttribute): # ModelAttribute is a tuple (below), check this first!
         return x.xValue
     else:
         if isinstance(x, ModelObject):
             e = x
         if e is not None:
             if e.get("{http://www.w3.org/2001/XMLSchema-instance}nil") == "true":
                 return []
             try:
                 if e.xValid >= VALID:
                     return e.xValue
             except AttributeError:
                 pass
             modelXbrl = x.modelXbrl
             modelConcept = modelXbrl.qnameConcepts.get(qname(x))
             if modelConcept is not None:
                 baseXsdType = modelConcept.baseXsdType
             v = XmlUtil.text(x)
     if baseXsdType in ("decimal", "float", "double"):
         try:
             x = float(v)
         except ValueError:
             raise XPathException(p, 'err:FORG0001', _('Atomizing {0} to a {1} does not have a proper value').format(x,baseXsdType))
     elif baseXsdType in ("integer",):
         try:
             x = int(v)
         except ValueError:
             raise XPathException(p, 'err:FORG0001', _('Atomizing {0} to an integer does not have a proper value').format(x))
     elif baseXsdType == "boolean":
         x = (v == "true" or v == "1")
     elif baseXsdType == "QName" and e is not None:
         x = qname(e, v)
     elif baseXsdType == "anyURI":
         x = anyURI(v.strip())
     elif baseXsdType in ("normalizedString","token","language","NMTOKEN","Name","NCName","ID","IDREF","ENTITY"):
         x = v.strip()
     elif baseXsdType == "XBRLI_DATEUNION":
         x = dateTime(v, type=DATEUNION)
     elif baseXsdType == "date":
         x = dateTime(v, type=DATE)
     elif baseXsdType == "dateTime":
         x = dateTime(v, type=DATETIME)
     elif baseXsdType:
         x = str(v)
     return x
Example #30
0
 def atomize(self, p, x):
     # sequence
     if isinstance(x, SEQUENCE_TYPES):
         sequence = []
         for item in self.flattenSequence(x):
             atomizedItem = self.atomize(p, item)
             if atomizedItem != []:
                 sequence.append(atomizedItem)
         return sequence
     # individual items
     if isinstance(x, _RANGE):
         return x
     baseXsdType = None
     e = None
     if isinstance(x, ModelFact):
         if x.isTuple:
             raise XPathException(
                 p, 'err:FOTY0012',
                 _('Atomizing tuple {0} that does not have a typed value').
                 format(x))
         if x.isNil:
             return []
         baseXsdType = x.concept.baseXsdType
         v = x.value  # resolves default value
         e = x
     elif isinstance(
             x, ModelAttribute
     ):  # ModelAttribute is a tuple (below), check this first!
         return x.xValue
     else:
         if isinstance(x, ModelObject):
             e = x
         if e is not None:
             if getattr(e, "xValid", 0) == VALID_NO_CONTENT:
                 raise XPathException(
                     p, 'err:FOTY0012',
                     _('Atomizing element {0} that does not have a typed value'
                       ).format(x))
             if e.get("{http://www.w3.org/2001/XMLSchema-instance}nil"
                      ) == "true":
                 return []
             try:
                 if e.xValid >= VALID:
                     return e.xValue
             except AttributeError:
                 pass
             modelXbrl = x.modelXbrl
             modelConcept = modelXbrl.qnameConcepts.get(qname(x))
             if modelConcept is not None:
                 baseXsdType = modelConcept.baseXsdType
             v = x.stringValue
     if baseXsdType in ("float", "double"):
         try:
             x = float(v)
         except ValueError:
             raise XPathException(
                 p, 'err:FORG0001',
                 _('Atomizing {0} to a {1} does not have a proper value').
                 format(x, baseXsdType))
     elif baseXsdType == "decimal":
         try:
             x = Decimal(v)
         except InvalidOperation:
             raise XPathException(
                 p, 'err:FORG0001',
                 _('Atomizing {0} to decimal does not have a proper value'))
     elif baseXsdType in ("integer", "nonPositiveInteger",
                          "negativeInteger", "nonNegativeInteger",
                          "positiveInteger", "long", "unsignedLong", "int",
                          "unsignedInt", "short", "unsignedShort", "byte",
                          "unsignedByte"):
         try:
             x = _INT(v)
         except ValueError:
             raise XPathException(
                 p, 'err:FORG0001',
                 _('Atomizing {0} to an integer does not have a proper value'
                   ).format(x))
     elif baseXsdType == "boolean":
         x = (v == "true" or v == "1")
     elif baseXsdType == "QName" and e is not None:
         x = qname(e, v)
     elif baseXsdType == "anyURI":
         x = anyURI(v.strip())
     elif baseXsdType in ("normalizedString", "token", "language",
                          "NMTOKEN", "Name", "NCName", "ID", "IDREF",
                          "ENTITY"):
         x = v.strip()
     elif baseXsdType == "XBRLI_DATEUNION":
         x = dateTime(v, type=DATEUNION)
     elif baseXsdType == "date":
         x = dateTime(v, type=DATE)
     elif baseXsdType == "dateTime":
         x = dateTime(v, type=DATETIME)
     elif baseXsdType in GREGORIAN_TYPES and isinstance(v, GREGORIAN_TYPES):
         x = v
     elif baseXsdType == "noContent":
         x = None  # can't be atomized
     elif baseXsdType:
         x = str(v)
     return x
Example #31
0
 def atomize(self, p, x):
     # sequence
     if hasattr(x, '__iter__') and not isinstance(x, str):
         sequence = []
         for item in self.flattenSequence(x):
             atomizedItem = self.atomize(p, item)
             if atomizedItem != []:
                 sequence.append(atomizedItem)
         return sequence
     # individual items
     if isinstance(x, range):
         return x
     baseXsdType = None
     e = None
     if isinstance(x, ModelObject.ModelFact):
         if x.isTuple:
             raise XPathException(
                 p, 'err:FOTY0012',
                 _('Atomizing tuple {0} that does not have a typed value').
                 format(x))
         if x.isNil:
             return []
         baseXsdType = x.concept.baseXsdType
         v = x.value  # resolves default value
         e = x.element
     else:
         if isinstance(x, ModelObject.ModelObject):
             e = x.element
         elif isinstance(x, xml.dom.Node):
             e = x
         if e:
             if x.nodeType == xml.dom.Node.ELEMENT_NODE:
                 if e.getAttributeNS(XbrlConst.xsi, "nil") == "true":
                     return []
                 modelXbrl = x.ownerDocument.modelDocument.modelXbrl
                 modelConcept = modelXbrl.qnameConcepts.get(qname(x))
                 if modelConcept:
                     baseXsdType = modelConcept.baseXsdType
                 v = XmlUtil.text(x)
             elif x.nodeType == xml.dom.Node.ATTRIBUTE_NODE:
                 e = x.ownerElement
                 modelXbrl = e.ownerDocument.modelDocument.modelXbrl
                 if x.namespaceURI:
                     attrQname = qname(x.namespaceURI, x.localName)
                 else:
                     attrQname = qname(x.localName)
                 modelConcept = modelXbrl.qnameConcepts.get(qname(e))
                 if modelConcept:
                     baseXsdType = modelConcept.baseXsdAttrType(
                         attrQname) if modelConcept else None
                 if baseXsdType is None:
                     attrObject = modelXbrl.qnameAttributes.get(attrQname)
                     if attrObject:
                         baseXsdType = attrObject.baseXsdType
                 v = x.value
     if baseXsdType in ("decimal", "float", "double"):
         try:
             x = float(v)
         except ValueError:
             raise XPathException(
                 p, 'err:FORG0001',
                 _('Atomizing {0} to a {1} does not have a proper value').
                 format(x, baseXsdType))
     elif baseXsdType in ("integer", ):
         try:
             x = int(v)
         except ValueError:
             raise XPathException(
                 p, 'err:FORG0001',
                 _('Atomizing {0} to an integer does not have a proper value'
                   ).format(x))
     elif baseXsdType == "boolean":
         x = (v == "true" or v == "1")
     elif baseXsdType == "QName" and e:
         x = qname(e, v)
     elif baseXsdType == "anyURI":
         x = anyURI(v.strip())
     elif baseXsdType in ("normalizedString", "token", "language",
                          "NMTOKEN", "Name", "NCName", "ID", "IDREF",
                          "ENTITY"):
         x = v.strip()
     elif baseXsdType == "XBRLI_DATEUNION":
         x = dateTime(v, type=DATEUNION)
     elif baseXsdType == "date":
         x = dateTime(v, type=DATE)
     elif baseXsdType == "dateTime":
         x = dateTime(v, type=DATETIME)
     elif baseXsdType:
         x = str(v)
     return x
Example #32
0
def validateValue(modelXbrl,
                  elt,
                  attrTag,
                  baseXsdType,
                  value,
                  isNillable=False,
                  facets=None):
    if baseXsdType:
        try:
            if (len(value) == 0 and not attrTag is None and not isNillable
                    and baseXsdType
                    not in ("anyType", "string", "normalizedString", "token",
                            "NMTOKEN", "anyURI", "noContent")):
                raise ValueError("missing value for not nillable element")
            xValid = VALID
            whitespaceReplace = (baseXsdType == "normalizedString")
            whitespaceCollapse = (not whitespaceReplace
                                  and baseXsdType != "string")
            pattern = baseXsdTypePatterns.get(baseXsdType)
            if facets:
                if "pattern" in facets:
                    pattern = facets["pattern"]
                    # note multiple patterns are or'ed togetner, which isn't yet implemented!
                if "whiteSpace" in facets:
                    whitespaceReplace, whitespaceCollapse = {
                        "preserve": (False, False),
                        "replace": (True, False),
                        "collapse": (False, True)
                    }
            if whitespaceReplace:
                value = normalizeWhitespacePattern.sub(' ', value)
            elif whitespaceCollapse:
                value = collapseWhitespacePattern.sub(' ', value.strip())
            if pattern is not None and pattern.match(value) is None:
                raise ValueError("pattern facet " +
                                 facets["pattern"].pattern if facets and
                                 "pattern" in facets else "pattern mismatch")
            if facets:
                if "enumeration" in facets and value not in facets[
                        "enumeration"]:
                    raise ValueError("is not in {1}".format(
                        value, facets["enumeration"]))
                if "length" in facets and len(value) != facets["length"]:
                    raise ValueError("length {0}, expected {1}".format(
                        len(value), facets["length"]))
                if "minLength" in facets and len(value) < facets["minLength"]:
                    raise ValueError("length {0}, minLength {1}".format(
                        len(value), facets["minLength"]))
                if "maxLength" in facets and len(value) > facets["maxLength"]:
                    raise ValueError("length {0}, maxLength {1}".format(
                        len(value), facets["maxLength"]))
            if baseXsdType == "noContent":
                if len(value) > 0 and not value.isspace():
                    raise ValueError("value content not permitted")
                xValue = sValue = None
            elif baseXsdType in {
                    "string", "normalizedString", "language", "token",
                    "NMTOKEN", "Name", "NCName", "IDREF", "ENTITY"
            }:
                xValue = sValue = value
            elif baseXsdType == "ID":
                xValue = sValue = value
                xValid = VALID_ID
            elif baseXsdType == "anyURI":
                xValue = anyURI(value)
                sValue = value
                if xValue and not UrlUtil.isValid(
                        xValue):  # allow empty strings to be valid anyURIs
                    raise ValueError("invalid anyURI value")
            elif not value:  # rest of types get None if nil/empty value
                xValue = sValue = None
            elif baseXsdType in ("decimal", "float", "double"):
                xValue = sValue = float(value)
                if facets:
                    if "totalDigits" in facets and len(value.replace(
                            ".", "")) > facets["totalDigits"]:
                        raise ValueError("totalDigits facet {0}".format(
                            facets["totalDigits"]))
                    if "fractionDigits" in facets and (
                            '.' in value and len(value[value.index('.') + 1:])
                            > facets["fractionDigits"]):
                        raise ValueError("fraction digits facet {0}".format(
                            facets["fractionDigits"]))
            elif baseXsdType in ("integer", ):
                xValue = sValue = int(value)
            elif baseXsdType == "boolean":
                if value in ("true", "1"):
                    xValue = sValue = True
                elif value in ("false", "0"):
                    xValue = sValue = False
                else:
                    raise ValueError
            elif baseXsdType == "QName":
                xValue = qname(elt, value, castException=ValueError)
                sValue = value
                ''' not sure here, how are explicitDimensions validated, but bad units not?
                if xValue.namespaceURI in modelXbrl.namespaceDocs:
                    if (xValue not in modelXbrl.qnameConcepts and 
                        xValue not in modelXbrl.qnameTypes and
                        xValue not in modelXbrl.qnameAttributes and
                        xValue not in modelXbrl.qnameAttributeGroups):
                        raise ValueError("qname not defined " + str(xValue))
                '''
            elif baseXsdType in ("XBRLI_DECIMALSUNION",
                                 "XBRLI_PRECISIONUNION"):
                xValue = sValue = value if value == "INF" else int(value)
            elif baseXsdType in ("XBRLI_NONZERODECIMAL"):
                xValue = sValue = int(value)
                if xValue == 0:
                    raise ValueError("invalid value")
            elif baseXsdType == "XBRLI_DATEUNION":
                xValue = dateTime(value,
                                  type=DATEUNION,
                                  castException=ValueError)
                sValue = value
            elif baseXsdType == "dateTime":
                xValue = dateTime(value,
                                  type=DATETIME,
                                  castException=ValueError)
                sValue = value
            elif baseXsdType == "date":
                xValue = dateTime(value, type=DATE, castException=ValueError)
                sValue = value
            elif baseXsdType == "regex-pattern":
                # for facet compiling
                try:
                    xValue = re.compile(value + "$")  # must match whole string
                    sValue = value
                except Exception as err:
                    raise ValueError(err)
            else:
                if baseXsdType in lexicalPatterns:
                    match = lexicalPatterns[baseXsdType].match(value)
                    if match is None:
                        raise ValueError("lexical pattern mismatch")
                    if baseXsdType == "gMonthDay":
                        month, day, zSign, zHrMin, zHr, zMin = match.groups()
                        if int(day) > {
                                2: 29,
                                4: 30,
                                6: 30,
                                9: 30,
                                11: 30,
                                1: 31,
                                3: 31,
                                5: 31,
                                7: 31,
                                8: 31,
                                10: 31,
                                12: 31
                        }[int(month)]:
                            raise ValueError(
                                "invalid day {0} for month {1}".format(
                                    day, month))
                xValue = value
                sValue = value
        except ValueError as err:
            if ModelInlineFact is not None and isinstance(
                    elt, ModelInlineFact):
                errElt = "{0} fact {1}".format(elt.elementQname, elt.qname)
            else:
                errElt = elt.elementQname
            if attrTag:
                modelXbrl.error(
                    "xmlSchema:valueError",
                    _("Element %(element)s attribute %(attribute)s type %(typeName)s value error: %(value)s, %(error)s"
                      ),
                    modelObject=elt,
                    element=errElt,
                    attribute=XmlUtil.clarkNotationToPrefixedName(
                        elt, attrTag, isAttribute=True),
                    typeName=baseXsdType,
                    value=value,
                    error=err)
            else:
                modelXbrl.error(
                    "xmlSchema:valueError",
                    _("Element %(element)s type %(typeName)s value error: %(value)s, %(error)s"
                      ),
                    modelObject=elt,
                    element=errElt,
                    typeName=baseXsdType,
                    value=value,
                    error=err)
            xValue = None
            sValue = value
            xValid = INVALID
    else:
        xValue = sValue = None
        xValid = UNKNOWN
    if attrTag:
        elt.xAttributes[attrTag] = ModelAttribute(elt, attrTag, xValid, xValue,
                                                  sValue, value)
    else:
        elt.xValid = xValid
        elt.xValue = xValue
        elt.sValue = sValue
Example #33
0
 def atomize(self, p, x):
     # sequence
     if hasattr(x, '__iter__') and not isinstance(x, str):
         sequence = []
         for item in self.flattenSequence(x):
             atomizedItem = self.atomize(p, item)
             if atomizedItem != []:
                 sequence.append(atomizedItem)
         return sequence
     # individual items
     if isinstance(x, range): 
         return x
     baseXsdType = None
     e = None
     if isinstance(x, ModelObject.ModelFact):
         if x.isTuple:
             raise XPathException(p, 'err:FOTY0012', _('Atomizing tuple {0} that does not have a typed value').format(x))
         if x.isNil:
             return []
         baseXsdType = x.concept.baseXsdType
         v = x.value # resolves default value
         e = x.element
     else:
         if isinstance(x, ModelObject.ModelObject):
             e = x.element
         elif isinstance(x, xml.dom.Node):
             e = x
         if e:
             if x.nodeType == xml.dom.Node.ELEMENT_NODE:
                 if e.getAttributeNS(XbrlConst.xsi,"nil") == "true":
                     return []
                 modelXbrl = x.ownerDocument.modelDocument.modelXbrl
                 modelConcept = modelXbrl.qnameConcepts.get(qname(x))
                 if modelConcept:
                     baseXsdType = modelConcept.baseXsdType
                 v = XmlUtil.text(x)
             elif x.nodeType == xml.dom.Node.ATTRIBUTE_NODE:
                 e = x.ownerElement
                 modelXbrl = e.ownerDocument.modelDocument.modelXbrl
                 if x.namespaceURI:
                     attrQname = qname(x.namespaceURI, x.localName)
                 else:
                     attrQname = qname(x.localName)
                 modelConcept = modelXbrl.qnameConcepts.get(qname(e))
                 if modelConcept:
                     baseXsdType = modelConcept.baseXsdAttrType(attrQname) if modelConcept else None
                 if baseXsdType is None:
                     attrObject = modelXbrl.qnameAttributes.get(attrQname)
                     if attrObject:
                         baseXsdType = attrObject.baseXsdType
                 v = x.value
     if baseXsdType in ("decimal", "float", "double"):
         try:
             x = float(v)
         except ValueError:
             raise XPathException(p, 'err:FORG0001', _('Atomizing {0} to a {1} does not have a proper value').format(x,baseXsdType))
     elif baseXsdType in ("integer",):
         try:
             x = int(v)
         except ValueError:
             raise XPathException(p, 'err:FORG0001', _('Atomizing {0} to an integer does not have a proper value').format(x))
     elif baseXsdType == "boolean":
         x = (v == "true" or v == "1")
     elif baseXsdType == "QName" and e:
         x = qname(e, v)
     elif baseXsdType == "anyURI":
         x = anyURI(v.strip())
     elif baseXsdType in ("normalizedString","token","language","NMTOKEN","Name","NCName","ID","IDREF","ENTITY"):
         x = v.strip()
     elif baseXsdType == "XBRLI_DATEUNION":
         x = dateTime(v, type=DATEUNION)
     elif baseXsdType == "date":
         x = dateTime(v, type=DATE)
     elif baseXsdType == "dateTime":
         x = dateTime(v, type=DATETIME)
     elif baseXsdType:
         x = str(v)
     return x