Ejemplo n.º 1
0
def _values_unequal(val1, val2, dec_scale, margin_scale=2):
    """
    Checks the values for equality based on their scaling.
    Returns False if the values are equal, otherwise True.

    :param val1: first value to round
    :type val1: :class:'~arelle.ModelXbrl.ModelXbrl'
    :param val2: second value to round
    :type val2: :class:'~arelle.ModelXbrl.ModelXbrl'
    :param dec_scale: precision on rounded value
    :type dec_scale: :class:'~decimal.Decimal'
    :param margin_scale: margin of scale for the margin of error
    :type margin_scale: float
    :return: True if the values are not equal
    :rtype: bool
    """

    if not _min_dec_valid(dec_scale):
        return False

    round_val1 = roundValue(val1, decimals=dec_scale)
    round_val2 = roundValue(val2, decimals=dec_scale)
    margin_of_error = (Decimal(margin_scale) *
                       (Decimal(10)**Decimal(-dec_scale)))
    return (round_val1 < round_val2 - margin_of_error
            or round_val1 > round_val2 + margin_of_error)
Ejemplo n.º 2
0
def _values_unequal(val1, val2, dec_scale, margin_scale=2):
    """
    Checks the values for equality based on their scaling.
    Returns False if the values are equal, otherwise True.

    :param val1: first value to round
    :type val1: :class:'~arelle.ModelXbrl.ModelXbrl'
    :param val2: second value to round
    :type val2: :class:'~arelle.ModelXbrl.ModelXbrl'
    :param dec_scale: precision on rounded value
    :type dec_scale: :class:'~decimal.Decimal'
    :param margin_scale: margin of scale for the margin of error
    :type margin_scale: float
    :return: True if the values are not equal
    :rtype: bool
    """

    if not _min_dec_valid(dec_scale):
        return False

    round_val1 = roundValue(val1, decimals=dec_scale)
    round_val2 = roundValue(val2, decimals=dec_scale)
    margin_of_error = (
        Decimal(margin_scale) * (Decimal(10) ** Decimal(-dec_scale))
    )
    return (
        round_val1 < round_val2 - margin_of_error or
        round_val1 > round_val2 + margin_of_error
    )
Ejemplo n.º 3
0
def _assets_eq_liability_equity(modelXbrl):
    assets_concept = modelXbrl.nameConcepts[_ASSETS_CONCEPT][
        0] if modelXbrl.nameConcepts[_ASSETS_CONCEPT] else None
    liability_equity_concept = modelXbrl.nameConcepts[_LIABILITIES_CONCEPT][
        0] if modelXbrl.nameConcepts[_LIABILITIES_CONCEPT] else None

    if assets_concept is not None and liability_equity_concept is not None:
        assets_facts = modelXbrl.factsByQname[assets_concept.qname]
        liability_equity_facts = modelXbrl.factsByQname[
            liability_equity_concept.qname]

        fact_dict = {}
        fact_dict[_ASSETS_CONCEPT] = assets_facts
        fact_dict[_LIABILITIES_CONCEPT] = liability_equity_facts
        fact_groups = facts.prepare_facts_for_calculation(fact_dict)

        for fact_group in fact_groups:
            fact_assets = fact_group[_ASSETS_CONCEPT]
            fact_liabilities = fact_group[_LIABILITIES_CONCEPT]
            if fact_assets.context is not None and fact_assets.context.instantDatetime is not None:
                dec_assets = inferredDecimals(fact_assets)
                dec_liabilities = inferredDecimals(fact_liabilities)
                min_dec = min(dec_assets, dec_liabilities)
                if roundValue(fact_assets.xValue,
                              decimals=min_dec) != roundValue(
                                  fact_liabilities.xValue, decimals=min_dec):
                    yield fact_assets, fact_liabilities
Ejemplo n.º 4
0
 def isVEqualTo(self, other, deemP0Equal=False):
     """(bool) -- v-equality of two facts
     
     Note that facts may be in different instances
     """
     if self.isTuple or other.isTuple:
         return False
     if self.isNil:
         return other.isNil
     if other.isNil:
         return False
     if not self.context.isEqualTo(other.context):
         return False
     if self.concept.isNumeric:
         if other.concept.isNumeric:
             if not self.unit.isEqualTo(other.unit):
                 return False
             if self.modelXbrl.modelManager.validateInferDecimals:
                 d = min((inferredDecimals(self), inferredDecimals(other))); p = None
                 if isnan(d) and deemP0Equal:
                     return True
             else:
                 d = None; p = min((inferredPrecision(self), inferredPrecision(other)))
                 if p == 0 and deemP0Equal:
                     return True
             return roundValue(self.value,precision=p,decimals=d) == roundValue(other.value,precision=p,decimals=d)
         else:
             return False
     selfValue = self.value
     otherValue = other.value
     if isinstance(selfValue,str) and isinstance(otherValue,str):
         return selfValue.strip() == otherValue.strip()
     else:
         return selfValue == otherValue
Ejemplo n.º 5
0
 def isVEqualTo(self, other):
     if self.isTuple or other.isTuple:
         return False
     if self.isNil:
         return other.isNil
     if other.isNil:
         return False
     if not self.context.isEqualTo(other.context):
         return False
     if self.concept.isNumeric:
         if other.concept.isNumeric:
             if not self.unit.isEqualTo(other.unit):
                 return False
             if self.modelXbrl.modelManager.validateInferDecimals:
                 d = min((inferredDecimals(self), inferredDecimals(other))); p = None
             else:
                 d = None; p = min((inferredPrecision(self), inferredPrecision(other)))
             return roundValue(float(self.value),precision=p,decimals=d) == roundValue(float(other.value),precision=p,decimals=d)
         else:
             return False
     selfValue = self.value
     otherValue = other.value
     if isinstance(selfValue,str) and isinstance(otherValue,str):
         return selfValue.strip() == otherValue.strip()
     else:
         return selfValue == otherValue
Ejemplo n.º 6
0
def _values_unequal(val1, val2, dec_scale, margin_scale=2):
    """
    Checks the values for equality based on their scaling.
    Returns False if the values are equal, otherwise True.
    """
    round_val1 = roundValue(val1, decimals=dec_scale)
    round_val2 = roundValue(val2, decimals=dec_scale)
    margin_of_error = Decimal(margin_scale) * (Decimal(10) ** Decimal(-dec_scale))
    return round_val1 < round_val2 - margin_of_error or round_val1 > round_val2 + margin_of_error
Ejemplo n.º 7
0
def parsed_value(fact: ModelFact):
    if fact is None:
        return None
    concept = fact.concept  # type: ModelConcept
    if concept is None or concept.isTuple or fact.isNil:
        return None
    if concept.isFraction:
        num, den = map(fractions.Fraction, fact.fractionValue)
        return num / den
    val = fact.value.strip()
    if concept.isInteger:
        return int(val)
    elif concept.isNumeric:
        dec = fact.decimals
        if dec is None or dec == "INF":  # show using decimals or reported format
            dec = len(val.partition(".")[2])
        else:  # max decimals at 28
            dec = max(
                min(int(dec), 28), -28
            )  # 2.7 wants short int, 3.2 takes regular int, don't use _INT here
        num = roundValue(val, fact.precision,
                         dec)  # round using reported decimals
        return num
    elif concept.baseXbrliType == 'dateItemType':
        return dateTime(val)
    elif concept.baseXbrliType == 'booleanItemType':
        return val.lower() in ('1', 'true')
    elif concept.isTextBlock:
        return ' '.join(val.split())
    return val
Ejemplo n.º 8
0
 def insertFactSet(modelFacts, tupleFactId):
     table = self.getTable('fact', 'fact_id', 
                           ('accession_id', 'tuple_fact_id', 'context_id', 'unit_id', 'element_id', 'effective_value', 'fact_value', 
                            'xml_id', 'precision_value', 'decimals_value', 
                            'is_precision_infinity', 'is_decimals_infinity', ), 
                           ('accession_id', 'xml_id'), 
                           tuple((accsId,
                                  tupleFactId,
                                  self.cntxId.get((accsId,fact.contextID)),
                                  self.unitId.get((accsId,fact.unitID)),
                                  self.conceptElementId(fact.concept),
                                  roundValue(fact.value, fact.precision, fact.decimals) if fact.isNumeric and not fact.isNil else None,
                                  fact.value,
                                  elementFragmentIdentifier(fact),
                                  fact.xAttributes['precision'].xValue if ('precision' in fact.xAttributes and isinstance(fact.xAttributes['precision'].xValue,int)) else None,
                                  fact.xAttributes['decimals'].xValue if ('decimals' in fact.xAttributes and isinstance(fact.xAttributes['decimals'].xValue,int)) else None,
                                  'precision' in fact.xAttributes and fact.xAttributes['precision'].xValue == 'INF',
                                  'decimals' in fact.xAttributes and fact.xAttributes['decimals'].xValue == 'INF',
                                  )
                                 for fact in modelFacts))
     factId = dict((xmlId, id)
                   for id, _accsId, xmlId in table)
     for fact in modelFacts:
         if fact.isTuple:
             insertFactSet(fact.modelTupleFacts, 
                           factId[elementFragmentIdentifier(fact)])
Ejemplo n.º 9
0
def _assets_eq_liability_equity(modelXbrl):
    assets_concept = modelXbrl.nameConcepts[_ASSETS_CONCEPT][0] if modelXbrl.nameConcepts[_ASSETS_CONCEPT] else None
    liability_equity_concept = modelXbrl.nameConcepts[_LIABILITIES_CONCEPT][0] if modelXbrl.nameConcepts[_LIABILITIES_CONCEPT] else None

    if assets_concept is not None and liability_equity_concept is not None:
        assets_facts = modelXbrl.factsByQname[assets_concept.qname]
        liability_equity_facts = modelXbrl.factsByQname[liability_equity_concept.qname]

        fact_dict = {}
        fact_dict[_ASSETS_CONCEPT] = assets_facts
        fact_dict[_LIABILITIES_CONCEPT] = liability_equity_facts
        fact_groups = facts.prepare_facts_for_calculation(fact_dict)

        for fact_group in fact_groups:
            fact_assets = fact_group[_ASSETS_CONCEPT]
            fact_liabilities = fact_group[_LIABILITIES_CONCEPT]
            if fact_assets.context is not None and fact_assets.context.instantDatetime is not None:
                dec_assets = inferredDecimals(fact_assets)
                dec_liabilities = inferredDecimals(fact_liabilities)
                min_dec = min(dec_assets, dec_liabilities)
                if roundValue(fact_assets.xValue, decimals=min_dec) != roundValue(fact_liabilities.xValue, decimals=min_dec):
                    yield fact_assets, fact_liabilities
Ejemplo n.º 10
0
 def isVEqualTo(self,
                other,
                deemP0Equal=False):  # facts may be in different instances
     if self.isTuple or other.isTuple:
         return False
     if self.isNil:
         return other.isNil
     if other.isNil:
         return False
     if not self.context.isEqualTo(other.context):
         return False
     if self.concept.isNumeric:
         if other.concept.isNumeric:
             if not self.unit.isEqualTo(other.unit):
                 return False
             if self.modelXbrl.modelManager.validateInferDecimals:
                 d = min((inferredDecimals(self), inferredDecimals(other)))
                 p = None
             else:
                 d = None
                 p = min(
                     (inferredPrecision(self), inferredPrecision(other)))
             if p == 0 and deemP0Equal:
                 return True
             return roundValue(self.value, precision=p,
                               decimals=d) == roundValue(other.value,
                                                         precision=p,
                                                         decimals=d)
         else:
             return False
     selfValue = self.value
     otherValue = other.value
     if isinstance(selfValue, str) and isinstance(otherValue, str):
         return selfValue.strip() == otherValue.strip()
     else:
         return selfValue == otherValue
Ejemplo n.º 11
0
 def insertFactSet(modelFacts, parentDatapointId):
     table = self.getTable('data_point', 'datapoint_id', 
                           ('report_id', 'document_id', 'xml_id', 'source_line', 
                            'parent_datapoint_id',  # tuple
                            'aspect_id',
                            'context_xml_id', 'entity_id', 'period_id', 'aspect_value_selections_id', 'unit_id',
                            'is_nil', 'precision_value', 'decimals_value', 'effective_value', 'value'), 
                           ('document_id', 'xml_id'), 
                           tuple((reportId,
                                  documentId,
                                  elementFragmentIdentifier(fact),
                                  fact.sourceline,
                                  parentDatapointId, # parent ID
                                  self.aspectQnameId.get(fact.qname),
                                  fact.contextID,
                                  self.entityId.get((reportId, cntx.entityIdentifier[0], cntx.entityIdentifier[1]))
                                      if cntx is not None else None,
                                  self.periodId.get((reportId,
                                                     cntx.startDatetime if cntx.isStartEndPeriod else None,
                                                     cntx.endDatetime if cntx.isStartEndPeriod else None,
                                                     cntx.isInstantPeriod,
                                                     cntx.isForeverPeriod)) if cntx is not None else None,
                                  cntxAspectValueSelectionSetId.get(cntx) if cntx is not None else None,
                                  self.unitId.get((reportId,fact.unit.md5hash)) if fact.unit is not None else None,
                                  fact.isNil,
                                  fact.precision,
                                  fact.decimals,
                                  roundValue(fact.value, fact.precision, fact.decimals) if fact.isNumeric and not fact.isNil else None,
                                  fact.value
                                  )
                                 for fact in modelFacts
                                 for cntx in (fact.context,)
                                 for documentId in (self.documentIds[fact.modelDocument],)))
     xmlIdDataPointId = dict(((docId, xmlId), datapointId)
                             for datapointId, docId, xmlId in table)
     self.factDataPointId.update(xmlIdDataPointId)
     for fact in modelFacts:
         if fact.isTuple:
             insertFactSet(fact.modelTupleFacts, 
                           xmlIdDataPointId[(self.documentIds[fact.modelDocument],
                                             elementFragmentIdentifier(fact))])
Ejemplo n.º 12
0
def _roundPrecision(node, sphinxContext, args):
    args = numericArgs(node, sphinxContext, args, 2)
    return roundValue(args[0], precision=args[1])
Ejemplo n.º 13
0
def _roundDecimals(node, sphinxContext, args):
    args = numericArgs(node, sphinxContext, args, 2)
    return roundValue(args[0], decmials=args[1])
Ejemplo n.º 14
0
def _roundItem(node, sphinxContext, args):
    fact = factArg(node, sphinxContext, args, 0)
    return roundValue(fact.xValue, decimals=fact.decimals, precision=fact.precision)
Ejemplo n.º 15
0
def loadEntityInformation(dts, rssItem):
    entityInformation = {}
    # identify tables
    disclosureSystem = dts.modelManager.disclosureSystem
    if disclosureSystem.EFM:
        reloadCache = False
        if rssItem is not None:
            accession = rssItem.url.split('/')[-2]
            fileUrl = os.path.dirname(rssItem.url) + '/' + accession[0:10] + '-' + accession[10:12] + '-' + accession[12:] + ".hdr.sgml"
            reloadCache = getattr(rssItem.modelXbrl, "reloadCache", False)
        elif dts.uri.startswith("http://www.sec.gov/Archives/edgar/data") and dts.uri.endswith(".xml"):
            accession = dts.uri.split('/')[-2]
            dirPart = os.path.dirname(dts.uri)
            if accession.endswith("-xbrl.zip"):  # might be an instance document inside a xbrl.zip file
                accession = dts.uri.split('/')[-3]
                dirPart = os.path.dirname(dirPart)
            fileUrl = dirPart + '/' + accession[0:10] + '-' + accession[10:12] + '-' + accession[12:] + ".hdr.sgml"
        else:
            fileUrl = ''
        if fileUrl:
            # try to load and use it
            normalizedUrl = dts.modelManager.cntlr.webCache.normalizeUrl(fileUrl)
            hdrSgml = ''
            try:
                filePath = dts.modelManager.cntlr.webCache.getfilename(normalizedUrl, reload=reloadCache)
                if filePath:
                    with open(filePath) as fh:
                        hdrSgml = fh.read()
                    
            except  (IOError, EnvironmentError) as err:
                dts.info("xpDB:headerSgmlDocumentLoadingError",
                                    _("Loading XBRL DB: header SGML document %(file)s loading error: %(error)s"),
                                    modelObject=dts, file=normalizedUrl, error=err)
                hdrSgml = ''
            record = ''
            formerCompanyNumber = 0
            for match in re.finditer(r"[<]([^>]+)[>]([^<\n\r]*)", hdrSgml, re.MULTILINE):
                tag = match.group(1).lower()
                v = match.group(2).replace("&lt;","<").replace("&gt;",">").replace("&amp;","&")
                if tag in ('business-address','mail-address'):
                    record = tag + '.'
                elif tag == 'former-company':
                    formerCompanyNumber += 1
                    record = "{}-{}.".format(tag, formerCompanyNumber)
                elif tag.startswith('/'):
                    record = ''
                elif v:
                    if tag.endswith("-datetime"):
                        try:
                            v = datetime.datetime(_INT(v[0:4]),_INT(v[4:6]),_INT(v[6:8]),_INT(v[8:10]),_INT(v[10:12]),_INT(v[12:14]))
                        except ValueError:
                            pass
                    elif tag.endswith("-date") or tag.startswith("date-of-"):
                        try:
                            v = datetime.date(_INT(v[0:4]),_INT(v[4:6]),_INT(v[6:8]))
                        except ValueError:
                            pass
                    elif tag.endswith("-year-end") and len(v) == 4:
                        v = "{0}-{1}".format(v[0:2],v[2:4])
                    elif tag in ('assigned-sic',):
                        try:
                            v = _INT(v)
                        except ValueError:
                            v = None

                    entityInformation[record + tag] = v
            # primary document if no rssItem
            if rssItem is None:
                # try to sgml txt file
                normalizedUrl = normalizedUrl.replace(".hdr.sgml", ".txt")
                httpDir = normalizedUrl.rpartition('/')[0]
                txtSgml = ''
                try:
                    filePath = dts.modelManager.cntlr.webCache.getfilename(normalizedUrl, reload=reloadCache)
                    if filePath:
                        with open(filePath, encoding='utf-8') as fh:
                            txtSgml = fh.read()
                        # remove from cache, very large file
                        os.remove(filePath)
                except  (IOError, EnvironmentError) as err:
                    dts.info("xpDB:txtSgmlDocumentLoadingError",
                                        _("Loading XBRL DB: txt SGML document %(file)s loading error: %(error)s"),
                                        modelObject=dts, file=normalizedUrl, error=err)
                    txtSgml = ''
                documentType = documentSequence = None
                itemsFound = 0
                for match in re.finditer(r"[<]([^>]+)[>]([^<\n\r]*)", txtSgml, re.MULTILINE):
                    tag = match.group(1).lower()
                    v = match.group(2).replace("&lt;","<").replace("&gt;",">").replace("&amp;","&")
                    if tag == 'sequence':
                        documentSequence = v
                    elif tag == 'type':
                        documentType = v
                    elif tag == 'filename':
                        if documentType.endswith('.INS') and 'instance-url' not in entityInformation:
                            entityInformation['instance-url'] = httpDir + '/' + v
                            documentType = documentSequence = None
                            itemsFound += 1
                        if documentSequence == '1':
                            entityInformation['primary-document-url'] = httpDir + '/' + v
                            documentType = documentSequence = None
                            itemsFound += 1
                        if itemsFound >= 2:
                            break
                del txtSgml # dereference big string
        # instance information
        for factName, entityField in (("EntityFilerCategory", "filer-category"), 
                                      ("EntityPublicFloat", "public-float"), 
                                      ("TradingSymbol", "trading-symbol"), 
                                      ("DocumentFisalYearFocus", "fiscal-year-focus"),
                                      ("DocumentFisalPeriodFocus", "fiscal-period-focus")):
            try:
                concept = dts.nameConcepts[factName][0] # get qname irrespective of taxonomy year
                facts = dts.factsByQname[concept.qname]
                for fact in facts:
                    if not fact.context.qnameDims: #default context
                        if factName in ("EntityPublicFloat",):
                            entityInformation[entityField] = roundValue(fact.value, fact.precision, fact.decimals) if fact.isNumeric and not fact.isNil else None
                        else:
                            entityInformation[entityField] = fact.value.strip() # may have white space
                        break
            except IndexError:
                pass
    return entityInformation
Ejemplo n.º 16
0
    def loadXbrlFromDB(self, loadDBsaveToFile):
        # load from database
        modelXbrl = self.modelXbrl
        
        # find instance in DB
        instanceURI = os.path.basename(loadDBsaveToFile)
        results = self.execute("SELECT InstanceID, ModuleID, EntityScheme, EntityIdentifier, PeriodEndDateOrInstant"
                               " FROM dInstance WHERE FileName = '{}'"
                               .format(instanceURI))
        instanceId = moduleId = None
        for instanceId, moduleId, entScheme, entId, datePerEnd in results:
            break

        # find module in DB        
        results = self.execute("SELECT XbrlSchemaRef FROM mModule WHERE ModuleID = {}".format(moduleId))
        xbrlSchemaRef = None
        for result in results:
            xbrlSchemaRef = result[0]
            break
        
        if not instanceId or not xbrlSchemaRef:
            raise XPDBException("sqlDB:MissingDTS",
                    _("The instance and module were not found for %(instanceURI)"),
                    instanceURI = instanceURI) 
            
        if modelXbrl.skipDTS:
            # find prefixes and namespaces in DB
            results = self.execute("SELECT * FROM [vwGetNamespacesPrefixes]")            
            dpmPrefixedNamespaces = dict((prefix, namespace)
                                         for owner, prefix, namespace in results)
            
        # create the instance document and resulting filing
        modelXbrl.blockDpmDBrecursion = True
        modelXbrl.modelDocument = createModelDocument(modelXbrl, 
                                                      Type.INSTANCE,
                                                      loadDBsaveToFile,
                                                      schemaRefs=[xbrlSchemaRef],
                                                      isEntry=True)
        ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults 
        
        addProcessingInstruction(modelXbrl.modelDocument.xmlRootElement, 
                                 'xbrl-streamable-instance', 
                                 'version="1.0" contextBuffer="1"')

        # add roleRef and arcroleRef (e.g. for footnotes, if any, see inlineXbrlDocue)
        
        # filing indicator code IDs
        # get filing indicators
        results = self.execute("SELECT mToT.TemplateOrTableCode "
                               "  FROM dFilingIndicator dFI, mTemplateOrTable mToT "
                               "  WHERE dFI.InstanceID = {} AND mTot.TemplateOrTableID = dFI.BusinessTemplateID"
                               .format(instanceId))
        filingIndicatorCodes = [code[0] for code in results]
        
        if filingIndicatorCodes:
            modelXbrl.createContext(entScheme,
                        entId,
                        'instant',
                        None,
                        datePerEnd,
                        None, # no dimensional validity checking (like formula does)
                        {}, [], [],
                        id='c')
            filingIndicatorsTuple = modelXbrl.createFact(qnFindFilingIndicators)
            for filingIndicatorCode in filingIndicatorCodes:
                modelXbrl.createFact(qnFindFilingIndicator, 
                                     parent=filingIndicatorsTuple,
                                     attributes={"contextRef": "c"}, 
                                     text=filingIndicatorCode)

        
        # facts in this instance
        factsTbl = self.execute("SELECT DataPointSignature, DataPointSignatureWithValuesForWildcards,"
                                " Unit, Decimals, NumericValue, DateTimeValue, BooleanValue, TextValue "
                                "FROM dFact WHERE InstanceID = {} "
                                "ORDER BY substr(CASE WHEN DataPointSignatureWithValuesForWildcards IS NULL "
                                "                          THEN DataPointSignature"
                                "                          ELSE DataPointSignatureWithValuesForWildcards"
                                "                END, instr(DataPointSignature,'|') + 1)"
                                .format(instanceId))
        
        # results tuple: factId, dec, varId, dpKey, entId, datePerEnd, unit, numVal, dateVal, boolVal, textVal

        # get typed dimension values
        prefixedNamespaces = modelXbrl.prefixedNamespaces
        prefixedNamespaces["iso4217"] = XbrlConst.iso4217
        if modelXbrl.skipDTS:
            prefixedNamespaces.update(dpmPrefixedNamespaces) # for skipDTS this is always needed
        
        cntxTbl = {} # index by d
        unitTbl = {}
        
        def typedDimElt(s):
            # add xmlns into s for known qnames
            tag, angleBrkt, rest = s[1:].partition('>')
            text, angleBrkt, rest = rest.partition("<")
            qn = qname(tag, prefixedNamespaces)
            # a modelObject xml element is needed for all of the instance functions to manage the typed dim
            return addChild(modelXbrl.modelDocument, qn, text=text, appendChild=False)
        
        # contexts and facts
        for dpSig, dpSigTypedDims, unit, dec, numVal, dateVal, boolVal, textVal in factsTbl:
            metric, sep, dims = (dpSigTypedDims or dpSig).partition('|')
            conceptQn = qname(metric.partition('(')[2][:-1], prefixedNamespaces)
            concept = modelXbrl.qnameConcepts.get(conceptQn)
            isNumeric = isBool = isDateTime = isQName = isText = False
            if concept is not None:
                if concept.isNumeric:
                    isNumeric = True
                else:
                    baseXbrliType = concept.baseXbrliType
                    if baseXbrliType == "booleanItemType":
                        isBool = True
                    elif baseXbrliType == "dateTimeItemType": # also is dateItemType?
                        isDateTime = True
                    elif baseXbrliType == "QNameItemType":
                        isQName = True
            else:
                c = conceptQn.localName[0]
                if c == 'm':
                    isNumeric = True
                elif c == 'd':
                    isDateTime = True
                elif c == 'b':
                    isBool = True
                elif c == 'e':
                    isQName = True
            isText = not (isNumeric or isBool or isDateTime or isQName)
            if isinstance(datePerEnd, _STR_BASE):
                datePerEnd = datetimeValue(datePerEnd, addOneDay=True)
            cntxKey = (dims, entId, datePerEnd)
            if cntxKey in cntxTbl:
                cntxId = cntxTbl[cntxKey]
            else:
                cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
                cntxTbl[cntxKey] = cntxId
                qnameDims = {}
                if dims:
                    for dim in dims.split('|'):
                        dQn, sep, dVal = dim[:-1].partition('(')
                        dimQname = qname(dQn, prefixedNamespaces)
                        if dVal.startswith('<'):
                            mem = typedDimElt(dVal)  # typed dim
                        else:
                            mem = qname(dVal, prefixedNamespaces) # explicit dim (even if treat-as-typed)
                        qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "scenario")
                    
                modelXbrl.createContext(entScheme,
                                        entId,
                                        'instant',
                                        None,
                                        datePerEnd,
                                        None, # no dimensional validity checking (like formula does)
                                        qnameDims, [], [],
                                        id=cntxId)
            if unit:
                if unit in unitTbl:
                    unitId = unitTbl[unit]
                else:
                    unitQn = qname(unit, prefixedNamespaces)
                    unitId = 'u{}'.format(unitQn.localName)
                    unitTbl[unit] = unitId
                    modelXbrl.createUnit([unitQn], [], id=unitId)
            else:
                unitId = None
            attrs = {"contextRef": cntxId}
            if unitId:
                attrs["unitRef"] = unitId
            if dec is not None:
                if isinstance(dec, float): # must be an integer
                    dec = int(dec)
                elif isinstance(dec, _STR_BASE) and '.' in dec:
                    dec = dec.partition('.')[0] # drop .0 from any SQLite string
                attrs["decimals"] = str(dec)  # somehow it is float from the database
            if False: # fact.isNil:
                attrs[XbrlConst.qnXsiNil] = "true"
                text = None
            elif numVal is not None:
                num = roundValue(numVal, None, dec) # round using reported decimals
                if dec is None or dec == "INF":  # show using decimals or reported format
                    dec = len(numVal.partition(".")[2])
                else: # max decimals at 28
                    dec = max( min(int(float(dec)), 28), -28) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here
                text = Locale.format(self.modelXbrl.locale, "%.*f", (dec, num))
            elif dateVal is not None:
                text = dateVal
            elif boolVal is not None:
                text = 'true' if boolVal.lower() in ('t', 'true', '1') else 'false'
            else:
                if isQName: # declare namespace
                    addQnameValue(modelXbrl.modelDocument, qname(textVal, prefixedNamespaces))
                text = textVal
            modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
            
        # add footnotes if any
        
        # save to file
        modelXbrl.saveInstance(overrideFilepath=loadDBsaveToFile)
        modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
        return modelXbrl.modelDocument
Ejemplo n.º 17
0
def evaluate(xpCtx, varSet, derivedFact):
    # there may be multiple consis assertions parenting any formula
    for consisAsserRel in xpCtx.modelXbrl.relationshipSet(
            XbrlConst.consistencyAssertionFormula).toModelObject(varSet):
        consisAsser = consisAsserRel.fromModelObject
        hasProportionalAcceptanceRadius = consisAsser.hasProportionalAcceptanceRadius
        hasAbsoluteAcceptanceRadius = consisAsser.hasAbsoluteAcceptanceRadius
        if derivedFact is None:
            continue
        isNumeric = derivedFact.isNumeric
        if isNumeric and not derivedFact.isNil:
            derivedFactInferredPrecision = inferredPrecision(derivedFact)
            if derivedFactInferredPrecision == 0 and not hasProportionalAcceptanceRadius and not hasAbsoluteAcceptanceRadius:
                if xpCtx.formulaOptions.traceVariableSetExpressionResult:
                    xpCtx.modelXbrl.error(
                        _("Consistency assertion {0} formula {1} fact {2} has zero precision and no radius is defined, skipping consistency assertion"
                          ).format(consisAsser.id, varSet.xlinkLabel,
                                   derivedFact), "info", "formula:trace")
                continue

        # check xbrl validity of new fact

        # find source facts which match derived fact
        aspectMatchedInputFacts = []
        isStrict = consisAsser.isStrict
        for inputFact in xpCtx.modelXbrl.facts:
            if (not inputFact.isNil and inputFact.qname == derivedFact.qname
                    and inputFact.context.isEqualTo(
                        derivedFact.context,
                        dimensionalAspectModel=(varSet.aspectModel
                                                == "dimensional")) and
                (not isNumeric or inputFact.unit.isEqualTo(derivedFact.unit))):
                aspectMatchedInputFacts.append(inputFact)

        if len(aspectMatchedInputFacts) == 0:
            if isStrict:
                if derivedFact.isNil:
                    isSatisfied = True
                else:
                    isSatisfied = False
            else:
                if xpCtx.formulaOptions.traceVariableSetExpressionResult:
                    xpCtx.modelXbrl.error(
                        _("Consistency assertion {0} Formula {1} no input facts matched to {2}, skipping consistency assertion"
                          ).format(consisAsser.id, varSet.xlinkLabel,
                                   derivedFact), "info", "formula:trace")
                continue
        elif derivedFact.isNil:
            isSatisfied = False
        else:
            isSatisfied = True

        paramQnamesAdded = []
        for paramRel in consisAsser.orderedVariableRelationships:
            paramQname = paramRel.variableQname
            paramVar = paramRel.toModelObject
            paramValue = xpCtx.inScopeVars.get(paramVar.qname)
            paramAlreadyInVars = paramQname in xpCtx.inScopeVars
            if not paramAlreadyInVars:
                paramQnamesAdded.append(paramQname)
                xpCtx.inScopeVars[paramQname] = paramValue
        for fact in aspectMatchedInputFacts:
            if isSatisfied != True:
                break
            if fact.isNil:
                if not derivedFact.isNil:
                    isSatisfied = False
            elif isNumeric:
                factInferredPrecision = inferredPrecision(fact)
                if factInferredPrecision == 0 and not hasProportionalAcceptanceRadius and not hasAbsoluteAcceptanceRadius:
                    if xpCtx.formulaOptions.traceVariableSetExpressionResult:
                        xpCtx.modelXbrl.error(
                            _("Consistency assertion {0} Formula {1} input fact matched to {2} has zero precision and no radius, skipping consistency assertion"
                              ).format(consisAsser.id, varSet.xlinkLabel,
                                       derivedFact), "info", "formula:trace")
                        isSatisfied = None
                        break
                if hasProportionalAcceptanceRadius or hasAbsoluteAcceptanceRadius:
                    acceptance = consisAsser.evalRadius(
                        xpCtx, derivedFact.vEqValue)
                    if acceptance is not None:
                        if hasProportionalAcceptanceRadius:
                            acceptance *= derivedFact.vEqValue
                        isSatisfied = fabs(derivedFact.vEqValue -
                                           fact.vEqValue) <= fabs(acceptance)
                    else:
                        isSatisfied = None  # no radius
                else:
                    p = min(derivedFactInferredPrecision,
                            factInferredPrecision)
                    if (p == 0 or roundValue(derivedFact.vEqValue, precision=p)
                            != roundValue(fact.vEqValue, precision=p)):
                        isSatisfied = False
            else:
                if not xEqual(fact.concept,
                              fact.element,
                              derivedFact.element,
                              equalMode=S_EQUAL2):
                    isSatisfied = False
        for paramQname in paramQnamesAdded:
            xpCtx.inScopeVars.pop(paramQname)
        if isSatisfied is None:
            continue  # no evaluation
        if xpCtx.formulaOptions.traceVariableSetExpressionResult:
            xpCtx.modelXbrl.error(
                _("Consistency Assertion {0} result {1}").format(
                    consisAsser.id, isSatisfied), "info", "formula:trace")
        message = consisAsser.message(isSatisfied)
        if message:
            xpCtx.modelXbrl.error(message.evaluate(xpCtx), "info",
                                  "message:" + consisAsser.id)
        if isSatisfied: consisAsser.countSatisfied += 1
        else: consisAsser.countNotSatisfied += 1
Ejemplo n.º 18
0
def evaluate(xpCtx, varSet, derivedFact):
    # there may be multiple consis assertions parenting any formula
    for consisAsserRel in xpCtx.modelXbrl.relationshipSet(XbrlConst.consistencyAssertionFormula).toModelObject(varSet):
        consisAsser = consisAsserRel.fromModelObject
        hasProportionalAcceptanceRadius = consisAsser.hasProportionalAcceptanceRadius
        hasAbsoluteAcceptanceRadius = consisAsser.hasAbsoluteAcceptanceRadius
        if derivedFact is None:
            continue
        isNumeric = derivedFact.isNumeric
        if isNumeric and not derivedFact.isNil:
            derivedFactInferredPrecision = inferredPrecision(derivedFact)
            if derivedFactInferredPrecision == 0 and not hasProportionalAcceptanceRadius and not hasAbsoluteAcceptanceRadius:
                if xpCtx.formulaOptions.traceVariableSetExpressionResult:
                    xpCtx.modelXbrl.error( _("Consistency assertion {0} formula {1} fact {2} has zero precision and no radius is defined, skipping consistency assertion").format(
                         consisAsser.id, varSet.xlinkLabel, derivedFact),
                        "info", "formula:trace")
                continue
    
        # check xbrl validity of new fact
        
        # find source facts which match derived fact
        aspectMatchedInputFacts = []
        isStrict = consisAsser.isStrict
        for inputFact in xpCtx.modelXbrl.facts:
            if (not inputFact.isNil and
                inputFact.qname == derivedFact.qname and
                inputFact.context.isEqualTo(derivedFact.context,
                                            dimensionalAspectModel=(varSet.aspectModel == "dimensional")) and
                (not isNumeric or inputFact.unit.isEqualTo(derivedFact.unit))):
                aspectMatchedInputFacts.append( inputFact )
        
        if len(aspectMatchedInputFacts) == 0:
            if isStrict:
                if derivedFact.isNil:
                    isSatisfied = True
                else:
                    isSatisfied = False
            else:
                if xpCtx.formulaOptions.traceVariableSetExpressionResult:
                    xpCtx.modelXbrl.error( _("Consistency assertion {0} Formula {1} no input facts matched to {2}, skipping consistency assertion").format( 
                        consisAsser.id, varSet.xlinkLabel, derivedFact),
                        "info", "formula:trace")
                continue
        elif derivedFact.isNil:
            isSatisfied = False
        else:
            isSatisfied = True
                
        paramQnamesAdded = []
        for paramRel in consisAsser.orderedVariableRelationships:
            paramQname = paramRel.variableQname
            paramVar = paramRel.toModelObject
            paramValue = xpCtx.inScopeVars.get(paramVar.qname)
            paramAlreadyInVars = paramQname in xpCtx.inScopeVars
            if not paramAlreadyInVars:
                paramQnamesAdded.append(paramQname)
                xpCtx.inScopeVars[paramQname] = paramValue
        for fact in aspectMatchedInputFacts:
            if isSatisfied != True: 
                break
            if fact.isNil:
                if not derivedFact.isNil:
                    isSatisfied = False
            elif isNumeric:
                factInferredPrecision = inferredPrecision(fact)
                if factInferredPrecision == 0 and not hasProportionalAcceptanceRadius and not hasAbsoluteAcceptanceRadius:
                    if xpCtx.formulaOptions.traceVariableSetExpressionResult:
                        xpCtx.modelXbrl.error( _("Consistency assertion {0} Formula {1} input fact matched to {2} has zero precision and no radius, skipping consistency assertion").format( 
                            consisAsser.id, varSet.xlinkLabel, derivedFact),
                            "info", "formula:trace")
                        isSatisfied = None
                        break
                if hasProportionalAcceptanceRadius or hasAbsoluteAcceptanceRadius:
                    acceptance = consisAsser.evalRadius(xpCtx, derivedFact.vEqValue)
                    if acceptance is not None:
                        if hasProportionalAcceptanceRadius:
                            acceptance *= derivedFact.vEqValue
                        isSatisfied = fabs(derivedFact.vEqValue - fact.vEqValue) <= fabs(acceptance)
                    else:
                        isSatisfied = None  # no radius
                else:
                    p = min(derivedFactInferredPrecision, factInferredPrecision)
                    if (p == 0 or
                        roundValue(derivedFact.vEqValue, precision=p) != roundValue(fact.vEqValue, precision=p)):
                        isSatisfied = False
            else:
                if not xEqual(fact.concept, fact.element, derivedFact.element, equalMode=S_EQUAL2):
                    isSatisfied = False
        for paramQname in paramQnamesAdded:
            xpCtx.inScopeVars.pop(paramQname)
        if isSatisfied is None:
            continue    # no evaluation
        if xpCtx.formulaOptions.traceVariableSetExpressionResult:
            xpCtx.modelXbrl.error( _("Consistency Assertion {0} result {1}").format( consisAsser.id, isSatisfied),
                "info", "formula:trace")
        message = consisAsser.message(isSatisfied)
        if message:
            xpCtx.modelXbrl.error(message.evaluate(xpCtx), "info", "message:" + consisAsser.id)
        if isSatisfied: consisAsser.countSatisfied += 1
        else: consisAsser.countNotSatisfied += 1
Ejemplo n.º 19
0
    def loadXbrlFromDB(self, loadDBsaveToFile):
        # load from database
        modelXbrl = self.modelXbrl
        
        # find instance in DB
        instanceURI = os.path.basename(loadDBsaveToFile)
        results = self.execute("SELECT InstanceID, ModuleID, EntityScheme, EntityIdentifier, PeriodEndDateOrInstant"
                               " FROM dInstance WHERE FileName = '{}'"
                               .format(instanceURI))
        instanceId = moduleId = None
        for instanceId, moduleId, entScheme, entId, datePerEnd in results:
            break

        # find module in DB        
        results = self.execute("SELECT XbrlSchemaRef FROM mModule WHERE ModuleID = {}".format(moduleId))
        xbrlSchemaRef = None
        for result in results:
            xbrlSchemaRef = result[0]
            break
        
        if not instanceId or not xbrlSchemaRef:
            raise XPDBException("sqlDB:MissingDTS",
                    _("The instance and module were not found for %(instanceURI)"),
                    instanceURI = instanceURI) 
            
        if modelXbrl.skipDTS:
            # find prefixes and namespaces in DB
            results = self.execute("SELECT * FROM [vwGetNamespacesPrefixes]")            
            dpmPrefixedNamespaces = dict((prefix, namespace)
                                         for owner, prefix, namespace in results)
            
        # create the instance document and resulting filing
        modelXbrl.blockDpmDBrecursion = True
        modelXbrl.modelDocument = createModelDocument(modelXbrl, 
                                                      Type.INSTANCE,
                                                      loadDBsaveToFile,
                                                      schemaRefs=[xbrlSchemaRef],
                                                      isEntry=True)
        ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults 
        
        addProcessingInstruction(modelXbrl.modelDocument.xmlRootElement, 
                                 'xbrl-streamable-instance', 
                                 'version="1.0" contextBuffer="1"')

        # add roleRef and arcroleRef (e.g. for footnotes, if any, see inlineXbrlDocue)
        
        # filing indicator code IDs
        # get filing indicators
        results = self.execute("SELECT mToT.TemplateOrTableCode "
                               "  FROM dFilingIndicator dFI, mTemplateOrTable mToT "
                               "  WHERE dFI.InstanceID = {} AND mTot.TemplateOrTableID = dFI.BusinessTemplateID"
                               .format(instanceId))
        filingIndicatorCodes = [code[0] for code in results]
        
        if filingIndicatorCodes:
            modelXbrl.createContext(entScheme,
                        entId,
                        'instant',
                        None,
                        datePerEnd,
                        None, # no dimensional validity checking (like formula does)
                        {}, [], [],
                        id='c')
            filingIndicatorsTuple = modelXbrl.createFact(qnFindFilingIndicators)
            for filingIndicatorCode in filingIndicatorCodes:
                modelXbrl.createFact(qnFindFilingIndicator, 
                                     parent=filingIndicatorsTuple,
                                     attributes={"contextRef": "c"}, 
                                     text=filingIndicatorCode)

        
        # facts in this instance
        factsTbl = self.execute("SELECT DataPointSignature, DataPointSignatureWithValuesForWildcards,"
                                " Unit, Decimals, NumericValue, DateTimeValue, BooleanValue, TextValue "
                                "FROM dFact WHERE InstanceID = {} "
                                "ORDER BY substr(CASE WHEN DataPointSignatureWithValuesForWildcards IS NULL "
                                "                          THEN DataPointSignature"
                                "                          ELSE DataPointSignatureWithValuesForWildcards"
                                "                END, instr(DataPointSignature,'|') + 1)"
                                .format(instanceId))
        
        # results tuple: factId, dec, varId, dpKey, entId, datePerEnd, unit, numVal, dateVal, boolVal, textVal

        # get typed dimension values
        prefixedNamespaces = modelXbrl.prefixedNamespaces
        prefixedNamespaces["iso4217"] = XbrlConst.iso4217
        if modelXbrl.skipDTS:
            prefixedNamespaces.update(dpmPrefixedNamespaces) # for skipDTS this is always needed
        
        cntxTbl = {} # index by d
        unitTbl = {}
        
        def typedDimElt(s):
            # add xmlns into s for known qnames
            tag, angleBrkt, rest = s[1:].partition('>')
            text, angleBrkt, rest = rest.partition("<")
            qn = qname(tag, prefixedNamespaces)
            # a modelObject xml element is needed for all of the instance functions to manage the typed dim
            return addChild(modelXbrl.modelDocument, qn, text=text, appendChild=False)
        
        # contexts and facts
        for dpSig, dpSigTypedDims, unit, dec, numVal, dateVal, boolVal, textVal in factsTbl:
            metric, sep, dims = (dpSigTypedDims or dpSig).partition('|')
            conceptQn = qname(metric.partition('(')[2][:-1], prefixedNamespaces)
            concept = modelXbrl.qnameConcepts.get(conceptQn)
            isNumeric = isBool = isDateTime = isQName = isText = False
            if concept is not None:
                if concept.isNumeric:
                    isNumeric = True
                else:
                    baseXbrliType = concept.baseXbrliType
                    if baseXbrliType == "booleanItemType":
                        isBool = True
                    elif baseXbrliType == "dateTimeItemType": # also is dateItemType?
                        isDateTime = True
                    elif baseXbrliType == "QNameItemType":
                        isQName = True
            else:
                c = conceptQn.localName[0]
                if c == 'm':
                    isNumeric = True
                elif c == 'd':
                    isDateTime = True
                elif c == 'b':
                    isBool = True
                elif c == 'e':
                    isQName = True
            isText = not (isNumeric or isBool or isDateTime or isQName)
            if isinstance(datePerEnd, _STR_BASE):
                datePerEnd = datetimeValue(datePerEnd, addOneDay=True)
            cntxKey = (dims, entId, datePerEnd)
            if cntxKey in cntxTbl:
                cntxId = cntxTbl[cntxKey]
            else:
                cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
                cntxTbl[cntxKey] = cntxId
                qnameDims = {}
                if dims:
                    for dim in dims.split('|'):
                        dQn, sep, dVal = dim[:-1].partition('(')
                        dimQname = qname(dQn, prefixedNamespaces)
                        if dVal.startswith('<'): # typed dim
                            mem = typedDimElt(dVal)
                        else:
                            mem = qname(dVal, prefixedNamespaces)
                        qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "scenario")
                    
                modelXbrl.createContext(entScheme,
                                        entId,
                                        'instant',
                                        None,
                                        datePerEnd,
                                        None, # no dimensional validity checking (like formula does)
                                        qnameDims, [], [],
                                        id=cntxId)
            if unit:
                if unit in unitTbl:
                    unitId = unitTbl[unit]
                else:
                    unitQn = qname(unit, prefixedNamespaces)
                    unitId = 'u{}'.format(unitQn.localName)
                    unitTbl[unit] = unitId
                    modelXbrl.createUnit([unitQn], [], id=unitId)
            else:
                unitId = None
            attrs = {"contextRef": cntxId}
            if unitId:
                attrs["unitRef"] = unitId
            if dec is not None:
                if isinstance(dec, float): # must be an integer
                    dec = int(dec)
                elif isinstance(dec, _STR_BASE) and '.' in dec:
                    dec = dec.partition('.')[0] # drop .0 from any SQLite string
                attrs["decimals"] = str(dec)  # somehow it is float from the database
            if False: # fact.isNil:
                attrs[XbrlConst.qnXsiNil] = "true"
                text = None
            elif numVal is not None:
                num = roundValue(numVal, None, dec) # round using reported decimals
                if dec is None or dec == "INF":  # show using decimals or reported format
                    dec = len(numVal.partition(".")[2])
                else: # max decimals at 28
                    dec = max( min(int(float(dec)), 28), -28) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here
                text = Locale.format(self.modelXbrl.locale, "%.*f", (dec, num))
            elif dateVal is not None:
                text = dateVal
            elif boolVal is not None:
                text = 'true' if boolVal.lower() in ('t', 'true', '1') else 'false'
            else:
                if isQName: # declare namespace
                    addQnameValue(modelXbrl.modelDocument, qname(textVal, prefixedNamespaces))
                text = textVal
            modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
            
        # add footnotes if any
        
        # save to file
        modelXbrl.saveInstance(overrideFilepath=loadDBsaveToFile)
        modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
        return modelXbrl.modelDocument
Ejemplo n.º 20
0
def loadEntityInformation(dts, entrypoint, rssItem):
    entityInformation = {}
    # identify tables
    disclosureSystem = dts.modelManager.disclosureSystem
    if disclosureSystem.validationType == "EFM":
        reloadCache = False
        if rssItem is not None:
            accession = rssItem.url.split('/')[-2]
            fileUrl = os.path.dirname(rssItem.url) + '/' + accession[0:10] + '-' + accession[10:12] + '-' + accession[12:] + ".hdr.sgml"
            reloadCache = getattr(rssItem.modelXbrl, "reloadCache", False)
        elif dts.uri.startswith("http://www.sec.gov/Archives/edgar/data") and dts.uri.endswith(".xml"):
            accession = dts.uri.split('/')[-2]
            dirPart = os.path.dirname(dts.uri)
            if accession.endswith("-xbrl.zip"):  # might be an instance document inside a xbrl.zip file
                accession = dts.uri.split('/')[-3]
                dirPart = os.path.dirname(dirPart)
            fileUrl = dirPart + '/' + accession[0:10] + '-' + accession[10:12] + '-' + accession[12:] + ".hdr.sgml"
        else:
            fileUrl = ''
        if fileUrl:
            # try to load and use it
            normalizedUrl = dts.modelManager.cntlr.webCache.normalizeUrl(fileUrl)
            hdrSgml = ''
            try:
                filePath = dts.modelManager.cntlr.webCache.getfilename(normalizedUrl, reload=reloadCache)
                if filePath:
                    with open(filePath) as fh:
                        hdrSgml = fh.read()
                    
            except  (IOError, EnvironmentError) as err:
                dts.info("xpDB:headerSgmlDocumentLoadingError",
                                    _("Loading XBRL DB: header SGML document %(file)s loading error: %(error)s"),
                                    modelObject=dts, file=normalizedUrl, error=err)
                hdrSgml = ''
            record = ''
            formerCompanyNumber = 0
            for match in re.finditer(r"[<]([^>]+)[>]([^<\n\r]*)", hdrSgml, re.MULTILINE):
                tag = match.group(1).lower()
                v = match.group(2).replace("&lt;","<").replace("&gt;",">").replace("&amp;","&")
                if tag in ('business-address','mail-address'):
                    record = tag + '.'
                elif tag == 'former-company':
                    formerCompanyNumber += 1
                    record = "{}-{}.".format(tag, formerCompanyNumber)
                elif tag.startswith('/'):
                    record = ''
                elif v:
                    if tag.endswith("-datetime"):
                        try:
                            v = datetime.datetime(_INT(v[0:4]),_INT(v[4:6]),_INT(v[6:8]),_INT(v[8:10]),_INT(v[10:12]),_INT(v[12:14]))
                        except ValueError:
                            pass
                    elif tag.endswith("-date") or tag.startswith("date-of-"):
                        try:
                            v = datetime.date(_INT(v[0:4]),_INT(v[4:6]),_INT(v[6:8]))
                        except ValueError:
                            pass
                    elif tag.endswith("-year-end") and len(v) == 4:
                        v = "{0}-{1}".format(v[0:2],v[2:4])
                    elif tag in ('assigned-sic',):
                        try:
                            v = _INT(v)
                        except ValueError:
                            v = None

                    entityInformation[record + tag] = v
            # primary document if no rssItem
            if rssItem is None:
                # try to sgml txt file
                normalizedUrl = normalizedUrl.replace(".hdr.sgml", ".txt")
                httpDir = normalizedUrl.rpartition('/')[0]
                txtSgml = ''
                try:
                    filePath = dts.modelManager.cntlr.webCache.getfilename(normalizedUrl, reload=reloadCache)
                    if filePath:
                        with open(filePath, encoding='utf-8') as fh:
                            txtSgml = fh.read()
                        # remove from cache, very large file
                        os.remove(filePath)
                except  (IOError, EnvironmentError) as err:
                    dts.info("xpDB:txtSgmlDocumentLoadingError",
                                        _("Loading XBRL DB: txt SGML document %(file)s loading error: %(error)s"),
                                        modelObject=dts, file=normalizedUrl, error=err)
                    txtSgml = ''
                documentType = documentSequence = None
                itemsFound = 0
                for match in re.finditer(r"[<]([^>]+)[>]([^<\n\r]*)", txtSgml, re.MULTILINE):
                    tag = match.group(1).lower()
                    v = match.group(2).replace("&lt;","<").replace("&gt;",">").replace("&amp;","&")
                    if tag == 'sequence':
                        documentSequence = v
                    elif tag == 'type':
                        documentType = v
                    elif tag == 'filename':
                        if documentType.endswith('.INS') and 'instance-url' not in entityInformation:
                            entityInformation['instance-url'] = httpDir + '/' + v
                            documentType = documentSequence = None
                            itemsFound += 1
                        if documentSequence == '1':
                            entityInformation['primary-document-url'] = httpDir + '/' + v
                            documentType = documentSequence = None
                            itemsFound += 1
                        if itemsFound >= 2:
                            break
                del txtSgml # dereference big string
        # instance information
        for factName, entityField in (("EntityFilerCategory", "filer-category"), 
                                      ("EntityPublicFloat", "public-float"), 
                                      ("TradingSymbol", "trading-symbol"), 
                                      ("DocumentFisalYearFocus", "fiscal-year-focus"),
                                      ("DocumentFisalPeriodFocus", "fiscal-period-focus"),
                                      ("DocumentType", "document-type")):
            try:
                concept = dts.nameConcepts[factName][0] # get qname irrespective of taxonomy year
                facts = dts.factsByQname[concept.qname]
                for fact in facts:
                    if not fact.context.qnameDims: #default context
                        if factName in ("EntityPublicFloat",):
                            entityInformation[entityField] = roundValue(fact.value, fact.precision, fact.decimals) if fact.isNumeric and not fact.isNil else None
                        else:
                            entityInformation[entityField] = fact.value.strip() # may have white space
                        break
            except IndexError:
                pass
    return entityInformation
Ejemplo n.º 21
0
 def insertFacts(self):
     accsId = self.accessionId
     self.showStatus("insert facts")
     # units
     table = self.getTable('unit', 'unit_id', 
                           ('accession_id', 'unit_xml_id'), 
                           ('accession_id', 'unit_xml_id'), 
                           tuple((accsId,
                                  unitId)
                                 for unitId in self.modelXbrl.units.keys()))
     self.unitId = dict(((_accsId, xmlId), id)
                        for id, _accsId, xmlId in table)
     # measures
     table = self.getTable('unit_measure', 'unit_measure_id', 
                           ('unit_id', 'qname_id', 'location_id'), 
                           ('qname_id', 'location_id'), 
                           tuple((self.unitId[(accsId,unit.id)],
                                  self.qnameId[measure],
                                  1 if (not unit.measures[1]) else (i + 1))
                                 for unit in self.modelXbrl.units.values()
                                 for i in range(2)
                                 for measure in unit.measures[i]))
     #table = self.getTable('enumeration_measure_location', 'enumeration_measure_location_id', 
     #                      ('description',), 
     #                      ('description',),
     #                      (('measure',), ('numerator',), ('denominator',)))
     # context
     table = self.getTable('context', 'context_id', 
                           ('accession_id', 'period_start', 'period_end', 'period_instant', 'specifies_dimensions', 'context_xml_id', 'entity_scheme', 'entity_identifier'), 
                           ('accession_id', 'context_xml_id'), 
                           tuple((accsId,
                                  cntx.startDatetime if cntx.isStartEndPeriod else None,
                                  cntx.endDatetime if cntx.isStartEndPeriod else None,
                                  cntx.instantDatetime if cntx.isInstantPeriod else None,
                                  bool(cntx.qnameDims),
                                  cntx.id,
                                  cntx.entityIdentifier[0],
                                  cntx.entityIdentifier[1])
                                 for cntx in self.modelXbrl.contexts.values()))
     self.cntxId = dict(((_accsId, xmlId), id)
                        for id, _accsId, xmlId in table)
     # context_dimension
     values = []
     for cntx in self.modelXbrl.contexts.values():
         for dim in cntx.qnameDims.values():
             values.append((self.cntxId[(accsId,cntx.id)],
                            self.qnameId[dim.dimensionQname],
                            self.qnameId.get(dim.memberQname), # may be None
                            self.qnameId.get(dim.typedMember.qname) if dim.isTyped else None,
                            False, # not default
                            dim.contextElement == "segment",
                            dim.typedMember.innerText if dim.isTyped else None))
         for dimQname, memQname in self.modelXbrl.qnameDimensionDefaults.items():
             if dimQname not in cntx.qnameDims:
                 values.append((self.cntxId[(accsId,cntx.id)],
                                self.qnameId[dimQname],
                                self.qnameId[memQname],
                                None,
                                True, # is default
                                True, # ambiguous and irrelevant for the XDT model
                                None))
     table = self.getTable('context_dimension', 'context_dimension_id', 
                           ('context_id', 'dimension_qname_id', 'member_qname_id', 'typed_qname_id', 'is_default', 'is_segment', 'typed_text_content'), 
                           ('dimension_qname_id',), 
                           values)
     # facts
     table = self.getTable('fact', 'fact_id', 
                           ('accession_id', 'context_id', 'unit_id', 'element_id', 'effective_value', 'fact_value', 
                            'xml_id', 'precision_value', 'decimals_value', 
                            'is_precision_infinity', 'is_decimals_infinity', ), 
                           ('accession_id', 'context_id', 'unit_id', 'element_id', 'fact_value'), 
                           tuple((accsId,
                                  self.cntxId.get((accsId,fact.contextID)),
                                  self.unitId.get((accsId,fact.unitID)),
                                  self.elementId.get(self.qnameId.get(fact.qname)),
                                  roundValue(fact.value, fact.precision, fact.decimals) if fact.isNumeric else None,
                                  fact.value,
                                  fact.id,
                                  fact.xAttributes['precision'].xValue if ('precision' in fact.xAttributes and isinstance(fact.xAttributes['precision'].xValue,int)) else None,
                                  fact.xAttributes['decimals'].xValue if ('decimals' in fact.xAttributes and isinstance(fact.xAttributes['decimals'].xValue,int)) else None,
                                  'precision' in fact.xAttributes and fact.xAttributes['precision'].xValue == 'INF',
                                  'decimals' in fact.xAttributes and fact.xAttributes['decimals'].xValue == 'INF',
                                  )
                                 for fact in self.modelXbrl.facts))
Ejemplo n.º 22
0
def _roundItem(node, sphinxContext, args):
    fact = factArg(node, sphinxContext, args, 0)
    return roundValue(fact.xValue,
                      decimals=fact.decimals,
                      precision=fact.precision)
Ejemplo n.º 23
0
def _roundDecimals(node, sphinxContext, args):
    args = numericArgs(node, sphinxContext, args, 2)
    return roundValue(args[0], decmials=args[1])
Ejemplo n.º 24
0
def _roundPrecision(node, sphinxContext, args):
    args = numericArgs(node, sphinxContext, args, 2)
    return roundValue(args[0], precision=args[1])
Ejemplo n.º 25
0
 def insertFacts(self):
     accsId = self.accessionId
     self.showStatus("insert facts")
     # units
     table = self.getTable(
         'unit', 'unit_id', ('accession_id', 'unit_xml_id'),
         ('accession_id', 'unit_xml_id'),
         tuple((accsId, unitId) for unitId in self.modelXbrl.units.keys()))
     self.unitId = dict(
         ((_accsId, xmlId), id) for id, _accsId, xmlId in table)
     # measures
     table = self.getTable(
         'unit_measure', 'unit_measure_id',
         ('unit_id', 'qname_id', 'location_id'),
         ('qname_id', 'location_id'),
         tuple((self.unitId[(accsId, unit.id)], self.qnameId[measure],
                1 if (not unit.measures[1]) else (i + 1))
               for unit in self.modelXbrl.units.values() for i in range(2)
               for measure in unit.measures[i]))
     #table = self.getTable('enumeration_measure_location', 'enumeration_measure_location_id',
     #                      ('description',),
     #                      ('description',),
     #                      (('measure',), ('numerator',), ('denominator',)))
     # context
     table = self.getTable(
         'context', 'context_id',
         ('accession_id', 'period_start', 'period_end', 'period_instant',
          'specifies_dimensions', 'context_xml_id', 'entity_scheme',
          'entity_identifier'), ('accession_id', 'context_xml_id'),
         tuple(
             (accsId, cntx.startDatetime if cntx.isStartEndPeriod else None,
              cntx.endDatetime if cntx.isStartEndPeriod else None,
              cntx.instantDatetime if cntx.isInstantPeriod else None,
              bool(cntx.qnameDims), cntx.id, cntx.entityIdentifier[0],
              cntx.entityIdentifier[1])
             for cntx in self.modelXbrl.contexts.values()))
     self.cntxId = dict(
         ((_accsId, xmlId), id) for id, _accsId, xmlId in table)
     # context_dimension
     values = []
     for cntx in self.modelXbrl.contexts.values():
         for dim in cntx.qnameDims.values():
             values.append((
                 self.cntxId[(accsId, cntx.id)],
                 self.qnameId[dim.dimensionQname],
                 self.qnameId.get(dim.memberQname),  # may be None
                 self.qnameId.get(dim.typedMember.qname)
                 if dim.isTyped else None,
                 False,  # not default
                 dim.contextElement == "segment",
                 dim.typedMember.innerText if dim.isTyped else None))
         for dimQname, memQname in self.modelXbrl.qnameDimensionDefaults.items(
         ):
             if dimQname not in cntx.qnameDims:
                 values.append((
                     self.cntxId[(accsId, cntx.id)],
                     self.qnameId[dimQname],
                     self.qnameId[memQname],
                     None,
                     True,  # is default
                     True,  # ambiguous and irrelevant for the XDT model
                     None))
     table = self.getTable(
         'context_dimension', 'context_dimension_id',
         ('context_id', 'dimension_qname_id', 'member_qname_id',
          'typed_qname_id', 'is_default', 'is_segment',
          'typed_text_content'), ('dimension_qname_id', ), values)
     # facts
     table = self.getTable(
         'fact', 'fact_id', (
             'accession_id',
             'context_id',
             'unit_id',
             'element_id',
             'effective_value',
             'fact_value',
             'xml_id',
             'precision_value',
             'decimals_value',
             'is_precision_infinity',
             'is_decimals_infinity',
         ), ('accession_id', 'context_id', 'unit_id', 'element_id',
             'fact_value'),
         tuple((
             accsId,
             self.cntxId.get((accsId, fact.contextID)),
             self.unitId.get((accsId, fact.unitID)),
             self.elementId.get(self.qnameId.get(fact.qname)),
             roundValue(fact.value, fact.precision, fact.decimals) if fact.
             isNumeric else None,
             fact.value,
             fact.id,
             fact.xAttributes['precision'].xValue if (
                 'precision' in fact.xAttributes and isinstance(
                     fact.xAttributes['precision'].xValue, int)) else None,
             fact.xAttributes['decimals'].xValue if (
                 'decimals' in fact.xAttributes and isinstance(
                     fact.xAttributes['decimals'].xValue, int)) else None,
             'precision' in fact.xAttributes
             and fact.xAttributes['precision'].xValue == 'INF',
             'decimals' in fact.xAttributes
             and fact.xAttributes['decimals'].xValue == 'INF',
         ) for fact in self.modelXbrl.facts))
Ejemplo n.º 26
0
def evaluate(xpCtx, varSet, derivedFact):
    # there may be multiple consis assertions parenting any formula
    for consisAsserRel in xpCtx.modelXbrl.relationshipSet(XbrlConst.consistencyAssertionFormula).toModelObject(varSet):
        consisAsser = consisAsserRel.fromModelObject
        hasProportionalAcceptanceRadius = consisAsser.hasProportionalAcceptanceRadius
        hasAbsoluteAcceptanceRadius = consisAsser.hasAbsoluteAcceptanceRadius
        if derivedFact is None:
            continue
        isNumeric = derivedFact.isNumeric
        if isNumeric and not derivedFact.isNil:
            derivedFactInferredPrecision = inferredPrecision(derivedFact)
            if derivedFactInferredPrecision == 0 and not hasProportionalAcceptanceRadius and not hasAbsoluteAcceptanceRadius:
                if xpCtx.formulaOptions.traceVariableSetExpressionResult:
                    xpCtx.modelXbrl.info(u"formula:trace",
                         _(u"Consistency assertion %(id)s formula %(xlinkLabel)s fact %(derivedFact)s has zero precision and no radius is defined, skipping consistency assertion"),
                         modelObject=consisAsser, id=consisAsser.id, xlinkLabel=varSet.xlinkLabel, derivedFact=derivedFact)
                continue
    
        # check xbrl validity of new fact
        
        # find source facts which match derived fact
        aspectMatchedInputFacts = []
        isStrict = consisAsser.isStrict
        for inputFact in xpCtx.modelXbrl.facts:
            if (not inputFact.isNil and
                inputFact.qname == derivedFact.qname and
                inputFact.context.isEqualTo(derivedFact.context,
                                            dimensionalAspectModel=(varSet.aspectModel == u"dimensional")) and
                (not isNumeric or inputFact.unit.isEqualTo(derivedFact.unit))):
                aspectMatchedInputFacts.append( inputFact )
        
        if len(aspectMatchedInputFacts) == 0:
            if isStrict:
                if derivedFact.isNil:
                    isSatisfied = True
                else:
                    isSatisfied = False
            else:
                if xpCtx.formulaOptions.traceVariableSetExpressionResult:
                    xpCtx.modelXbrl.info(u"formula:trace",
                         _(u"Consistency assertion %(id)s formula %(xlinkLabel)s no input facts matched to %(derivedFact)s, skipping consistency assertion"),
                         modelObject=consisAsser, id=consisAsser.id, xlinkLabel=varSet.xlinkLabel, derivedFact=derivedFact)
                continue
        elif derivedFact.isNil:
            isSatisfied = False
        else:
            isSatisfied = True
                
        paramQnamesAdded = []
        for paramRel in consisAsser.orderedVariableRelationships:
            paramQname = paramRel.variableQname
            paramVar = paramRel.toModelObject
            paramValue = xpCtx.inScopeVars.get(paramVar.parameterQname)
            paramAlreadyInVars = paramQname in xpCtx.inScopeVars
            if not paramAlreadyInVars:
                paramQnamesAdded.append(paramQname)
                xpCtx.inScopeVars[paramQname] = paramValue
        acceptance = None
        for fact in aspectMatchedInputFacts:
            if isSatisfied != True: 
                break
            if fact.isNil:
                if not derivedFact.isNil:
                    isSatisfied = False
            elif isNumeric:
                factInferredPrecision = inferredPrecision(fact)
                if factInferredPrecision == 0 and not hasProportionalAcceptanceRadius and not hasAbsoluteAcceptanceRadius:
                    if xpCtx.formulaOptions.traceVariableSetExpressionResult:
                        xpCtx.modelXbrl.info(u"formula:trace",
                             _(u"Consistency assertion %(id)s formula %(xlinkLabel)s input fact matched to %(derivedFact)s has zero precision and no radius, skipping consistency assertion"),
                             modelObject=consisAsser, id=consisAsser.id, xlinkLabel=varSet.xlinkLabel, derivedFact=derivedFact)
                        isSatisfied = None
                        break
                if hasProportionalAcceptanceRadius or hasAbsoluteAcceptanceRadius:
                    acceptance = consisAsser.evalRadius(xpCtx, derivedFact.vEqValue)
                    if acceptance is not None:
                        if hasProportionalAcceptanceRadius:
                            acceptance *= derivedFact.vEqValue
                        isSatisfied = fabs(derivedFact.vEqValue - fact.vEqValue) <= fabs(acceptance)
                    else:
                        isSatisfied = None  # no radius
                else:
                    p = min(derivedFactInferredPrecision, factInferredPrecision)
                    if (p == 0 or
                        roundValue(derivedFact.value, precision=p) != roundValue(fact.value, precision=p)):
                        isSatisfied = False
            else:
                if not xEqual(fact, derivedFact, equalMode=S_EQUAL2):
                    isSatisfied = False
        if isSatisfied is not None:  # None means no evaluation
            if xpCtx.formulaOptions.traceVariableSetExpressionResult:
                xpCtx.modelXbrl.info(u"formula:trace",
                     _(u"Consistency assertion %(id)s result %(result)s"),
                     modelObject=consisAsser, id=consisAsser.id, result=isSatisfied)
            message = consisAsser.message(isSatisfied)
            if message is not None:
                xpCtx.inScopeVars[XbrlConst.qnCaAspectMatchedFacts] = aspectMatchedInputFacts
                xpCtx.inScopeVars[XbrlConst.qnCaAcceptanceRadius] = acceptance
                xpCtx.inScopeVars[XbrlConst.qnCaAbsoluteAcceptanceRadiusExpression] = consisAsser.get(u"absoluteAcceptanceRadius")
                xpCtx.inScopeVars[XbrlConst.qnCaProportionalAcceptanceRadiusExpression] = consisAsser.get(u"proportionalAcceptanceRadius")
                xpCtx.modelXbrl.info(u"message:" + consisAsser.id, message.evaluate(xpCtx),
                                     modelObject=message,
                                     messageCodes=(u"message:{variableSetID|xlinkLabel}"))
                xpCtx.inScopeVars.pop(XbrlConst.qnCaAspectMatchedFacts)
                xpCtx.inScopeVars.pop(XbrlConst.qnCaAcceptanceRadius)
                xpCtx.inScopeVars.pop(XbrlConst.qnCaAbsoluteAcceptanceRadiusExpression)
                xpCtx.inScopeVars.pop(XbrlConst.qnCaProportionalAcceptanceRadiusExpression)
            if isSatisfied: consisAsser.countSatisfied += 1
            else: consisAsser.countNotSatisfied += 1
        for paramQname in paramQnamesAdded:
            xpCtx.inScopeVars.pop(paramQname)