Exemplo n.º 1
0
    def __init__(
        self,
        underlying: str,
        currency: Currency,
        optionType: OptionType,
        expiration: date,
        strike: Decimal,
        multiplier: Decimal = Decimal(100),
        exchange: Optional[str] = None,
        symbol: Optional[str] = None,
    ):
        if not underlying:
            raise ValueError("Expected non-empty underlying symbol for Option")
        if not strike.is_finite() or strike <= 0:
            raise ValueError(f"Expected positive strike price: {strike}")
        if not multiplier.is_finite() or multiplier <= 0:
            raise ValueError(f"Expected positive multiplier: {multiplier}")

        self.underlying = underlying
        self.optionType = optionType
        self.expiration = expiration
        self.strike = self.quantizeStrike(strike)

        if symbol is None:
            # https://en.wikipedia.org/wiki/Option_symbol#The_OCC_Option_Symbol
            symbol = f"{underlying:6}{expiration.strftime('%y%m%d')}{optionType.value}{(strike * 1000):08.0f}"

        super().__init__(symbol=symbol,
                         currency=currency,
                         multiplier=multiplier,
                         exchange=exchange)
Exemplo n.º 2
0
def holdings(
    val: pd.DataFrame, holds: np.ndarray, i: pd.DataFrame, t: int, aum_t: Decimal
) -> Decimal:
    open_price = Decimal(val[i].loc["open"][t])

    # If the open price is NaN, this instrument's open wasn't recorded at time t.
    # So let's use the previous day's calculation.
    if not open_price.is_finite():
        prev_day: Decimal = holds[t - 1][val.columns.get_loc(i)]
        return prev_day
    else:
        # TODO: make the exchange rate flexible. This should generally be the dollar value of 1 point of instrument i.
        exchange_rate = Decimal(1)
        # The purpose of the (weights_i_t * sum_of_weights ^ -1) in the holdings calculation is to de-lever the allocations.
        sum_of_weights: Decimal = val.loc["weight"].iloc[t].abs().sum()

        # If today was t + 1, one way to calculate today's holdings would be by using the opening price, since for a future
        # we may not know the closing price of a new contract at roll time.
        # If open prices are unavailable, then the last close price at t will work too.
        next_open = Decimal(val[i].loc["close"][t])
        if not next_open.is_finite():
            last_close_price: Decimal = holds[t - 1][val.columns.get_loc(i)]
            return last_close_price

        weighted_holding: Decimal = Decimal(val[i].loc["weight"][t]) * aum_t / Decimal(
            next_open
        ) * exchange_rate * Decimal(sum_of_weights)
        return weighted_holding
Exemplo n.º 3
0
def formatNumberPair(value, *errors, precision=2, separator=" +- "):
	c = Context(prec=precision)

	largest_error = None
	for error in errors:
		e = c.create_decimal_from_float(error)
		if e.is_finite():
			if largest_error:
				largest_error = largest_error.max_mag(e)
			else:
				largest_error = e

	results = []

	value = Decimal(value)

	if largest_error and value.is_finite():
		value = value.quantize(largest_error)
	results.append(value)

	for error in errors:
		error = Decimal(error)
		if largest_error and error.is_finite():
			error = error.quantize(largest_error)
		results.append(error)
	return tuple((str(r) for r in results))
Exemplo n.º 4
0
def receive_fmi_temperature(
) -> Tuple[Optional[Decimal], Optional[arrow.Arrow]]:
    temp, ts = None, None

    try:
        starttime = arrow.now().shift(
            hours=-1).to('UTC').format('YYYY-MM-DDTHH:mm:ss') + 'Z'
        result = get_url(
            'https://opendata.fmi.fi/wfs?request=getFeature&storedquery_id=fmi::observations::weather'
            '::simple&place={place}&parameters=temperature&starttime={starttime}'
            .format(place=config.FMI_LOCATION, starttime=starttime))
    except Exception as e:
        logger.exception(e)
    else:
        if result.status_code != 200:
            logger.error('%d: %s' % (result.status_code, result.content))
        else:
            try:
                wfs_member = xmltodict.parse(result.content).get(
                    'wfs:FeatureCollection', {}).get('wfs:member')
                temp_data = wfs_member[-1].get('BsWfs:BsWfsElement')
                if temp_data and 'BsWfs:Time' in temp_data and 'BsWfs:ParameterValue' in temp_data:
                    ts = arrow.get(temp_data['BsWfs:Time']).to(config.TIMEZONE)
                    temp = Decimal(temp_data['BsWfs:ParameterValue'])
                    if not temp.is_finite():
                        raise TypeError()
            except (KeyError, TypeError):
                temp, ts = None, None

    logger.info('temp:%s ts:%s', temp, ts)
    return temp, ts
Exemplo n.º 5
0
def _parseFiniteDecimal(input: str) -> Decimal:
    with localcontext(ctx=Context(traps=[DivisionByZero, Overflow])):
        value = Decimal(input)
        if not value.is_finite():
            raise ValueError(f"Input is not numeric: {input}")

        return value
Exemplo n.º 6
0
    def _apply(self, value):
        allowed_types = (str, int, float, DecimalType,)
        if self.allow_tuples:
            # Python's Decimal type supports both tuples and lists.
            # :py:meth:`decimal.Decimal.__init__`
            allowed_types += (list, tuple,)

        value = self._filter(value, Type(allowed_types))

        if self._has_errors:
            return value

        try:
            d = DecimalType(value)
        except (InvalidOperation, TypeError, ValueError):
            return self._invalid_value(value, self.CODE_INVALID, exc_info=True)

        # Decimal's constructor also accepts values such as 'NaN' or
        # '+Inf', which aren't valid in this context.
        # :see: decimal.Decimal._parser
        if not d.is_finite():
            return self._invalid_value(
                value=value,
                reason=self.CODE_NON_FINITE,
                exc_info=True,
            )

        if self.max_precision is not None:
            d = d.quantize(self.max_precision)

        return d
Exemplo n.º 7
0
 def to_python(self, value, timezone_in_use):
     if not isinstance(value, Decimal):
         try:
             value = Decimal(value)
         except:
             raise ValueError('Invalid value for %s - %r' % (self.__class__.__name__, value))
     if not value.is_finite():
             raise ValueError('Non-finite value for %s - %r' % (self.__class__.__name__, value))
     return self._round(value)
Exemplo n.º 8
0
def test_decimal_from_float(f):
    d = Decimal(f)
    if isfinite(f) and d.is_finite():
        py_d = Real(d)
        assert isclose(py_d.decode(), d), (d, f.hex())
    else:
        with pytest.raises(ValueError, message=repr(f)):
            Real(f)
        with pytest.raises(ValueError, message=repr(d)):
            Real(d)
Exemplo n.º 9
0
def convert_decimal(value: Decimal) -> Union[float, int]:
    if isinstance(value, Decimal):
        if not value.is_finite():
            return str(value)
        if str(value).find('.') > -1:
            return float(value.real)
        else:
            return int(value)
    else:
        raise Exception('{} is not Decimal'.format(value))
Exemplo n.º 10
0
def doUnit(cur):
	id=None
	y="0"
	f=None
	symbol=None
	for (k, v) in attrs(cur):
		if k == "id": id = v
		elif k == "name": name = v
		elif k == "eval": e=v
		# elif k == "ref": pass
		elif k == "href": pass
		elif k == "factor": f = v
		elif k == "y": y = v
		elif k == "symbol": symbol = v
		elif k == "no": pass
		else: raise RuntimeError("Unexpected attribute %r=%r" % (k, v))
	no = doCheckNo(cur)
	if not symbol:
		no_symbol.append((name, no))
	if no in conv_map:
		 raise RuntimeError("no %X in conv_map" % no)
	doCheckName(cur)
	if f:
		f=Decimal(f) 
	else:
		f=dcalc(e) 
	assert f.is_finite()
	assert (1/f).is_finite()
	assert Decimal(y).is_finite()
	assert no not in conv_map
	assert no not in types_map
	doCheckEval(cur)
	conv_map[no] = (f, Decimal(y), name)
	cur = cur.firstChild
	while cur:
		x = cur
		(cur, t) = (x.nextSibling, x.nodeType)
		if t == x.ELEMENT_NODE:
			t = x.tagName
			if t == "same":
				iB = doCheckNo(x, False)
				vB = dcalc(x.getAttribute("equals"))
				vA = Decimal(x.getAttribute("value") or 1)
				same_list.append((no, iB, vB, vA))
			else:
				raise RuntimeError("unexpected element %r" % x.tagName)
		elif t == x.TEXT_NODE:
			if x.previousSibling:
				if x.data.strip():
					raise RuntimeError("unexpected TEXT_NODE %r" % x.data.strip())
		elif t in (x.COMMENT_NODE, ):
			pass
		else:
			raise RuntimeError("unexpected node %r %r" % (t, x.nodeName))
Exemplo n.º 11
0
    def next_unstalled_actions(self, time: Decimal):
        if time.is_finite() and time > self.current_plan_execution_limit:
            return NoAction.ExecutionLimitReached
        elif self.plan.empty():
            return NoAction.PlanEmpty
        elif self.plan.peek().start_time > time:
            return NoAction.TooEarly

        all_actions = self.plan.get()
        unstalled_actions = [action for action in all_actions if self.is_action_available(action, time)]
        return unstalled_actions
Exemplo n.º 12
0
def parse_number(s):
    parsed = None
    try:
        parsed = Decimal(s)

        if not parsed.is_finite() or parsed > Decimal(
                "999999999999999999999999"):
            parsed = None
    except Exception:
        pass
    return parsed
Exemplo n.º 13
0
def _represent_decimal(self: ruamel.yaml.BaseRepresenter,
                       data: decimal.Decimal) -> ruamel.yaml.ScalarNode:
    if data.is_finite():
        s = str(
            _POINT_ZERO_DECIMAL + data
        )  # The zero addition is to force float-like string representation
    elif data.is_nan():
        s = ".nan"
    elif data.is_infinite():
        s = ".inf" if data > 0 else "-.inf"
    else:
        assert False
    return self.represent_scalar("tag:yaml.org,2002:float", s)
Exemplo n.º 14
0
def decimal_validator(v) -> Decimal:
    if isinstance(v, Decimal):
        return v
    elif isinstance(v, (bytes, bytearray)):
        v = v.decode()

    v = str(v).strip()

    with change_exception(errors.DecimalError, DecimalException):
        v = Decimal(v)

    if not v.is_finite():
        raise errors.DecimalIsNotFiniteError()

    return v
Exemplo n.º 15
0
 def from_decimal(cls, dec):
     from decimal import Decimal
     if isinstance(dec, numbers.Integral):
         dec = Decimal(int(dec))
     elif not isinstance(dec, Decimal):
         raise TypeError('%s.from_decimal() only takes Decimals, not %r (%s)' % (cls.__name__, dec, type(dec).__name__))
     if not dec.is_finite():
         raise TypeError('Cannot convert %s to %s.' % (dec, cls.__name__))
     (sign, digits, exp) = dec.as_tuple()
     digits = int(''.join(map(str, digits)))
     if sign:
         digits = -digits
     if exp >= 0:
         return cls(digits*10**exp)
     return cls(digits, 10**(-exp))
Exemplo n.º 16
0
def xrp_to_drops(xrp: Union[int, float, Decimal]) -> str:
    """
    Convert a numeric XRP amount to drops of XRP.

    Args:
        xrp: Numeric representation of whole XRP

    Returns:
        Equivalent amount in drops of XRP

    Raises:
        TypeError: if ``xrp`` is given as a string
        XRPRangeException: if the given amount of XRP is invalid
    """
    if type(xrp) == str:  # type: ignore
        # This protects people from passing drops to this function and getting
        # a million times as many drops back.
        raise TypeError(
            "XRP provided as a string. Use a number format" "like Decimal or int."
        )
    setcontext(_DROPS_CONTEXT)
    try:
        xrp_d = Decimal(xrp)
    except InvalidOperation:
        raise XRPRangeException(f"Not a valid amount of XRP: '{xrp}'")

    if not xrp_d.is_finite():  # NaN or an Infinity
        raise XRPRangeException(f"Not a valid amount of XRP: '{xrp}'")

    if xrp_d < ONE_DROP and xrp_d != 0:
        raise XRPRangeException(f"XRP amount {xrp} is too small.")
    if xrp_d > MAX_XRP:
        raise XRPRangeException(f"XRP amount {xrp} is too large.")

    drops_amount = (xrp_d / ONE_DROP).quantize(Decimal(1))
    drops_str = str(drops_amount).strip()

    # This should never happen, but is a precaution against Decimal doing
    # something unexpected.
    if not fullmatch(_DROPS_REGEX, drops_str):
        raise XRPRangeException(
            f"xrp_to_drops failed sanity check. Value "
            f"'{drops_str}' does not match the drops regex"
        )

    return drops_str
Exemplo n.º 17
0
 def from_decimal(cls, dec):
     """Converts a finite Decimal instance to a rational number, exactly."""
     from decimal import Decimal
     if isinstance(dec, numbers.Integral):
         dec = Decimal(int(dec))
     elif not isinstance(dec, Decimal):
         raise TypeError('%s.from_decimal() only takes Decimals, not %r (%s)' % (cls.__name__, dec, type(dec).__name__))
     if not dec.is_finite():
         raise TypeError('Cannot convert %s to %s.' % (dec, cls.__name__))
     sign, digits, exp = dec.as_tuple()
     digits = int(''.join(map(str, digits)))
     if sign:
         digits = -digits
     if exp >= 0:
         return cls(digits * 10 ** exp)
     else:
         return cls(digits, 10 ** (-exp))
Exemplo n.º 18
0
def test_decimal_from_float(f):
    d = Decimal(f)
    if isfinite(f) and d.is_finite():
        try:
            # PDF is limited to ~5 sig figs
            decstr = str(d.quantize(Decimal('1.000000')))
        except InvalidOperation:
            return  # PDF doesn't support exponential notation
        try:
            py_d = Object.parse(decstr)
        except RuntimeError as e:
            if 'overflow' in str(e) or 'underflow' in str(e):
                py_d = Object.parse(str(f))

        assert isclose(py_d, d, abs_tol=1e-5), (d, f.hex())
    else:
        with pytest.raises(PdfError):
            Object.parse(str(d))
Exemplo n.º 19
0
def decimal_validator(v) -> Decimal:
    if isinstance(v, Decimal):
        return v
    elif isinstance(v, (bytes, bytearray)):
        v = v.decode()

    v = str(v).strip()

    try:
        v = Decimal(v)
    except DecimalException as e:
        raise TypeError(
            f'value is not a valid decimal, got {display_as_type(v)}') from e

    if not v.is_finite():
        raise TypeError(
            f'value is not a valid decimal, got {display_as_type(v)}')

    return v
Exemplo n.º 20
0
 def from_decimal(cls, dec):
     """Converts a finite Decimal instance to a rational number, exactly."""
     from decimal import Decimal
     if isinstance(dec, numbers.Integral):
         dec = Decimal(int(dec))
     elif not isinstance(dec, Decimal):
         raise TypeError(
             "%s.from_decimal() only takes Decimals, not %r (%s)" %
             (cls.__name__, dec, type(dec).__name__))
     if not dec.is_finite():
         # Catches infinities and nans.
         raise TypeError("Cannot convert %s to %s." % (dec, cls.__name__))
     sign, digits, exp = dec.as_tuple()
     digits = int(''.join(map(str, digits)))
     if sign:
         digits = -digits
     if exp >= 0:
         return cls(digits * 10 ** exp)
     else:
         return cls(digits, 10 ** -exp)
Exemplo n.º 21
0
    def validate(self, value: Decimal) -> Iterable[ValidationError]:
        if not value.is_finite():  # check for Inf/NaN/sNaN/qNaN
            yield self.validation_error(f"Illegal value in decimal: {value!r}")

        decimal_tuple: Optional[DecimalTuple] = None

        mdp = self.max_decimal_places
        if mdp:
            decimal_tuple = value.as_tuple()
            if abs(decimal_tuple.exponent) > mdp:
                yield self.validation_error(
                    f"{self.field} must have less than {mdp} decimal places.")
        max_digits = self.max_digits
        if max_digits:
            if decimal_tuple is None:
                decimal_tuple = value.as_tuple()
            digits = len(decimal_tuple.digits[:decimal_tuple.exponent])
            if digits > max_digits:
                yield self.validation_error(
                    f"{self.field} must have less than {max_digits} digits.")
Exemplo n.º 22
0
def str_to_decimal(s: str, maxlen: int = DECIMAL_MAXLEN) -> Optional[Decimal]:
    """Convert string to :class:`~decimal.Decimal`.

    Args:
        s (str): Number to convert.
        maxlen (int): Max length of string.  Default is 100.

    Raises:
        ValueError: if length exceeds maximum length, or if value is not
            a valid number (e.g. Inf, NaN or sNaN).

    Returns:
        Decimal: Converted number.
    """
    if s is None:
        return None
    if len(s) > maxlen:
        raise ValueError(
            f'string of length {len(s)} is longer than limit ({maxlen})')
    v = Decimal(s)
    if not v.is_finite():  # check for Inf/NaN/sNaN/qNaN
        raise ValueError(f'Illegal value in decimal: {s!r}')
    return v
Exemplo n.º 23
0
def streamingExtensionsLoader(modelXbrl, mappedUri, filepath, **kwargs):
    # check if big instance and has header with an initial incomplete tree walk (just 2 elements
    if not _streamingExtensionsCheck:
        return None
    
    # track whether modelXbrl has been validated by this streaming extension
    modelXbrl._streamingExtensionValidated = False
        
    def logSyntaxErrors(parsercontext):
        for error in parsercontext.error_log:
            modelXbrl.error("xmlSchema:syntax",
                    _("%(error)s, %(fileName)s, line %(line)s, column %(column)s, %(sourceAction)s source element"),
                    modelObject=modelXbrl, fileName=os.path.basename(filepath), 
                    error=error.message, line=error.line, column=error.column, sourceAction="streaming")
    #### note: written for iterparse of lxml prior to version 3.3, otherwise rewrite to use XmlPullParser ###
    #### note: iterparse wants a binary file, but file is text mode
    _file, = modelXbrl.fileSource.file(filepath, binary=True)
    startedAt = time.time()
    modelXbrl.profileActivity()
    ''' this seems twice as slow as iterparse
    class instInfoTarget():
        def __init__(self, element_factory=None, parser=None):
            self.newTree = True
            self.streamingAspects = None
            self.foundInstance = False
            self.creationSoftwareComment = ''
            self.currentEltTag = "(before xbrli:xbrl)"
            self.numRootFacts = 0
        def start(self, tag, attrib, nsmap=None):
            if self.newTree:
                if tag == "{http://www.xbrl.org/2003/instance}xbrl":
                    self.foundInstance = True
                    self.newTree = False
                else: # break 
                    raise NotInstanceDocumentException()
            elif not tag.startswith("{http://www.xbrl.org/"):
                self.numRootFacts += 1
                if self.numRootFacts % 1000 == 0:
                    modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
            self.currentEltTag = tag
        def end(self, tag):
            pass
        def data(self, data):
            pass
        def comment(self, text):
            if not self.foundInstance: # accumulate comments before xbrli:xbrl
                self.creationSoftwareComment += ('\n' if self.creationSoftwareComment else '') + text
            elif not self.creationSoftwareComment:
                self.creationSoftwareComment = text # or first comment after xbrli:xbrl
        def pi(self, target, data):
            if target == "xbrl-streamable-instance":
                if self.currentEltTag == "{http://www.xbrl.org/2003/instance}xbrl":
                    self.streamingAspects = dict(etree.PI(target,data).attrib.copy()) # dereference target results
                else:
                    modelXbrl.error("streamingExtensions:headerMisplaced",
                            _("Header is misplaced: %(target)s, must follow xbrli:xbrl element but was found at %(element)s"),
                            modelObject=modelXbrl, target=target, element=self.currentEltTag)
        def close(self):
            if not self.creationSoftwareComment:
                self.creationSoftwareComment = None
            return True
    instInfo = instInfoTarget()
    infoParser = etree.XMLParser(recover=True, huge_tree=True, target=instInfo)
    try:
        etree.parse(_file, parser=infoParser, base_url=filepath)
    except NotInstanceDocumentException:
        pass
    '''
    foundErrors = False
    foundInstance = False
    streamingAspects = None
    creationSoftwareComment = None
    instInfoNumRootFacts = 0
    numElts = 0
    elt = None
    instInfoContext = etree.iterparse(_file, events=("start","end"), huge_tree=True)
    for event, elt in instInfoContext:
        if event == "start":
            if elt.getparent() is not None:
                if elt.getparent().tag == "{http://www.xbrl.org/2003/instance}xbrl":
                    if not foundInstance:
                        foundInstance = True
                        pi = precedingProcessingInstruction(elt, "xbrl-streamable-instance")
                        if pi is None:
                            break
                        else:
                            streamingAspects = dict(pi.attrib.copy())
                            if creationSoftwareComment is None:
                                creationSoftwareComment = precedingComment(elt)
                    if not elt.tag.startswith("{http://www.xbrl.org/"):
                        instInfoNumRootFacts += 1
                        if instInfoNumRootFacts % 1000 == 0:
                            modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
                elif not foundInstance:       
                    break
            elif elt.tag == "{http://www.xbrl.org/2003/instance}xbrl":
                creationSoftwareComment = precedingComment(elt)
                if precedingProcessingInstruction(elt, "xbrl-streamable-instance") is not None:
                    modelXbrl.error("streamingExtensions:headerMisplaced",
                            _("Header is misplaced: %(error)s, must follow xbrli:xbrl element"),
                            modelObject=elt)
        elif event == "end":
            elt.clear()
            numElts += 1
            if numElts % 1000 == 0 and elt.getparent() is not None:
                while elt.getprevious() is not None and elt.getparent() is not None:
                    del elt.getparent()[0]
    if elt is not None:
        elt.clear()
    
    _file.seek(0,io.SEEK_SET) # allow reparsing
    if not foundInstance or streamingAspects is None:
        del elt
        _file.close()
        return None
    modelXbrl.profileStat(_("streaming tree check"), time.time() - startedAt)
    startedAt = time.time()
    try:
        version = Decimal(streamingAspects.get("version"))
        if int(version) != 1:
            modelXbrl.error("streamingExtensions:unsupportedVersion",
                    _("Streaming version %(version)s, major version number must be 1"),
                    modelObject=elt, version=version)
            foundErrors = True
    except (InvalidOperation, OverflowError):
        modelXbrl.error("streamingExtensions:versionError",
                _("Version %(version)s, number must be 1.n"),
                modelObject=elt, version=streamingAspects.get("version", "(none)"))
        foundErrors = True
    for bufAspect in ("contextBuffer", "unitBuffer", "footnoteBuffer"):
        try:
            bufLimit = Decimal(streamingAspects.get(bufAspect, "INF"))
            if bufLimit < 1 or (bufLimit.is_finite() and bufLimit % 1 != 0):
                raise InvalidOperation
            elif bufAspect == "contextBuffer":
                contextBufferLimit = bufLimit
            elif bufAspect == "unitBuffer":
                unitBufferLimit = bufLimit
            elif bufAspect == "footnoteBuffer":
                footnoteBufferLimit = bufLimit
        except InvalidOperation:
            modelXbrl.error("streamingExtensions:valueError",
                    _("Streaming %(attrib)s %(value)s, number must be a positive integer or INF"),
                    modelObject=elt, attrib=bufAspect, value=streamingAspects.get(bufAspect))
            foundErrors = True
    if _streamingExtensionsValidate:
        incompatibleValidations = []
        _validateDisclosureSystem = modelXbrl.modelManager.validateDisclosureSystem
        _disclosureSystem = modelXbrl.modelManager.disclosureSystem
        if _validateDisclosureSystem and _disclosureSystem.EFM:
            incompatibleValidations.append("EFM")
        if _validateDisclosureSystem and _disclosureSystem.GFM:
            incompatibleValidations.append("GFM")
        if _validateDisclosureSystem and _disclosureSystem.EBA:
            incompatibleValidations.append("EBA")
        if _validateDisclosureSystem and _disclosureSystem.HMRC:
            incompatibleValidations.append("EBA")
        if modelXbrl.modelManager.validateCalcLB:
            incompatibleValidations.append("calculation LB")
        if incompatibleValidations:            
            modelXbrl.error("streamingExtensions:incompatibleValidation",
                    _("Streaming instance validation does not support %(incompatibleValidations)s validation"),
                    modelObject=modelXbrl, incompatibleValidations=', '.join(incompatibleValidations))
            foundErrors = True
    if instInfoContext.error_log:
        foundErrors = True
    logSyntaxErrors(instInfoContext)
    del instInfoContext # dereference
    
    if foundErrors:
        _file.close()
        return None

    _encoding = XmlUtil.encoding(_file.read(512))
    _file.seek(0,io.SEEK_SET) # allow reparsing

    if _streamingExtensionsValidate:
        validator = Validate(modelXbrl)
        instValidator = validator.instValidator

    eltMdlObjs = {}
    contextBuffer = []
    unitBuffer = []
    footnoteBuffer = []
    factBuffer = []
    numFacts = 1
    
    class modelLoaderTarget():
        def __init__(self, element_factory=None, parser=None):
            self.newTree = True
            self.currentMdlObj = None
            self.beforeInstanceStream = True
            self.numRootFacts = 1
        def start(self, tag, attrib, nsmap=None):
            mdlObj = _parser.makeelement(tag, attrib=attrib, nsmap=nsmap)
            mdlObj.sourceline = 1
            if self.newTree:
                self.newTree = False
                self.currentMdlObj = mdlObj
                modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, mdlObj.getroottree())
                modelXbrl.modelDocument = modelDocument # needed for incremental validation
                mdlObj.init(modelDocument)
                modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement 
                modelDocument.parserLookupName = _parserLookupName
                modelDocument.parserLookupClass = _parserLookupClass
                modelDocument.xmlRootElement = mdlObj
                modelDocument.schemaLocationElements.add(mdlObj)
                modelDocument.documentEncoding = _encoding
                modelDocument._creationSoftwareComment = creationSoftwareComment
                modelXbrl.info("streamingExtensions:streaming",
                               _("Stream processing this instance."),
                               modelObject = modelDocument)    
            else:
                self.currentMdlObj.append(mdlObj)
                self.currentMdlObj = mdlObj
                mdlObj._init()
                ns = mdlObj.namespaceURI
                ln = mdlObj.localName
                if (self.beforeInstanceStream and (
                    (ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or
                    (ns == XbrlConst.xbrli and ln in ("context", "unit")) or
                    (ns not in (XbrlConst.link, XbrlConst.xbrli)))):
                    self.beforeInstanceStream = False
                    if _streamingExtensionsValidate:
                        instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters())
                    else: # need default dimensions
                        ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
            return mdlObj
        def end(self, tag):
            modelDocument = modelXbrl.modelDocument
            mdlObj = self.currentMdlObj
            parentMdlObj = mdlObj.getparent()
            self.currentMdlObj = parentMdlObj
            ns = mdlObj.namespaceURI
            ln = mdlObj.localName
            if ns == XbrlConst.xbrli:
                if ln == "context":
                    if mdlObj.get("sticky"):
                        del mdlObj.attrib["sticky"]
                        XmlValidate.validate(modelXbrl, mdlObj)
                        modelDocument.contextDiscover(mdlObj)
                    else:
                        if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit:
                            # drop before adding as dropped may have same id as added
                            cntx = contextBuffer.pop(0)
                            dropContext(modelXbrl, cntx)
                            del parentMdlObj[parentMdlObj.index(cntx)]
                            cntx = None
                        XmlValidate.validate(modelXbrl, mdlObj)
                        modelDocument.contextDiscover(mdlObj)
                        if contextBufferLimit.is_finite():
                            contextBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        contextsToCheck = (mdlObj,)
                        instValidator.checkContexts(contextsToCheck)
                        if modelXbrl.hasXDT:
                            instValidator.checkContextsDimensions(contextsToCheck)
                        del contextsToCheck # dereference
                elif ln == "unit":
                    if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit:
                        # drop before additing as dropped may have same id as added
                        unit = unitBuffer.pop(0)
                        dropUnit(modelXbrl, unit)
                        del parentMdlObj[parentMdlObj.index(unit)]
                        unit = None 
                    XmlValidate.validate(modelXbrl, mdlObj)
                    modelDocument.unitDiscover(mdlObj)
                    if unitBufferLimit.is_finite():
                        unitBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        instValidator.checkUnits( (mdlObj,) )
                elif ln == "xbrl": # end of document
                    # check remaining footnote refs
                    for footnoteLink in footnoteBuffer:
                        checkFootnoteHrefs(modelXbrl, footnoteLink)
            elif ns == XbrlConst.link:
                if ln == "footnoteLink":
                    XmlValidate.validate(modelXbrl, mdlObj)
                    footnoteLinks = (mdlObj,)
                    modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True)
                    if footnoteBufferLimit.is_finite():
                        footnoteBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        instValidator.checkLinks(footnoteLinks)
                        if len(footnoteBuffer) > footnoteBufferLimit:
                            # check that hrefObjects for locators were all satisfied
                                # drop before addition as dropped may have same id as added
                            footnoteLink = footnoteBuffer.pop(0)
                            checkFootnoteHrefs(modelXbrl, footnoteLink)
                            dropFootnoteLink(modelXbrl, footnoteLink)
                            del parentMdlObj[parentMdlObj.index(footnoteLink)]
                            footnoteLink = None
                    footnoteLinks = None
                elif ln in ("schemaRef", "linkbaseRef"):
                    modelDocument.discoverHref(mdlObj)
                elif not modelXbrl.skipDTS:
                    if ln in ("roleRef", "arcroleRef"):
                        modelDocument.linkbaseDiscover((mdlObj,), inInstance=True)
            elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl:
                self.numRootFacts += 1
                XmlValidate.validate(modelXbrl, mdlObj)
                modelDocument.factDiscover(mdlObj, modelXbrl.facts)
                if _streamingExtensionsValidate:
                    factsToCheck = (mdlObj,)
                    instValidator.checkFacts(factsToCheck)
                    if modelXbrl.hasXDT:
                        instValidator.checkFactsDimensions(factsToCheck)
                    del factsToCheck
                    dropFact(modelXbrl, mdlObj, modelXbrl.facts)
                    del parentMdlObj[parentMdlObj.index(mdlObj)]
                if self.numRootFacts % 1000 == 0:
                    modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts, 
                                                                                              100.0 * self.numRootFacts / instInfoNumRootFacts), 
                                              minTimeToShow=20.0)
            return mdlObj
        def data(self, data):
            self.currentMdlObj.text = data
        def comment(self, text):
            pass
        def pi(self, target, data):
            pass
        def close(self):
            return None
        
    _parser, _parserLookupName, _parserLookupClass = parser(modelXbrl, filepath, target=modelLoaderTarget())
    etree.parse(_file, parser=_parser, base_url=filepath)
    logSyntaxErrors(_parser)
    _file.close()
    if _streamingExtensionsValidate and validator is not None:
        del instValidator
        validator.close()
        # track that modelXbrl has been validated by this streaming extension
        modelXbrl._streamingExtensionValidated = True
        
    modelXbrl.profileStat(_("streaming complete"), time.time() - startedAt)
    return modelXbrl.modelDocument
Exemplo n.º 24
0
def streamingExtensionsLoader(modelXbrl, mappedUri, filepath, *args, **kwargs):
    # check if big instance and has header with an initial incomplete tree walk (just 2 elements
    if not _streamingExtensionsCheck:
        return None

    # track whether modelXbrl has been validated by this streaming extension
    modelXbrl._streamingExtensionValidated = False

    def logSyntaxErrors(parsercontext):
        for error in parsercontext.error_log:
            modelXbrl.error(
                "xmlSchema:syntax",
                _("%(error)s, %(fileName)s, line %(line)s, column %(column)s, %(sourceAction)s source element"
                  ),
                modelObject=modelXbrl,
                fileName=os.path.basename(filepath),
                error=error.message,
                line=error.line,
                column=error.column,
                sourceAction="streaming")

    #### note: written for iterparse of lxml prior to version 3.3, otherwise rewrite to use XmlPullParser ###
    #### note: iterparse wants a binary file, but file is text mode
    _file, = modelXbrl.fileSource.file(filepath, binary=True)
    startedAt = time.time()
    modelXbrl.profileActivity()
    ''' this seems twice as slow as iterparse
    class instInfoTarget():
        def __init__(self, element_factory=None, parser=None):
            self.newTree = True
            self.streamingAspects = None
            self.foundInstance = False
            self.creationSoftwareComment = ''
            self.currentEltTag = "(before xbrli:xbrl)"
            self.numRootFacts = 0
        def start(self, tag, attrib, nsmap=None):
            if self.newTree:
                if tag == "{http://www.xbrl.org/2003/instance}xbrl":
                    self.foundInstance = True
                    self.newTree = False
                else: # break 
                    raise NotInstanceDocumentException()
            elif not tag.startswith("{http://www.xbrl.org/"):
                self.numRootFacts += 1
                if self.numRootFacts % 1000 == 0:
                    modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
            self.currentEltTag = tag
        def end(self, tag):
            pass
        def data(self, data):
            pass
        def comment(self, text):
            if not self.foundInstance: # accumulate comments before xbrli:xbrl
                self.creationSoftwareComment += ('\n' if self.creationSoftwareComment else '') + text
            elif not self.creationSoftwareComment:
                self.creationSoftwareComment = text # or first comment after xbrli:xbrl
        def pi(self, target, data):
            if target == "xbrl-streamable-instance":
                if self.currentEltTag == "{http://www.xbrl.org/2003/instance}xbrl":
                    self.streamingAspects = dict(etree.PI(target,data).attrib.copy()) # dereference target results
                else:
                    modelXbrl.error("streamingExtensions:headerMisplaced",
                            _("Header is misplaced: %(target)s, must follow xbrli:xbrl element but was found at %(element)s"),
                            modelObject=modelXbrl, target=target, element=self.currentEltTag)
        def close(self):
            if not self.creationSoftwareComment:
                self.creationSoftwareComment = None
            return True
    instInfo = instInfoTarget()
    infoParser = etree.XMLParser(recover=True, huge_tree=True, target=instInfo)
    try:
        etree.parse(_file, parser=infoParser, base_url=filepath)
    except NotInstanceDocumentException:
        pass
    '''
    foundErrors = False
    foundInstance = False
    streamingAspects = None
    creationSoftwareComment = None
    instInfoNumRootFacts = 0
    numElts = 0
    elt = None
    instInfoContext = etree.iterparse(_file,
                                      events=("start", "end"),
                                      huge_tree=True)
    try:
        for event, elt in instInfoContext:
            if event == "start":
                if elt.getparent() is not None:
                    if elt.getparent(
                    ).tag == "{http://www.xbrl.org/2003/instance}xbrl":
                        if not foundInstance:
                            foundInstance = True
                            pi = precedingProcessingInstruction(
                                elt, "xbrl-streamable-instance")
                            if pi is None:
                                break
                            else:
                                streamingAspects = dict(pi.attrib.copy())
                                if creationSoftwareComment is None:
                                    creationSoftwareComment = precedingComment(
                                        elt)
                        if not elt.tag.startswith("{http://www.xbrl.org/"):
                            instInfoNumRootFacts += 1
                            if instInfoNumRootFacts % 1000 == 0:
                                modelXbrl.profileActivity(
                                    "... streaming tree check",
                                    minTimeToShow=20.0)
                    elif not foundInstance:
                        break
                elif elt.tag == "{http://www.xbrl.org/2003/instance}xbrl":
                    creationSoftwareComment = precedingComment(elt)
                    if precedingProcessingInstruction(
                            elt, "xbrl-streamable-instance") is not None:
                        modelXbrl.error(
                            "streamingExtensions:headerMisplaced",
                            _("Header is misplaced: %(error)s, must follow xbrli:xbrl element"
                              ),
                            modelObject=elt)
            elif event == "end":
                elt.clear()
                numElts += 1
                if numElts % 1000 == 0 and elt.getparent() is not None:
                    while elt.getprevious() is not None and elt.getparent(
                    ) is not None:
                        del elt.getparent()[0]
    except etree.XMLSyntaxError as err:
        modelXbrl.error("xmlSchema:syntax",
                        _("Unrecoverable error: %(error)s"),
                        error=err)
        _file.close()
        return err

    _file.seek(0, io.SEEK_SET)  # allow reparsing
    if not foundInstance or streamingAspects is None:
        del elt
        _file.close()
        return None
    modelXbrl.profileStat(_("streaming tree check"), time.time() - startedAt)
    startedAt = time.time()
    try:
        version = Decimal(streamingAspects.get("version"))
        if int(version) != 1:
            modelXbrl.error(
                "streamingExtensions:unsupportedVersion",
                _("Streaming version %(version)s, major version number must be 1"
                  ),
                modelObject=elt,
                version=version)
            foundErrors = True
    except (InvalidOperation, OverflowError):
        modelXbrl.error("streamingExtensions:versionError",
                        _("Version %(version)s, number must be 1.n"),
                        modelObject=elt,
                        version=streamingAspects.get("version", "(none)"))
        foundErrors = True
    for bufAspect in ("contextBuffer", "unitBuffer", "footnoteBuffer"):
        try:
            bufLimit = Decimal(streamingAspects.get(bufAspect, "INF"))
            if bufLimit < 1 or (bufLimit.is_finite() and bufLimit % 1 != 0):
                raise InvalidOperation
            elif bufAspect == "contextBuffer":
                contextBufferLimit = bufLimit
            elif bufAspect == "unitBuffer":
                unitBufferLimit = bufLimit
            elif bufAspect == "footnoteBuffer":
                footnoteBufferLimit = bufLimit
        except InvalidOperation:
            modelXbrl.error(
                "streamingExtensions:valueError",
                _("Streaming %(attrib)s %(value)s, number must be a positive integer or INF"
                  ),
                modelObject=elt,
                attrib=bufAspect,
                value=streamingAspects.get(bufAspect))
            foundErrors = True
    if _streamingExtensionsValidate:
        incompatibleValidations = []
        _validateDisclosureSystem = modelXbrl.modelManager.validateDisclosureSystem
        _disclosureSystem = modelXbrl.modelManager.disclosureSystem
        if _validateDisclosureSystem and _disclosureSystem.validationType == "EFM":
            incompatibleValidations.append("EFM")
        if _validateDisclosureSystem and _disclosureSystem.validationType == "GFM":
            incompatibleValidations.append("GFM")
        if _validateDisclosureSystem and _disclosureSystem.validationType == "HMRC":
            incompatibleValidations.append("HMRC")
        if modelXbrl.modelManager.validateCalcLB:
            incompatibleValidations.append("calculation LB")
        if incompatibleValidations:
            modelXbrl.error(
                "streamingExtensions:incompatibleValidation",
                _("Streaming instance validation does not support %(incompatibleValidations)s validation"
                  ),
                modelObject=modelXbrl,
                incompatibleValidations=', '.join(incompatibleValidations))
            foundErrors = True
    if instInfoContext.error_log:
        foundErrors = True
    logSyntaxErrors(instInfoContext)
    del instInfoContext  # dereference

    for pluginMethod in pluginClassMethods("Streaming.BlockStreaming"):
        _blockingPluginName = pluginMethod(modelXbrl)
        if _blockingPluginName:  # name of blocking plugin is returned
            modelXbrl.error(
                "streamingExtensions:incompatiblePlugIn",
                _("Streaming instance not supported by plugin %(blockingPlugin)s"
                  ),
                modelObject=modelXbrl,
                blockingPlugin=_blockingPluginName)
            foundErrors = True

    if foundErrors:
        _file.close()
        return None

    _encoding = XmlUtil.encoding(_file.read(512))
    _file.seek(0, io.SEEK_SET)  # allow reparsing

    if _streamingExtensionsValidate:
        validator = Validate(modelXbrl)
        instValidator = validator.instValidator

    contextBuffer = []
    contextsToDrop = []
    unitBuffer = []
    unitsToDrop = []
    footnoteBuffer = []
    footnoteLinksToDrop = []

    _streamingFactsPlugin = any(
        True for pluginMethod in pluginClassMethods("Streaming.Facts"))
    _streamingValidateFactsPlugin = (_streamingExtensionsValidate and any(
        True
        for pluginMethod in pluginClassMethods("Streaming.ValidateFacts")))
    ''' this is very much slower than iterparse
    class modelLoaderTarget():
        def __init__(self, element_factory=None, parser=None):
            self.newTree = True
            self.currentMdlObj = None
            self.beforeInstanceStream = True
            self.beforeStartStreamingPlugin = True
            self.numRootFacts = 1
            modelXbrl.makeelementParentModelObject = None
            modelXbrl.isStreamingMode = True
            self.factsCheckVersion = None
            self.factsCheckMd5s = Md5Sum()
        def start(self, tag, attrib, nsmap=None):
            modelXbrl.makeelementParentModelObject = self.currentMdlObj # pass parent to makeelement for ModelObjectFactory
            mdlObj = _parser.makeelement(tag, attrib=attrib, nsmap=nsmap)
            mdlObj.sourceline = 1
            if self.newTree:
                self.newTree = False
                self.currentMdlObj = mdlObj
                modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, mdlObj.getroottree())
                modelXbrl.modelDocument = modelDocument # needed for incremental validation
                mdlObj.init(modelDocument)
                modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement 
                modelDocument.parserLookupName = _parserLookupName
                modelDocument.parserLookupClass = _parserLookupClass
                modelDocument.xmlRootElement = mdlObj
                modelDocument.schemaLocationElements.add(mdlObj)
                modelDocument.documentEncoding = _encoding
                modelDocument._creationSoftwareComment = creationSoftwareComment
                modelXbrl.info("streamingExtensions:streaming",
                               _("Stream processing this instance."),
                               modelObject = modelDocument)
            else:
                self.currentMdlObj.append(mdlObj)
                self.currentMdlObj = mdlObj
                mdlObj._init()
                ns = mdlObj.namespaceURI
                ln = mdlObj.localName
                if (self.beforeInstanceStream and (
                    (ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or
                    (ns == XbrlConst.xbrli and ln in ("context", "unit")) or
                    (ns not in (XbrlConst.link, XbrlConst.xbrli)))):
                    self.beforeInstanceStream = False
                    if _streamingExtensionsValidate:
                        instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters(modelXbrl.prefixedNamespaces))
                    else: # need default dimensions
                        ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
                elif not self.beforeInstanceStream and self.beforeStartStreamingPlugin:
                    for pluginMethod in pluginClassMethods("Streaming.Start"):
                        pluginMethod(modelXbrl)
                    self.beforeStartStreamingPlugin = False
            return mdlObj
        def end(self, tag):
            modelDocument = modelXbrl.modelDocument
            mdlObj = self.currentMdlObj
            parentMdlObj = mdlObj.getparent()
            self.currentMdlObj = parentMdlObj
            ns = mdlObj.namespaceURI
            ln = mdlObj.localName
            if ns == XbrlConst.xbrli:
                if ln == "context":
                    if mdlObj.get("sticky"):
                        del mdlObj.attrib["sticky"]
                        XmlValidate.validate(modelXbrl, mdlObj)
                        modelDocument.contextDiscover(mdlObj)
                    else:
                        if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit:
                            # drop before adding as dropped may have same id as added
                            cntx = contextBuffer.pop(0)
                            if _streamingValidateFactsPlugin:
                                contextsToDrop.append(cntx)
                            else:
                                dropContext(modelXbrl, cntx)
                                del parentMdlObj[parentMdlObj.index(cntx)]
                            cntx = None
                        #>>XmlValidate.validate(modelXbrl, mdlObj)
                        #>>modelDocument.contextDiscover(mdlObj)
                        if contextBufferLimit.is_finite():
                            contextBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        contextsToCheck = (mdlObj,)
                        instValidator.checkContexts(contextsToCheck)
                        if modelXbrl.hasXDT:
                            instValidator.checkContextsDimensions(contextsToCheck)
                        del contextsToCheck # dereference
                elif ln == "unit":
                    if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit:
                        # drop before adding as dropped may have same id as added
                        unit = unitBuffer.pop(0)
                        if _streamingValidateFactsPlugin:
                            unitsToDrop.append(unit)
                        else:
                            dropUnit(modelXbrl, unit)
                            del parentMdlObj[parentMdlObj.index(unit)]
                        unit = None 
                    #>>XmlValidate.validate(modelXbrl, mdlObj)
                    #>>modelDocument.unitDiscover(mdlObj)
                    if unitBufferLimit.is_finite():
                        unitBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        instValidator.checkUnits( (mdlObj,) )
                elif ln == "xbrl": # end of document
                    # check remaining batched facts if any
                    if _streamingValidateFactsPlugin:
                        # plugin attempts to process batch of all root facts not yet processed (not just current one)
                        # finish any final batch of facts
                        if len(modelXbrl.facts) > 0:
                            factsToCheck = modelXbrl.facts.copy()
                            factsHaveBeenProcessed = True
                            # can block facts deletion if required data not yet available, such as numeric unit for DpmDB
                            for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"):
                                if not pluginMethod(modelXbrl, factsToCheck):
                                    factsHaveBeenProcessed = False
                            if factsHaveBeenProcessed:
                                for fact in factsToCheck:
                                    dropFact(modelXbrl, fact, modelXbrl.facts)
                                    del parentMdlObj[parentMdlObj.index(fact)]
                                for cntx in contextsToDrop:
                                    dropContext(modelXbrl, cntx)
                                    del parentMdlObj[parentMdlObj.index(cntx)]
                                for unit in unitsToDrop:
                                    dropUnit(modelXbrl, unit)
                                    del parentMdlObj[parentMdlObj.index(unit)]
                                for footnoteLink in footnoteLinksToDrop:
                                    dropFootnoteLink(modelXbrl, footnoteLink)
                                    del parentMdlObj[parentMdlObj.index(footnoteLink)]
                                fact = cntx = unit = footnoteLink = None
                                del contextsToDrop[:]
                                del unitsToDrop[:]
                                del footnoteLinksToDrop[:]
                            del factsToCheck
                    # check remaining footnote refs
                    for footnoteLink in footnoteBuffer:
                        checkFootnoteHrefs(modelXbrl, footnoteLink)
                    for pluginMethod in pluginClassMethods("Streaming.Finish"):
                        pluginMethod(modelXbrl)
            elif ns == XbrlConst.link:
                if ln == "footnoteLink":
                    XmlValidate.validate(modelXbrl, mdlObj)
                    footnoteLinks = (mdlObj,)
                    modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True)
                    if footnoteBufferLimit.is_finite():
                        footnoteBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        instValidator.checkLinks(footnoteLinks)
                        if len(footnoteBuffer) > footnoteBufferLimit:
                            # check that hrefObjects for locators were all satisfied
                                # drop before addition as dropped may have same id as added
                            footnoteLink = footnoteBuffer.pop(0)
                            checkFootnoteHrefs(modelXbrl, footnoteLink)
                            if _streamingValidateFactsPlugin:
                                footnoteLinksToDrop.append(footnoteLink)
                            else:
                                dropFootnoteLink(modelXbrl, footnoteLink)
                                del parentMdlObj[parentMdlObj.index(footnoteLink)]
                            footnoteLink = None
                    footnoteLinks = None
                elif ln in ("schemaRef", "linkbaseRef"):
                    modelDocument.discoverHref(mdlObj)
                elif not modelXbrl.skipDTS:
                    if ln in ("roleRef", "arcroleRef"):
                        modelDocument.linkbaseDiscover((mdlObj,), inInstance=True)
            elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl:
                self.numRootFacts += 1
                #>>XmlValidate.validate(modelXbrl, mdlObj)
                #>>modelDocument.factDiscover(mdlObj, modelXbrl.facts)
                if self.factsCheckVersion:
                    self.factCheckFact(mdlObj)
                if _streamingExtensionsValidate or _streamingValidateFactsPlugin:
                    factsToCheck = (mdlObj,)  # validate current fact by itself
                    if _streamingExtensionsValidate:
                        instValidator.checkFacts(factsToCheck)
                        if modelXbrl.hasXDT:
                            instValidator.checkFactsDimensions(factsToCheck)
                    if _streamingValidateFactsPlugin:
                        # plugin attempts to process batch of all root facts not yet processed (not just current one)
                        # use batches of 1000 facts
                        if len(modelXbrl.facts) > 1000:
                            factsToCheck = modelXbrl.facts.copy()
                            factsHaveBeenProcessed = True
                            # can block facts deletion if required data not yet available, such as numeric unit for DpmDB
                            for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"):
                                if not pluginMethod(modelXbrl, factsToCheck):
                                    factsHaveBeenProcessed = False
                            if factsHaveBeenProcessed:
                                for fact in factsToCheck:
                                    dropFact(modelXbrl, fact, modelXbrl.facts)
                                    del parentMdlObj[parentMdlObj.index(fact)]
                                for cntx in contextsToDrop:
                                    dropContext(modelXbrl, cntx)
                                    del parentMdlObj[parentMdlObj.index(cntx)]
                                for unit in unitsToDrop:
                                    dropUnit(modelXbrl, unit)
                                    del parentMdlObj[parentMdlObj.index(unit)]
                                for footnoteLink in footnoteLinksToDrop:
                                    dropFootnoteLink(modelXbrl, footnoteLink)
                                    del parentMdlObj[parentMdlObj.index(footnoteLink)]
                                fact = cntx = unit = footnoteLink = None
                                del contextsToDrop[:]
                                del unitsToDrop[:]
                                del footnoteLinksToDrop[:]
                            del factsToCheck # dereference fact or batch of facts
                    else:
                        dropFact(modelXbrl, mdlObj, modelXbrl.facts) # single fact has been processed
                        del parentMdlObj[parentMdlObj.index(mdlObj)]
                if self.numRootFacts % 1000 == 0:
                    pass
                    #modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts, 
                    #                                                                          100.0 * self.numRootFacts / instInfoNumRootFacts), 
                    #                          minTimeToShow=20.0)
                    gc.collect()
                    sys.stdout.write ("\rAt fact {} of {} mem {}".format(self.numRootFacts, instInfoNumRootFacts, modelXbrl.modelManager.cntlr.memoryUsed))
            return mdlObj
        def data(self, data):
            self.currentMdlObj.text = data
        def comment(self, text):
            pass
        def pi(self, target, data):
            if target == "xbrl-facts-check":
                _match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", data)
                if _match:
                    _matchGroups = _match.groups()
                    if len(_matchGroups) == 2:
                        if _matchGroups[0] == "version":
                            self.factsCheckVersion = _matchGroups[1]
                        elif _matchGroups[0] == "sum-of-fact-md5s":
                            try:
                                expectedMd5 = Md5Sum(_matchGroups[1])
                                if self.factsCheckMd5s != expectedMd5:
                                    modelXbrl.warning("streamingExtensions:xbrlFactsCheckWarning",
                                            _("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"),
                                            modelObject=modelXbrl, expectedMd5=expectedMd5, actualMd5Sum=self.factsCheckMd5s)
                                else:
                                    modelXbrl.info("info",
                                            _("Successful XBRL facts sum of md5s."),
                                            modelObject=modelXbrl)
                            except ValueError:
                                modelXbrl.error("streamingExtensions:xbrlFactsCheckError",
                                        _("Invalid sum-of-md5s %(sumOfMd5)s"),
                                        modelObject=modelXbrl, sumOfMd5=_matchGroups[1])
        def close(self):
            del modelXbrl.makeelementParentModelObject
            return None
        
        def factCheckFact(self, fact):
            self.factsCheckMd5s += fact.md5sum
            for _tupleFact in fact.modelTupleFacts:
                self.factCheckFact(_tupleFact)
        
    _parser, _parserLookupName, _parserLookupClass = parser(modelXbrl, filepath, target=modelLoaderTarget())
    etree.parse(_file, parser=_parser, base_url=filepath)
    logSyntaxErrors(_parser)
    '''
    # replace modelLoaderTarget with iterparse (as it now supports CustomElementClassLookup)
    streamingParserContext = etree.iterparse(_file,
                                             events=("start", "end"),
                                             huge_tree=True)
    from arelle.ModelObjectFactory import setParserElementClassLookup
    modelXbrl.isStreamingMode = True  # must be set before setting element class lookup
    (_parser, _parserLookupName,
     _parserLookupClass) = setParserElementClassLookup(streamingParserContext,
                                                       modelXbrl)
    foundInstance = False
    beforeInstanceStream = beforeStartStreamingPlugin = True
    numRootFacts = 0
    factsCheckVersion = None

    def factCheckFact(fact):
        modelDocument._factsCheckMd5s += fact.md5sum
        for _tupleFact in fact.modelTupleFacts:
            factCheckFact(_tupleFact)

    for event, mdlObj in streamingParserContext:
        if event == "start":
            if mdlObj.tag == "{http://www.xbrl.org/2003/instance}xbrl":
                modelDocument = ModelDocument(modelXbrl, Type.INSTANCE,
                                              mappedUri, filepath,
                                              mdlObj.getroottree())
                modelXbrl.modelDocument = modelDocument  # needed for incremental validation
                mdlObj.init(modelDocument)
                modelDocument.parser = _parser  # needed for XmlUtil addChild's makeelement
                modelDocument.parserLookupName = _parserLookupName
                modelDocument.parserLookupClass = _parserLookupClass
                modelDocument.xmlRootElement = mdlObj
                modelDocument.schemaLocationElements.add(mdlObj)
                modelDocument.documentEncoding = _encoding
                modelDocument._creationSoftwareComment = precedingComment(
                    mdlObj)
                modelDocument._factsCheckMd5s = Md5Sum()
                modelXbrl.info("streamingExtensions:streaming",
                               _("Stream processing this instance."),
                               modelObject=modelDocument)
            elif mdlObj.getparent() is not None:
                mdlObj._init()  # requires discovery as part of start elements
                if mdlObj.getparent(
                ).tag == "{http://www.xbrl.org/2003/instance}xbrl":
                    if not foundInstance:
                        foundInstance = True
                        pi = precedingProcessingInstruction(
                            mdlObj, "xbrl-facts-check")
                        if pi is not None:
                            factsCheckVersion = pi.attrib.get("version", None)
                elif not foundInstance:
                    break
                ns = mdlObj.qname.namespaceURI
                ln = mdlObj.qname.localName
                if beforeInstanceStream:
                    if ((ns == XbrlConst.link
                         and ln not in ("schemaRef", "linkbaseRef")) or
                        (ns == XbrlConst.xbrli and ln in ("context", "unit"))
                            or (ns not in (XbrlConst.link, XbrlConst.xbrli))):
                        beforeInstanceStream = False
                        if _streamingExtensionsValidate:
                            instValidator.validate(
                                modelXbrl,
                                modelXbrl.modelManager.formulaOptions.
                                typedParameters(modelXbrl.prefixedNamespaces))
                        else:  # need default dimensions
                            ValidateXbrlDimensions.loadDimensionDefaults(
                                modelXbrl)
                elif not beforeInstanceStream and beforeStartStreamingPlugin:
                    for pluginMethod in pluginClassMethods("Streaming.Start"):
                        pluginMethod(modelXbrl)
                    beforeStartStreamingPlugin = False
        elif event == "end":
            parentMdlObj = mdlObj.getparent()
            ns = mdlObj.namespaceURI
            ln = mdlObj.localName
            if ns == XbrlConst.xbrli:
                if ln == "context":
                    if mdlObj.get("sticky"):
                        del mdlObj.attrib["sticky"]
                        XmlValidate.validate(modelXbrl, mdlObj)
                        modelDocument.contextDiscover(mdlObj)
                    else:
                        if len(contextBuffer) >= contextBufferLimit:
                            # drop before adding as dropped may have same id as added
                            cntx = contextBuffer.pop(0)
                            if _streamingFactsPlugin or _streamingValidateFactsPlugin:
                                contextsToDrop.append(cntx)
                            else:
                                dropContext(modelXbrl, cntx)
                                #>>del parentMdlObj[parentMdlObj.index(cntx)]
                            cntx = None
                        XmlValidate.validate(modelXbrl, mdlObj)
                        modelDocument.contextDiscover(mdlObj)
                        if contextBufferLimit.is_finite():
                            contextBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        contextsToCheck = (mdlObj, )
                        instValidator.checkContexts(contextsToCheck)
                        if modelXbrl.hasXDT:
                            instValidator.checkContextsDimensions(
                                contextsToCheck)
                        del contextsToCheck  # dereference
                elif ln == "unit":
                    if len(unitBuffer) >= unitBufferLimit:
                        # drop before additing as dropped may have same id as added
                        unit = unitBuffer.pop(0)
                        if _streamingFactsPlugin or _streamingValidateFactsPlugin:
                            unitsToDrop.append(unit)
                        else:
                            dropUnit(modelXbrl, unit)
                            #>>del parentMdlObj[parentMdlObj.index(unit)]
                        unit = None
                    XmlValidate.validate(modelXbrl, mdlObj)
                    modelDocument.unitDiscover(mdlObj)
                    if unitBufferLimit.is_finite():
                        unitBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        instValidator.checkUnits((mdlObj, ))
                elif ln == "xbrl":  # end of document
                    # check remaining batched facts if any
                    if _streamingFactsPlugin or _streamingValidateFactsPlugin:
                        # plugin attempts to process batch of all root facts not yet processed (not just current one)
                        # finish any final batch of facts
                        if len(modelXbrl.facts) > 0:
                            factsToCheck = modelXbrl.facts.copy()
                            # can block facts deletion if required data not yet available, such as numeric unit for DpmDB
                            if _streamingValidateFactsPlugin:
                                for pluginMethod in pluginClassMethods(
                                        "Streaming.ValidateFacts"):
                                    pluginMethod(instValidator, factsToCheck)
                            if _streamingFactsPlugin:
                                for pluginMethod in pluginClassMethods(
                                        "Streaming.Facts"):
                                    pluginMethod(modelXbrl, factsToCheck)
                            for fact in factsToCheck:
                                dropFact(modelXbrl, fact, modelXbrl.facts)
                                #>>del parentMdlObj[parentMdlObj.index(fact)]
                            for cntx in contextsToDrop:
                                dropContext(modelXbrl, cntx)
                                #>>del parentMdlObj[parentMdlObj.index(cntx)]
                            for unit in unitsToDrop:
                                dropUnit(modelXbrl, unit)
                                #>>del parentMdlObj[parentMdlObj.index(unit)]
                            for footnoteLink in footnoteLinksToDrop:
                                dropFootnoteLink(modelXbrl, footnoteLink)
                                #>>del parentMdlObj[parentMdlObj.index(footnoteLink)]
                            fact = cntx = unit = footnoteLink = None
                            del contextsToDrop[:]
                            del unitsToDrop[:]
                            del footnoteLinksToDrop[:]
                            del factsToCheck
                    # check remaining footnote refs
                    for footnoteLink in footnoteBuffer:
                        checkFootnoteHrefs(modelXbrl, footnoteLink)
                    pi = childProcessingInstruction(mdlObj,
                                                    "xbrl-facts-check",
                                                    reversed=True)
                    if pi is not None:  # attrib is in .text, not attrib, no idea why!!!
                        _match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']",
                                           pi.text)
                        if _match:
                            _matchGroups = _match.groups()
                            if len(_matchGroups) == 2:
                                if _matchGroups[0] == "sum-of-fact-md5s":
                                    try:
                                        expectedMd5 = Md5Sum(_matchGroups[1])
                                        if modelDocument._factsCheckMd5s != expectedMd5:
                                            modelXbrl.warning(
                                                "streamingExtensions:xbrlFactsCheckWarning",
                                                _("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"
                                                  ),
                                                modelObject=modelXbrl,
                                                expectedMd5=expectedMd5,
                                                actualMd5Sum=modelDocument.
                                                _factsCheckMd5s)
                                        else:
                                            modelXbrl.info(
                                                "info",
                                                _("Successful XBRL facts sum of md5s."
                                                  ),
                                                modelObject=modelXbrl)
                                    except ValueError:
                                        modelXbrl.error(
                                            "streamingExtensions:xbrlFactsCheckError",
                                            _("Invalid sum-of-md5s %(sumOfMd5)s"
                                              ),
                                            modelObject=modelXbrl,
                                            sumOfMd5=_matchGroups[1])
                    if _streamingValidateFactsPlugin:
                        for pluginMethod in pluginClassMethods(
                                "Streaming.ValidateFinish"):
                            pluginMethod(instValidator)
                    if _streamingFactsPlugin:
                        for pluginMethod in pluginClassMethods(
                                "Streaming.Finish"):
                            pluginMethod(modelXbrl)
            elif ns == XbrlConst.link:
                if ln in ("schemaRef", "linkbaseRef"):
                    modelDocument.discoverHref(
                        mdlObj,
                        urlRewritePluginClass=
                        "ModelDocument.InstanceSchemaRefRewriter")
                elif ln in ("roleRef", "arcroleRef"):
                    modelDocument.linkbaseDiscover((mdlObj, ), inInstance=True)
                elif ln == "footnoteLink":
                    XmlValidate.validate(modelXbrl, mdlObj)
                    footnoteLinks = (mdlObj, )
                    modelDocument.linkbaseDiscover(footnoteLinks,
                                                   inInstance=True)
                    if footnoteBufferLimit.is_finite():
                        footnoteBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        instValidator.checkLinks(footnoteLinks)
                        if len(footnoteBuffer) > footnoteBufferLimit:
                            # check that hrefObjects for locators were all satisfied
                            # drop before addition as dropped may have same id as added
                            footnoteLink = footnoteBuffer.pop(0)
                            checkFootnoteHrefs(modelXbrl, footnoteLink)
                            if _streamingValidateFactsPlugin:
                                footnoteLinksToDrop.append(footnoteLink)
                            else:
                                dropFootnoteLink(modelXbrl, footnoteLink)
                                #>>del parentMdlObj[parentMdlObj.index(footnoteLink)]
                            footnoteLink = None
                    footnoteLinks = None
            elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl and isinstance(
                    mdlObj, ModelFact):
                numRootFacts += 1
                XmlValidate.validate(modelXbrl, mdlObj)
                modelDocument.factDiscover(mdlObj, modelXbrl.facts)
                if factsCheckVersion:
                    factCheckFact(mdlObj)
                if _streamingExtensionsValidate or _streamingFactsPlugin or _streamingValidateFactsPlugin:
                    factsToCheck = (mdlObj,
                                    )  # validate current fact by itself
                    if _streamingExtensionsValidate:
                        instValidator.checkFacts(factsToCheck)
                        if modelXbrl.hasXDT:
                            instValidator.checkFactsDimensions(factsToCheck)
                    if _streamingFactsPlugin or _streamingValidateFactsPlugin:
                        # plugin attempts to process batch of all root facts not yet processed (not just current one)
                        # use batches of 1000 facts
                        if len(modelXbrl.facts) > 1000:
                            factsToCheck = modelXbrl.facts.copy()
                            # can block facts deletion if required data not yet available, such as numeric unit for DpmDB
                            if _streamingValidateFactsPlugin:
                                for pluginMethod in pluginClassMethods(
                                        "Streaming.ValidateFacts"):
                                    pluginMethod(instValidator, factsToCheck)
                            if _streamingFactsPlugin:
                                for pluginMethod in pluginClassMethods(
                                        "Streaming.Facts"):
                                    pluginMethod(modelXbrl, factsToCheck)
                            for fact in factsToCheck:
                                dropFact(modelXbrl, fact, modelXbrl.facts)
                                #>>del parentMdlObj[parentMdlObj.index(fact)]
                            for cntx in contextsToDrop:
                                dropContext(modelXbrl, cntx)
                                #>>del parentMdlObj[parentMdlObj.index(cntx)]
                            for unit in unitsToDrop:
                                dropUnit(modelXbrl, unit)
                                #>>del parentMdlObj[parentMdlObj.index(unit)]
                            for footnoteLink in footnoteLinksToDrop:
                                dropFootnoteLink(modelXbrl, footnoteLink)
                                #>>del parentMdlObj[parentMdlObj.index(footnoteLink)]
                            fact = cntx = unit = footnoteLink = None
                            del contextsToDrop[:]
                            del unitsToDrop[:]
                            del footnoteLinksToDrop[:]
                            del factsToCheck  # dereference fact or batch of facts
                    else:
                        dropFact(
                            modelXbrl, mdlObj,
                            modelXbrl.facts)  # single fact has been processed
                        #>>del parentMdlObj[parentMdlObj.index(mdlObj)]
                if numRootFacts % 1000 == 0:
                    pass
                    #modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts,
                    #                                                                          100.0 * self.numRootFacts / instInfoNumRootFacts),
                    #                          minTimeToShow=20.0)
                    #gc.collect()
                    #sys.stdout.write ("\rAt fact {} of {} mem {}".format(numRootFacts, instInfoNumRootFacts, modelXbrl.modelManager.cntlr.memoryUsed))
    if mdlObj is not None:
        mdlObj.clear()
    del _parser, _parserLookupName, _parserLookupClass

    if _streamingExtensionsValidate and validator is not None:
        _file.close()
        del instValidator
        validator.close()
        # track that modelXbrl has been validated by this streaming extension
        modelXbrl._streamingExtensionValidated = True

    modelXbrl.profileStat(_("streaming complete"), time.time() - startedAt)
    return modelXbrl.modelDocument
Exemplo n.º 25
0
    def patch(self, request, *args, **kwargs):  # pylint: disable=unused-argument
        """
        Take PATCH requests, check permissions, and alter state based on them.
        """
        uuid = self.kwargs['uuid']
        data = request.data
        user = request.user
        # We shouldn't save any data until all validation checks out
        course_modified = False
        course = AuthorizationHelpers.get_course(uuid, user)
        if course is None:
            raise Http404

        if "live" in data:
            live = data['live']
            if not isinstance(live, bool):
                raise ValidationError("live must be a bool")
            if not AuthorizationHelpers.can_edit_own_liveness(course, user):
                raise ValidationError(
                    "User doesn't have permission to edit course liveness")

            course.live = live
            course_modified = True

        if "title" in data:
            title = data['title']
            if not isinstance(title, string_types) or title == '':
                raise ValidationError("title must be a non-empty string")
            if not AuthorizationHelpers.can_edit_own_content(course, user):
                raise ValidationError(
                    "User doesn't have permission to edit course descriptions or titles"
                )

            course.title = title
            course_modified = True

        if "description" in data:
            description = data['description']
            if not isinstance(description, string_types):
                raise ValidationError("description must be a string")
            if not AuthorizationHelpers.can_edit_own_content(course, user):
                raise ValidationError(
                    "User doesn't have permission to edit course descriptions or titles"
                )

            course.description = description
            course_modified = True

        modules_to_save = {}
        if "modules" in data:
            if not isinstance(data['modules'], list):
                raise ValidationError("modules must be a list of modules")

            for module_data in data['modules']:
                if not isinstance(module_data, dict):
                    raise ValidationError("Each module must be an object")
                if 'uuid' not in module_data:
                    raise ValidationError("Missing key uuid")
                if module_data['uuid'] in modules_to_save:
                    raise ValidationError("Duplicate module")

                try:
                    # We've already checked permissions for course above
                    # so this is a simple object lookup
                    module = Module.objects.get(uuid=module_data['uuid'],
                                                course=course)
                except Module.DoesNotExist:
                    raise ValidationError(
                        "Unable to find module {uuid}".format(
                            uuid=module_data['uuid']))

                if 'title' in module_data:
                    title = module_data['title']
                    if not isinstance(title, string_types) or title == '':
                        raise ValidationError(
                            "title must be a non-empty string")
                    if not AuthorizationHelpers.can_edit_own_content(
                            course, user):
                        raise ValidationError(
                            "User doesn't have permission to edit course descriptions or titles"
                        )

                    module.title = title
                    modules_to_save[module.uuid] = module

                if 'price_without_tax' in module_data:
                    price_without_tax = module_data['price_without_tax']
                    if price_without_tax is not None:
                        if not isinstance(price_without_tax, string_types):
                            raise ValidationError(
                                'price_without_tax must be a string')
                        try:
                            price_without_tax = Decimal(price_without_tax)
                        except DecimalException:
                            raise ValidationError(
                                'price_without_tax is not a valid number')
                        if not price_without_tax.is_finite():
                            raise ValidationError(
                                'price_without_tax is not a valid number')
                        if price_without_tax < 0:
                            raise ValidationError(
                                'price_without_tax is not a valid number')

                    if not AuthorizationHelpers.can_edit_own_price(
                            course, user):
                        raise ValidationError(
                            "User doesn't have permission to edit module price"
                        )

                    module.price_without_tax = price_without_tax
                    modules_to_save[module.uuid] = module

        if course_modified:
            course.save()
        for module in modules_to_save.values():
            module.save()
        return Response(status=200)
Exemplo n.º 26
0
class ComplexDecimal(object):
    def __init__(self,real,i="0.0"):
        if type(real) != Decimal: self.real = Decimal(real)
        else: self.real = real
        if type(i) != Decimal: self.imaginary = Decimal(i)
        else: self.imaginary = i

        self.e = constants.e
        self.pi = constants.pi
        
    #Basic mathamtical functions ----------------------------------------
    def conj(self):
        """Returns the conjugate for a number"""
        return ComplexDecimal(self.real,-self.imaginary)
        
    def re(self):
        """Returns real portion"""
        return self.real
        
    def im(self):
        """Returns imaginary portion"""
        return self.imaginary
        
    def floor(self):
        """Floors a number"""
        return ComplexDecimal(self.real.to_integral(), self.imaginary.to_integral())
        
    def ceil(self):
        """Ceilings a number"""
        return ComplexDecimal(math.ceil(self.real),math.ceil(self.imaginary))
        
    def round(self,prec=0):
        """Rounds a number to prec decimal places"""
        return ComplexDecimal(round(self.real,prec),round(self.imaginary,prec))
        
    def exp(self):
        """Returns e^self"""
        if self.imaginary == 0:
            return ComplexDecimal(str(self.e ** self.real ))
        return ComplexDecimal(self.e**(self.real) * Decimal(self.cos_raw(self.imaginary)), self.e**(self.real) * Decimal(self.sin_raw(self.imaginary)))
        
    def isPrime_raw(self,x):
        if x<=1: return False
        for i in range(2,int(math.floor(math.sqrt(x))) + 1):
            if x%i==0: return False
        return True
        
    def isPrime(self):
        """Gets if self is prime"""
        if self.imaginary == 0: 
            return self.isPrime_raw(self.real)
        if self.real == 0:
            return self.isPrime_raw(abs(self.imaginary))
        return self.isPrime_raw(self.real * self.real + self.imaginary*self.imaginary)
    
    def phase(self):
        """Returns angle between x axis"""
        return Decimal(math.atan2(self.imaginary , self.real))
        
    def polar(self): #Converts to polar
        return (abs(self), self.phase())
        
    def toRect(self,r,phi):
        return ComplexDecimal(r) * ComplexDecimal(self.cos_raw(phi) , self.sin_raw(phi))
        
        
    #Trig --------------------------------------------
    #1/functions, log, 
    
    def cos_raw(self,x,prec=Decimal("0.0000000000000000000001")):
        """Raw cosine function for real numbers only"""
        p = Decimal("0")
        s = t = Decimal("1.0")
        while abs(t/s) > prec:
            p+=Decimal("1")
            t = (-t * x * x) / ((Decimal("2") * p - Decimal("1")) * (Decimal("2") * p))
            s += t
        if round(s,50) in [1,0,-1]: #If it's close enough round to get perfect answers (No one likes 1E-70 as cos(pi))
            return Decimal(round(s,50))
        return s
    
    def sin_raw(self,x,prec=Decimal("0.00000000000000000000001")):
        return self.cos_raw(x+self.pi/Decimal("2"))
        
    def tan_raw(self,x,prec=Decimal("0.00000000000000000000001")):
        return self.sin_raw(x,prec) / self.cos_raw(x,prec)
        
    def atan_raw(self,x): #Not relatively accurate
        returned = Decimal("0")
        for j in range(0,50):
            i = Decimal(j)
            a = Decimal("-1")**i * x ** (Decimal("2")*i + Decimal("1"))
            b = Decimal("2")*i + Decimal("1")
            returned += a/b
        return returned
    
    def cos(self):
        """Compute cos of self"""
        if self.imaginary == 0:
            return ComplexDecimal(self.cos_raw(self.real))
        try:
            returned = ComplexDecimal(
                self.cos_raw(self.real) * ComplexDecimal(self.imaginary).cosh().real,
                -self.sin_raw(self.real) * ComplexDecimal(self.imaginary).sinh().real
            )
            returned.imaginary = -returned.imaginary
            return returned
        except: return ComplexDecimal("inf")
        
    def sin(self):
        """Compute sin of self"""
        if self.imaginary == 0:
            return ComplexDecimal(self.sin_raw(self.real))
        try: 
            returned = ComplexDecimal(
                self.sin_raw(self.real) * ComplexDecimal(self.imaginary).cosh().real,
                -self.cos_raw(self.real) * ComplexDecimal(self.imaginary).sinh().real
            )
            returned.imaginary = -returned.imaginary
            return returned
        except: return ComplexDecimal("inf")
        
    def tan(self):
        """Compute tangent of self"""
        try: return self.sin() / self.cos()
        except: return ComplexDecimal("inf")
        
    def acos(self):
        """Attempt to compute arcosine"""
        if self.imaginary == 0:
            return ComplexDecimal(math.acos(self))
        A = ComplexDecimal(((Decimal(1) + self.real)**Decimal(2) + self.imaginary**Decimal(2))**Decimal(0.5) - ((Decimal(1) - self.real)**Decimal(0.5) + self.imaginary**Decimal(2))**Decimal(0.5)) / ComplexDecimal(2)
        B = ComplexDecimal(((Decimal(1) + self.real)**Decimal(2) + self.imaginary**Decimal(2))**Decimal(0.5) + ((Decimal(1) - self.real)**Decimal(0.5) + self.imaginary**Decimal(2))**Decimal(0.5)) / ComplexDecimal(2)
        return ComplexDecimal(A.acos(), -(B+(B*B - ComplexDecimal(1))**ComplexDecimal(0.5)).ln() )
        
    def asin(self):
        """Attempt to compute arcsine"""
        if self.imaginary == 0:
            return ComplexDecimal(math.asin(self))
        A = ComplexDecimal(((Decimal(1) + self.real)**Decimal(2) + self.imaginary**Decimal(2))**Decimal(0.5) - ((Decimal(1) - self.real)**Decimal(0.5) + self.imaginary**Decimal(2))**Decimal(0.5)) / ComplexDecimal(2)
        B = ComplexDecimal(((Decimal(1) + self.real)**Decimal(2) + self.imaginary**Decimal(2))**Decimal(0.5) + ((Decimal(1) - self.real)**Decimal(0.5) + self.imaginary**Decimal(2))**Decimal(0.5)) / ComplexDecimal(2)
        return ComplexDecimal(A.asin(), (B+(B*B - ComplexDecimal(1))**ComplexDecimal(0.5)).ln() )
    
    def atan(self):
        """Attempt to compute arctangent"""
        result = (ComplexDecimal(0,1)+self) / (ComplexDecimal(0,1) - self)
        return ComplexDecimal(0,0.5) * result.ln()
        
        
    def sinh(self):
        """Hyperbolic sine of self"""
        returned = (self.exp() - (-self).exp()) / ComplexDecimal("2")
        returned.imaginary = -returned.imaginary
        return returned
        
    def cosh(self):
        """Hyperbolic cosine of self"""
        returned = (self.exp() + (-self).exp()) / ComplexDecimal("2")
        returned.imaginary = -returned.imaginary
        return returned
        
    def tanh(self):
        """Hyperbolic tangent of self"""
        return self.sinh()/self.cosh()
        
    def acosh(self):
        """Arc hyperbolic cosine"""
        returned = self + (self*self - ComplexDecimal(1))**0.5
        return returned.ln()
        
    def asinh(self):
        """Arc hyperbolic sine"""
        returned = self + (self*self + ComplexDecimal(1))**0.5
        return returned.ln()
        
    def atanh(self):
        """Arc hyperbolic  tan"""
        a = (ComplexDecimal(1)+self).ln() - (ComplexDecimal(1)-self).ln()
        return ComplexDecimal(0.5) * a
        
    def ln_raw(self,x):
        returned = Decimal("0")
        for i in range(0,25):
            a = Decimal("1") / (Decimal(i) * Decimal("2") + Decimal("1"))
            b = ( (x - Decimal("1"))/(x + Decimal("1")) ) ** (Decimal(i) * Decimal("2") + Decimal("1"))
            returned += a*b
        return returned*2
    
    def ln(self):
        """Natural logarithim of self"""
        if self.imaginary == 0:
            try: return ComplexDecimal(self.ln_raw(self.real))
            except: pass
        p = self.polar()
        return ComplexDecimal(self.ln_raw(p[0]), p[1])
    
    def log(self,x=None):
        """Compute log of number in base x"""
        if not x: x = self.e
        if self.imaginary == 0:
            try: return ComplexDecimal(math.log(self.real,x))
            except: pass
        p = self.polar()
        return ComplexDecimal(math.log(p[0],x), p[1])
        
    def log10(self):
        """Compute log base 10"""
        if self.imaginary == 0:
            return ComplexDecimal(math.log10(self.real))
        return self.ln() / ComplexDecimal("10").ln()
        
    #Replace builtins --------------------------------
    # power, comparasion,
    #To float, int, complex
    
    def __str__(self):
        """Converts to str"""
        if self.imaginary == 0:
            return str(self.real)
        return (str(self.real)+"+"+str(self.imaginary)+"i").replace("+-","-")
        
    def __abs__(self):
        """Absolute value"""
        return (self.real**2 + self.imaginary**2).sqrt()
    
    def __add__(self,other):
        """Add 2 numbers together"""
        return ComplexDecimal(self.real+other.real, self.imaginary+other.imaginary)
    
    def __sub__(self,other):
        """Subtract 2 numbers"""
        return ComplexDecimal(self.real-other.real, self.imaginary-other.imaginary)
    
    def __mul__(self,other):
        """Multiply 2 numbers"""
        if self.imaginary == 0 and other.imaginary == 0:
            return ComplexDecimal(self.real*other.real,"0")
        return ComplexDecimal( self.real*other.real - self.imaginary*other.imaginary,self.real*other.imaginary + self.imaginary*other.real )
   
    def __div__(self,other):
        """Divide 2 numbers"""
        if self.imaginary == 0 and other.imaginary == 0:
            return ComplexDecimal(self.real/other.real,"0")
        a = (self.real*other.real + self.imaginary*other.imaginary)/(other.real*other.real + other.imaginary*other.imaginary)
        b = (self.imaginary*other.real - self.real*other.imaginary)/(other.real*other.real + other.imaginary*other.imaginary)
        return ComplexDecimal(a,b)
    
    def __truediv__(self,other):
        """Divide 2 numbers"""
        if self.imaginary == 0 and other.imaginary == 0:
            return ComplexDecimal(self.real/other.real,"0")
        a = (self.real*other.real + self.imaginary*other.imaginary)/(other.real*other.real + other.imaginary*other.imaginary)
        b = (self.imaginary*other.real - self.real*other.imaginary)/(other.real*other.real + other.imaginary*other.imaginary)
        return ComplexDecimal(a,b)
    
    def __neg__(self):
        """Negates a number"""
        return ComplexDecimal(-self.real,-self.imaginary)
    
    def __pos__(self):
        """Positive number"""
        return ComplexDecimal(abs(self.real),abs(self.imaginary))
    
    def __inverse__(self):
        """1/ number"""
        return ComplexDecimal(1)/self
    
    def __mod__(self,other):#a%b = a + b * ciel(-a/b) 
        """Modolus"""
        return self+ other* ((-self/other).ceil())
        
    #Powers
    def __pow__(self,other):
        """Powers :D"""
        if self.imaginary == 0 and other.imaginary == 0:
            return ComplexDecimal(self.real**other.real)
        if other.imaginary == 0:
            polar = self.polar()
            return self.toRect(polar[0]**other.real, polar[1]*other.real)
        elif other.real == 0:
            a = ComplexDecimal(self.real); b = ComplexDecimal(self.imaginary)
            c = ComplexDecimal(other.real); d = ComplexDecimal(other.imaginary)
            x = (-d * (b/a).atan()).exp() * ((a*a+b*b).ln() * d / ComplexDecimal(2)).cos() 
            y = ComplexDecimal(0,1) * (-d * (b/a).atan()).exp() * ((a*a+b*b).ln() * d / ComplexDecimal(2)).sin()
            return x+y

        b = other.real; c = other.imaginary
        returned = self**(ComplexDecimal(b)) * self**(ComplexDecimal(0,1) * ComplexDecimal(c))
        returned.imaginary = -returned.imaginary
        return returned
    
    #Additional conversions
    def __complex__(self): 
        """Convert to complex"""
        return complex(float(self.real),float(self.imaginary))
    def __int__(self):
        """Convert to int"""
        return int(self.real)
    def __float__(self):
        """Convert to float"""
        return float(self.real)
        
    #Comparasions
    def __lt__(self,other): #>
        if self.imaginary != 0 or other.imaginary != 0:
            raise errors.ComparasionError("Complex comparasion is not supported")
        return self.real < other.real
        
    def __le__(self,other): #>=
        if self.imaginary != 0 or other.imaginary != 0:
            raise errors.ComparasionError("Complex comparasion is not supported")
        return self.real <= other.real
        
    def __eq__(self,other): #==
        if self.real == other.real and self.imaginary == other.imaginary:
            return True
        return False
        
    def __ne__(self,other): #!=
        return not self.__eq__(other)
        
    def __gt__(self,other): #<
        if self.imaginary != 0 or other.imaginary != 0:
            raise errors.ComparasionError("Complex comparasion is not supported")
        return self.real > other.real
        
    def __ge__(self,other): #<=
        if self.imaginary != 0 or other.imaginary != 0:
            raise errors.ComparasionError("Complex comparasion is not supported")
        return self.real >= other.real
        
    #Some things reimplemented from decimal class for complex ComplexDecimals
    #================================================================
    def copy_abs(self): return abs(self)
    def copy_negate(self): return -self
    
    def copy_sign(self,other):
        if other < 0:
            return -abs(self)
        return abs(self)
        
    def is_finite(self):
        return self.real.is_finite() and self.imaginary.is_finite()
    
    def is_infinite(self):
        return not self.is_finite()
    
    def is_nan(self):
        return self.real.is_nan() or self.imaginary.is_nan()
        
    def is_signed(self):
        return self < ComplexDecimal(0)
    
    def is_zero(self):
        return self.real.is_zero() and self.imaginary.is_zero()
    
    def radix(self): return Decimal(10)
    def sqrt(self): 
        if self.imaginary.is_zero():
            return ComplexDecimal(self.real.sqrt())
        return self ** ComplexDecimal("0.5")
        
    def to_eng_string(self): 
        returned = self.real.to_eng_string() + " + " + self.imaginary.to_eng_string + "i"
        return returned.replace(" + -"," - ")
Exemplo n.º 27
0
class ComplexDecimal(object):
    def __init__(self, real, i="0.0"):
        if type(real) != Decimal: self.real = Decimal(real)
        else: self.real = real
        if type(i) != Decimal: self.imaginary = Decimal(i)
        else: self.imaginary = i

        self.e = constants.e
        self.pi = constants.pi

    #Basic mathamtical functions ----------------------------------------
    def conj(self):
        """Returns the conjugate for a number"""
        return ComplexDecimal(self.real, -self.imaginary)

    def re(self):
        """Returns real portion"""
        return self.real

    def im(self):
        """Returns imaginary portion"""
        return self.imaginary

    def floor(self):
        """Floors a number"""
        return ComplexDecimal(self.real.to_integral(),
                              self.imaginary.to_integral())

    def ceil(self):
        """Ceilings a number"""
        return ComplexDecimal(math.ceil(self.real), math.ceil(self.imaginary))

    def round(self, prec=0):
        """Rounds a number to prec decimal places"""
        return ComplexDecimal(round(self.real, prec),
                              round(self.imaginary, prec))

    def exp(self):
        """Returns e^self"""
        if self.imaginary == 0:
            return ComplexDecimal(str(self.e**self.real))
        return ComplexDecimal(
            self.e**(self.real) * Decimal(self.cos_raw(self.imaginary)),
            self.e**(self.real) * Decimal(self.sin_raw(self.imaginary)))

    def isPrime_raw(self, x):
        if x <= 1: return False
        for i in range(2, int(math.floor(math.sqrt(x))) + 1):
            if x % i == 0: return False
        return True

    def isPrime(self):
        """Gets if self is prime"""
        if self.imaginary == 0:
            return self.isPrime_raw(self.real)
        if self.real == 0:
            return self.isPrime_raw(abs(self.imaginary))
        return self.isPrime_raw(self.real * self.real +
                                self.imaginary * self.imaginary)

    def phase(self):
        """Returns angle between x axis"""
        return Decimal(math.atan2(self.imaginary, self.real))

    def polar(self):  #Converts to polar
        return (Decimal(abs(self).real), self.phase())

    def toRect(self, r, phi):
        return ComplexDecimal(r) * ComplexDecimal(self.cos_raw(phi),
                                                  self.sin_raw(phi))

    #Trig --------------------------------------------
    #1/functions, log,

    def cos_raw(self, x, prec=Decimal("0.0000000000000000000001")):
        """Raw cosine function for real numbers only"""
        # p = Decimal("0")
        # s = t = Decimal("1.0")
        # while abs(t/s) > prec:
        #     p+=Decimal("1")
        #     t = (-t * x * x) / ((Decimal("2") * p - Decimal("1")) * (Decimal("2") * p))
        #     s += t
        # if round(s,50) in [1,0,-1]: #If it's close enough round to get perfect answers (No one likes 1E-70 as cos(pi))
        #     return Decimal(round(s,50))
        # return s
        return Decimal(math.cos(x))

    def sin_raw(self, x, prec=Decimal("0.00000000000000000000001")):
        #return self.cos_raw(x+self.pi/Decimal("2"))
        return Decimal(math.sin(x))

    def tan_raw(self, x, prec=Decimal("0.00000000000000000000001")):
        return self.sin_raw(x, prec) / self.cos_raw(x, prec)

    def atan_raw(self, x):  #Not relatively accurate
        returned = Decimal("0")
        for j in range(0, 50):
            i = Decimal(j)
            a = Decimal("-1")**i * x**(Decimal("2") * i + Decimal("1"))
            b = Decimal("2") * i + Decimal("1")
            returned += a / b
        return returned

    def cos(self):
        """Compute cos of self"""
        if self.imaginary == 0:
            return ComplexDecimal(self.cos_raw(self.real))
        try:
            returned = ComplexDecimal(
                self.cos_raw(self.real) *
                ComplexDecimal(self.imaginary).cosh().real,
                -self.sin_raw(self.real) *
                ComplexDecimal(self.imaginary).sinh().real)
            returned.imaginary = -returned.imaginary
            return returned
        except:
            return ComplexDecimal("inf")

    def sin(self):
        """Compute sin of self"""
        if self.imaginary == 0:
            return ComplexDecimal(self.sin_raw(self.real))
        try:
            returned = ComplexDecimal(
                self.sin_raw(self.real) *
                ComplexDecimal(self.imaginary).cosh().real,
                -self.cos_raw(self.real) *
                ComplexDecimal(self.imaginary).sinh().real)
            returned.imaginary = -returned.imaginary
            return returned
        except:
            return ComplexDecimal("inf")

    def tan(self):
        """Compute tangent of self"""
        try:
            return self.sin() / self.cos()
        except:
            return ComplexDecimal("inf")

    def acos(self):
        """Attempt to compute arcosine"""
        if self.imaginary == 0:
            return ComplexDecimal(math.acos(self))
        A = ComplexDecimal((
            (Decimal(1) + self.real)**Decimal(2) +
            self.imaginary**Decimal(2))**Decimal(0.5) - (
                (Decimal(1) - self.real)**Decimal(0.5) +
                self.imaginary**Decimal(2))**Decimal(0.5)) / ComplexDecimal(2)
        B = ComplexDecimal((
            (Decimal(1) + self.real)**Decimal(2) +
            self.imaginary**Decimal(2))**Decimal(0.5) + (
                (Decimal(1) - self.real)**Decimal(0.5) +
                self.imaginary**Decimal(2))**Decimal(0.5)) / ComplexDecimal(2)
        return ComplexDecimal(
            A.acos(),
            -(B + (B * B - ComplexDecimal(1))**ComplexDecimal(0.5)).ln())

    def asin(self):
        """Attempt to compute arcsine"""
        if self.imaginary == 0:
            return ComplexDecimal(math.asin(self))
        A = ComplexDecimal((
            (Decimal(1) + self.real)**Decimal(2) +
            self.imaginary**Decimal(2))**Decimal(0.5) - (
                (Decimal(1) - self.real)**Decimal(0.5) +
                self.imaginary**Decimal(2))**Decimal(0.5)) / ComplexDecimal(2)
        B = ComplexDecimal((
            (Decimal(1) + self.real)**Decimal(2) +
            self.imaginary**Decimal(2))**Decimal(0.5) + (
                (Decimal(1) - self.real)**Decimal(0.5) +
                self.imaginary**Decimal(2))**Decimal(0.5)) / ComplexDecimal(2)
        return ComplexDecimal(
            A.asin(),
            (B + (B * B - ComplexDecimal(1))**ComplexDecimal(0.5)).ln())

    def atan(self):
        """Attempt to compute arctangent"""
        result = (ComplexDecimal(0, 1) + self) / (ComplexDecimal(0, 1) - self)
        return ComplexDecimal(0, 0.5) * result.ln()

    def sinh(self):
        """Hyperbolic sine of self"""
        returned = (self.exp() - (-self).exp()) / ComplexDecimal("2")
        returned.imaginary = -returned.imaginary
        return returned

    def cosh(self):
        """Hyperbolic cosine of self"""
        returned = (self.exp() + (-self).exp()) / ComplexDecimal("2")
        returned.imaginary = -returned.imaginary
        return returned

    def tanh(self):
        """Hyperbolic tangent of self"""
        return self.sinh() / self.cosh()

    def acosh(self):
        """Arc hyperbolic cosine"""
        returned = self + (self * self - ComplexDecimal(1))**0.5
        return returned.ln()

    def asinh(self):
        """Arc hyperbolic sine"""
        returned = self + (self * self + ComplexDecimal(1))**0.5
        return returned.ln()

    def atanh(self):
        """Arc hyperbolic  tan"""
        a = (ComplexDecimal(1) + self).ln() - (ComplexDecimal(1) - self).ln()
        return ComplexDecimal(0.5) * a

    def ln_raw(self, x):
        returned = Decimal("0")
        for i in range(0, 25):
            a = Decimal("1") / (Decimal(i) * Decimal("2") + Decimal("1"))
            b = ((x - Decimal("1")) /
                 (x + Decimal("1")))**(Decimal(i) * Decimal("2") +
                                       Decimal("1"))
            returned += a * b
        return returned * 2

    def ln(self):
        """Natural logarithim of self"""
        if self.imaginary == 0:
            try:
                return ComplexDecimal(self.ln_raw(self.real))
            except:
                pass
        p = self.polar()
        return ComplexDecimal(self.ln_raw(p[0]), p[1])

    def log(self, x=None):
        """Compute log of number in base x"""
        if not x: x = self.e
        if self.imaginary == 0:
            try:
                return ComplexDecimal(math.log(self.real, x))
            except:
                pass
        p = self.polar()
        return ComplexDecimal(math.log(p[0], x), p[1])

    def log10(self):
        """Compute log base 10"""
        if self.imaginary == 0:
            return ComplexDecimal(math.log10(self.real))
        return self.ln() / ComplexDecimal("10").ln()

    #Replace builtins --------------------------------
    # power, comparasion,
    #To float, int, complex

    def __str__(self):
        """Converts to str"""
        if self.imaginary == 0:
            return str(self.real)
        elif self.real == 0:
            if self.imaginary == 1: return "i"
            if self.imaginary == -1: return "-i"
            return str(self.imaginary) + "i"
        return (str(self.real) + "+" + str(self.imaginary) + "i").replace(
            "+-", "-")

    def __abs__(self):
        """Absolute value"""
        return ComplexDecimal((self.real**2 + self.imaginary**2).sqrt())

    def __add__(self, other):
        """Add 2 numbers together"""
        return ComplexDecimal(self.real + other.real,
                              self.imaginary + other.imaginary)

    def __sub__(self, other):
        """Subtract 2 numbers"""
        return ComplexDecimal(self.real - other.real,
                              self.imaginary - other.imaginary)

    def __mul__(self, other):
        """Multiply 2 numbers"""
        if self.imaginary == 0 and other.imaginary == 0:
            return ComplexDecimal(self.real * other.real, "0")
        return ComplexDecimal(
            self.real * other.real - self.imaginary * other.imaginary,
            self.real * other.imaginary + self.imaginary * other.real)

    def __div__(self, other):
        """Divide 2 numbers"""
        if self.imaginary == 0 and other.imaginary == 0:
            return ComplexDecimal(self.real / other.real, "0")
        a = (self.real * other.real + self.imaginary * other.imaginary) / (
            other.real * other.real + other.imaginary * other.imaginary)
        b = (self.imaginary * other.real - self.real * other.imaginary) / (
            other.real * other.real + other.imaginary * other.imaginary)
        return ComplexDecimal(a, b)

    def __truediv__(self, other):
        """Divide 2 numbers"""
        if self.imaginary == 0 and other.imaginary == 0:
            return ComplexDecimal(self.real / other.real, "0")
        a = (self.real * other.real + self.imaginary * other.imaginary) / (
            other.real * other.real + other.imaginary * other.imaginary)
        b = (self.imaginary * other.real - self.real * other.imaginary) / (
            other.real * other.real + other.imaginary * other.imaginary)
        return ComplexDecimal(a, b)

    def __neg__(self):
        """Negates a number"""
        return ComplexDecimal(-self.real, -self.imaginary)

    def __pos__(self):
        """Positive number"""
        return ComplexDecimal(abs(self.real), abs(self.imaginary))

    def __inverse__(self):
        """1/ number"""
        return ComplexDecimal(1) / self

    def __mod__(self, other):  #a%b = a + b * ciel(-a/b)
        """Modolus"""
        return self + other * ((-self / other).ceil())

    #Powers
    def __pow__(self, other):
        """Powers :D"""
        if self.imaginary == 0 and other.imaginary == 0:
            return ComplexDecimal(self.real**other.real)
        if other.imaginary == 0:
            polar = self.polar()
            return self.toRect(polar[0]**other.real, polar[1] * other.real)
        elif other.real == 0:
            a = ComplexDecimal(self.real)
            b = ComplexDecimal(self.imaginary)
            c = ComplexDecimal(other.real)
            d = ComplexDecimal(other.imaginary)
            x = (-d * (b / a).atan()).exp() * (
                (a * a + b * b).ln() * d / ComplexDecimal(2)).cos()
            y = ComplexDecimal(0, 1) * (-d * (b / a).atan()).exp() * (
                (a * a + b * b).ln() * d / ComplexDecimal(2)).sin()
            return x + y

        b = other.real
        c = other.imaginary
        returned = self**(ComplexDecimal(b)) * self**(ComplexDecimal(0, 1) *
                                                      ComplexDecimal(c))
        returned.imaginary = -returned.imaginary
        return returned

    #Additional conversions
    def __complex__(self):
        """Convert to complex"""
        return complex(float(self.real), float(self.imaginary))

    def __int__(self):
        """Convert to int"""
        return int(self.real)

    def __float__(self):
        """Convert to float"""
        return float(self.real)

    #Comparasions
    def __lt__(self, other):  #>
        if self.imaginary != 0 or other.imaginary != 0:
            raise errors.ComparasionError(
                "Complex comparasion is not supported")
        return self.real < other.real

    def __le__(self, other):  #>=
        if self.imaginary != 0 or other.imaginary != 0:
            raise errors.ComparasionError(
                "Complex comparasion is not supported")
        return self.real <= other.real

    def __eq__(self, other):  #==
        if self.real == other.real and self.imaginary == other.imaginary:
            return True
        return False

    def __ne__(self, other):  #!=
        return not self.__eq__(other)

    def __gt__(self, other):  #<
        if self.imaginary != 0 or other.imaginary != 0:
            raise errors.ComparasionError(
                "Complex comparasion is not supported")
        return self.real > other.real

    def __ge__(self, other):  #<=
        if self.imaginary != 0 or other.imaginary != 0:
            raise errors.ComparasionError(
                "Complex comparasion is not supported")
        return self.real >= other.real

    #Some things reimplemented from decimal class for complex ComplexDecimals
    #================================================================
    def copy_abs(self):
        return abs(self)

    def copy_negate(self):
        return -self

    def copy_sign(self, other):
        if other < 0:
            return -abs(self)
        return abs(self)

    def is_finite(self):
        return self.real.is_finite() and self.imaginary.is_finite()

    def is_infinite(self):
        return not self.is_finite()

    def is_nan(self):
        return self.real.is_nan() or self.imaginary.is_nan()

    def is_signed(self):
        return self < ComplexDecimal(0)

    def is_zero(self):
        return self.real.is_zero() and self.imaginary.is_zero()

    def radix(self):
        return Decimal(10)

    def sqrt(self):
        if self.imaginary.is_zero():
            try:
                return ComplexDecimal(self.real.sqrt())
            except:
                return ComplexDecimal(0, (-self.real).sqrt())
        return self**ComplexDecimal("0.5")

    def to_eng_string(self):
        returned = self.real.to_eng_string(
        ) + " + " + self.imaginary.to_eng_string + "i"
        return returned.replace(" + -", " - ")
Exemplo n.º 28
0
def streamingExtensionsLoader(modelXbrl, mappedUri, filepath):
    # check if big instance and has header with an initial incomplete tree walk (just 2 elements
    def logSyntaxErrors(parsercontext):
        for error in parsercontext.error_log:
            modelXbrl.error("xmlSchema:syntax",
                    _("%(error)s, %(fileName)s, line %(line)s, column %(column)s, %(sourceAction)s source element"),
                    modelObject=modelDocument, fileName=os.path.basename(filepath), 
                    error=error.message, line=error.line, column=error.column, sourceAction="streaming")
    #### note: written for iterparse of lxml prior to version 3.3, otherwise rewrite to use XmlPullParser ###
    #### note: iterparse wants a binary file, but file is text mode
    _file, = modelXbrl.fileSource.file(filepath, binary=True)
    startedAt = time.time()
    modelXbrl.profileActivity()
    parsercontext = etree.iterparse(_file, events=("start","end"), huge_tree=True)
    foundInstance = False
    foundErrors = False
    streamingAspects = None
    numRootFacts1 = 0
    numElts = 0
    elt = None
    for event, elt in parsercontext:
        if event == "start":
            if elt.getparent() is not None:
                if elt.getparent().tag == "{http://www.xbrl.org/2003/instance}xbrl":
                    if not foundInstance:
                        foundInstance = True
                        pi = precedingProcessingInstruction(elt, "xbrl-streamable-instance")
                        if pi is None:
                            break
                        else:
                            streamingAspects = dict(pi.attrib.copy())
                    if not elt.tag.startswith("{http://www.xbrl.org/"):
                        numRootFacts1 += 1
                        if numRootFacts1 % 1000 == 0:
                            modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
                elif not foundInstance:       
                    break
            elif elt.tag == "{http://www.xbrl.org/2003/instance}xbrl" and precedingProcessingInstruction(elt, "xbrl-streamable-instance") is not None:
                modelXbrl.error("streamingExtensions:headerMisplaced",
                        _("Header is misplaced: %(error)s, must follow xbrli:xbrl element"),
                        modelObject=elt)
        elif event == "end":
            elt.clear()
            numElts += 1
            if numElts % 1000 == 0 and elt.getparent() is not None:
                while elt.getprevious() is not None and elt.getparent() is not None:
                    del elt.getparent()[0]
    if elt is not None:
        elt.clear()
    _file.seek(0,io.SEEK_SET) # allow reparsing
    if not foundInstance or streamingAspects is None:
        del elt, parsercontext
        _file.close()
        return None
    modelXbrl.profileStat(_("streaming tree check"), time.time() - startedAt)
    startedAt = time.time()
    try:
        version = Decimal(streamingAspects.get("version"))
        if int(version) != 1:
            modelXbrl.error("streamingExtensions:unsupportedVersion",
                    _("Streaming version %(version)s, major version number must be 1"),
                    modelObject=elt, version=version)
            foundErrors = True
    except (InvalidOperation, OverflowError):
        modelXbrl.error("streamingExtensions:versionError",
                _("Version %(version)s, number must be 1.n"),
                modelObject=elt, version=streamingAspects.get("version", "(none)"))
        foundErrors = True
    for bufAspect in ("contextBuffer", "unitBuffer", "footnoteBuffer"):
        try:
            bufLimit = Decimal(streamingAspects.get(bufAspect, "INF"))
            if bufLimit < 1 or (bufLimit.is_finite() and bufLimit % 1 != 0):
                raise InvalidOperation
            elif bufAspect == "contextBuffer":
                contextBufferLimit = bufLimit
            elif bufAspect == "unitBuffer":
                unitBufferLimit = bufLimit
            elif bufAspect == "footnoteBuffer":
                footnoteBufferLimit = bufLimit
        except InvalidOperation:
            modelXbrl.error("streamingExtensions:valueError",
                    _("Streaming %(attrib)s %(value)s, number must be a positive integer or INF"),
                    modelObject=elt, attrib=bufAspect, value=streamingAspects.get(bufAspect))
            foundErrors = True
    if parsercontext.error_log:
        foundErrors = True
    logSyntaxErrors(parsercontext)
    
    if foundErrors:
        _file.close()
        return None
    parsercontext = etree.iterparse(_file, events=("start","end"), huge_tree=True)
    _parser, _parserLookupName, _parserLookupClass = parser(modelXbrl,filepath)
    eltMdlObjs = {}
    beforeInstanceStream = True
    validator = None
    contextBuffer = []
    unitBuffer = []
    footnoteBuffer = []
    factBuffer = []
    numFacts = numRootFacts2 = 1
    for event, elt in parsercontext:
        if event == "start":
            mdlObj = _parser.makeelement(elt.tag, attrib=elt.attrib, nsmap=elt.nsmap)
            mdlObj.sourceline = elt.sourceline
            eltMdlObjs[elt] = mdlObj
            if elt.getparent() is None:
                modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, etree.ElementTree(mdlObj))
                modelDocument.xmlRootElement = mdlObj
                modelXbrl.modelDocument = modelDocument # needed for incremental validation
                mdlObj.init(modelDocument)
                modelXbrl.info("streamingExtensions:streaming",
                               _("Stream processing this instance."),
                               modelObject = modelDocument)    
            else:
                eltMdlObjs[elt.getparent()].append(mdlObj)
                mdlObj._init()
                ns = mdlObj.namespaceURI
                ln = mdlObj.localName
                if (beforeInstanceStream and (
                    (ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or
                    (ns == XbrlConst.xbrli and ln in ("context", "unit")) or
                    (ns not in (XbrlConst.link, XbrlConst.xbrli)))):
                    beforeInstanceStream = False
                    if _streamingExtensionsValidate:
                        validator = Validate(modelXbrl)
                        validator.instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters())
                    else: # need default dimensions
                        ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
            mdlObj = None # deref
                        
        elif event == "end":
            mdlObj = eltMdlObjs.pop(elt)
            if elt.text: # text available after child nodes processed
                mdlObj.text = elt.text
            ns = mdlObj.namespaceURI
            ln = mdlObj.localName
            parentMdlObj = mdlObj.getparent()
            if ns == XbrlConst.xbrli:
                if ln == "context":
                    if mdlObj.get("sticky"):
                        del mdlObj.attrib["sticky"]
                        modelDocument.contextDiscover(mdlObj)
                    else:
                        if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit:
                            # drop before adding as dropped may have same id as added
                            cntx = contextBuffer.pop(0)
                            dropContext(modelXbrl, cntx)
                            del parentMdlObj[parentMdlObj.index(cntx)]
                            cntx = None
                        modelDocument.contextDiscover(mdlObj)
                        if contextBufferLimit.is_finite():
                            contextBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        contextsToCheck = (mdlObj,)
                        validator.instValidator.checkContexts(contextsToCheck)
                        if modelXbrl.hasXDT:
                            validator.instValidator.checkContextsDimensions(contextsToCheck)
                        del contextsToCheck # dereference
                elif ln == "unit":
                    if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit:
                        # drop before additing as dropped may have same id as added
                        unit = unitBuffer.pop(0)
                        dropUnit(modelXbrl, unit)
                        del parentMdlObj[parentMdlObj.index(unit)]
                        unit = None 
                    modelDocument.unitDiscover(mdlObj)
                    if unitBufferLimit.is_finite():
                        unitBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        validator.instValidator.checkUnits( (mdlObj,) )
                elif ln == "xbrl": # end of document
                    # check remaining footnote refs
                    for footnoteLink in footnoteBuffer:
                        checkFootnoteHrefs(modelXbrl, footnoteLink)
                elt.clear()
            elif ns == XbrlConst.link:
                if ln in ("schemaRef", "linkbaseRef"):
                    modelDocument.discoverHref(mdlObj)
                elif ln in ("roleRef", "arcroleRef"):
                    modelDocument.linkbaseDiscover((mdlObj,), inInstance=True)
                elif ln == "footnoteLink":
                    footnoteLinks = (mdlObj,)
                    modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True)
                    if footnoteBufferLimit.is_finite():
                        footnoteBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        validator.instValidator.checkLinks(footnoteLinks)
                        if len(footnoteBuffer) > footnoteBufferLimit:
                            # check that hrefObjects for locators were all satisfied
                                # drop before addition as dropped may have same id as added
                            footnoteLink = footnoteBuffer.pop(0)
                            checkFootnoteHrefs(modelXbrl, footnoteLink)
                            dropFootnoteLink(modelXbrl, footnoteLink)
                            del parentMdlObj[parentMdlObj.index(footnoteLink)]
                            footnoteLink = None
                    footnoteLinks = None
                elt.clear()
            elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl:
                numRootFacts2 += 1
                modelDocument.factDiscover(mdlObj, modelXbrl.facts)
                XmlValidate.validate(modelXbrl, mdlObj)
                if _streamingExtensionsValidate:
                    factsToCheck = (mdlObj,)
                    validator.instValidator.checkFacts(factsToCheck)
                    if modelXbrl.hasXDT:
                        validator.instValidator.checkFactsDimensions(factsToCheck)
                    del factsToCheck
                    dropFact(modelXbrl, mdlObj, modelXbrl.facts)
                    del parentMdlObj[parentMdlObj.index(mdlObj)]
                if numRootFacts2 % 1000 == 0:
                    modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(numRootFacts2, numRootFacts1, 100.0 * numRootFacts2 / numRootFacts1), 
                                              minTimeToShow=20.0)
                # get rid of root element from iterparse's tree
                elt.clear()
                while elt.getprevious() is not None:  # cleans up any prior siblings
                    del elt.getparent()[0]
            mdlObj = None # deref
    logSyntaxErrors(parsercontext)
    del parsercontext
    if validator is not None:
        validator.close()
    _file.close()
    modelXbrl.profileStat(_("streaming complete"), time.time() - startedAt)
    return modelDocument
Exemplo n.º 29
0
def streamingExtensionsLoader(modelXbrl, mappedUri, filepath, **kwargs):
    # check if big instance and has header with an initial incomplete tree walk (just 2 elements
    if not _streamingExtensionsCheck:
        return None
    
    # track whether modelXbrl has been validated by this streaming extension
    modelXbrl._streamingExtensionValidated = False
        
    def logSyntaxErrors(parsercontext):
        for error in parsercontext.error_log:
            modelXbrl.error("xmlSchema:syntax",
                    _("%(error)s, %(fileName)s, line %(line)s, column %(column)s, %(sourceAction)s source element"),
                    modelObject=modelXbrl, fileName=os.path.basename(filepath), 
                    error=error.message, line=error.line, column=error.column, sourceAction="streaming")
    #### note: written for iterparse of lxml prior to version 3.3, otherwise rewrite to use XmlPullParser ###
    #### note: iterparse wants a binary file, but file is text mode
    _file, = modelXbrl.fileSource.file(filepath, binary=True)
    startedAt = time.time()
    modelXbrl.profileActivity()
    ''' this seems twice as slow as iterparse
    class instInfoTarget():
        def __init__(self, element_factory=None, parser=None):
            self.newTree = True
            self.streamingAspects = None
            self.foundInstance = False
            self.creationSoftwareComment = ''
            self.currentEltTag = "(before xbrli:xbrl)"
            self.numRootFacts = 0
        def start(self, tag, attrib, nsmap=None):
            if self.newTree:
                if tag == "{http://www.xbrl.org/2003/instance}xbrl":
                    self.foundInstance = True
                    self.newTree = False
                else: # break 
                    raise NotInstanceDocumentException()
            elif not tag.startswith("{http://www.xbrl.org/"):
                self.numRootFacts += 1
                if self.numRootFacts % 1000 == 0:
                    modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
            self.currentEltTag = tag
        def end(self, tag):
            pass
        def data(self, data):
            pass
        def comment(self, text):
            if not self.foundInstance: # accumulate comments before xbrli:xbrl
                self.creationSoftwareComment += ('\n' if self.creationSoftwareComment else '') + text
            elif not self.creationSoftwareComment:
                self.creationSoftwareComment = text # or first comment after xbrli:xbrl
        def pi(self, target, data):
            if target == "xbrl-streamable-instance":
                if self.currentEltTag == "{http://www.xbrl.org/2003/instance}xbrl":
                    self.streamingAspects = dict(etree.PI(target,data).attrib.copy()) # dereference target results
                else:
                    modelXbrl.error("streamingExtensions:headerMisplaced",
                            _("Header is misplaced: %(target)s, must follow xbrli:xbrl element but was found at %(element)s"),
                            modelObject=modelXbrl, target=target, element=self.currentEltTag)
        def close(self):
            if not self.creationSoftwareComment:
                self.creationSoftwareComment = None
            return True
    instInfo = instInfoTarget()
    infoParser = etree.XMLParser(recover=True, huge_tree=True, target=instInfo)
    try:
        etree.parse(_file, parser=infoParser, base_url=filepath)
    except NotInstanceDocumentException:
        pass
    '''
    foundErrors = False
    foundInstance = False
    streamingAspects = None
    creationSoftwareComment = None
    instInfoNumRootFacts = 0
    numElts = 0
    elt = None
    instInfoContext = etree.iterparse(_file, events=("start","end"), huge_tree=True)
    for event, elt in instInfoContext:
        if event == "start":
            if elt.getparent() is not None:
                if elt.getparent().tag == "{http://www.xbrl.org/2003/instance}xbrl":
                    if not foundInstance:
                        foundInstance = True
                        pi = precedingProcessingInstruction(elt, "xbrl-streamable-instance")
                        if pi is None:
                            break
                        else:
                            streamingAspects = dict(pi.attrib.copy())
                            if creationSoftwareComment is None:
                                creationSoftwareComment = precedingComment(elt)
                    if not elt.tag.startswith("{http://www.xbrl.org/"):
                        instInfoNumRootFacts += 1
                        if instInfoNumRootFacts % 1000 == 0:
                            modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
                elif not foundInstance:       
                    break
            elif elt.tag == "{http://www.xbrl.org/2003/instance}xbrl":
                creationSoftwareComment = precedingComment(elt)
                if precedingProcessingInstruction(elt, "xbrl-streamable-instance") is not None:
                    modelXbrl.error("streamingExtensions:headerMisplaced",
                            _("Header is misplaced: %(error)s, must follow xbrli:xbrl element"),
                            modelObject=elt)
        elif event == "end":
            elt.clear()
            numElts += 1
            if numElts % 1000 == 0 and elt.getparent() is not None:
                while elt.getprevious() is not None and elt.getparent() is not None:
                    del elt.getparent()[0]
    if elt is not None:
        elt.clear()
    
    _file.seek(0,io.SEEK_SET) # allow reparsing
    if not foundInstance or streamingAspects is None:
        del elt
        _file.close()
        return None
    modelXbrl.profileStat(_("streaming tree check"), time.time() - startedAt)
    startedAt = time.time()
    try:
        version = Decimal(streamingAspects.get("version"))
        if int(version) != 1:
            modelXbrl.error("streamingExtensions:unsupportedVersion",
                    _("Streaming version %(version)s, major version number must be 1"),
                    modelObject=elt, version=version)
            foundErrors = True
    except (InvalidOperation, OverflowError):
        modelXbrl.error("streamingExtensions:versionError",
                _("Version %(version)s, number must be 1.n"),
                modelObject=elt, version=streamingAspects.get("version", "(none)"))
        foundErrors = True
    for bufAspect in ("contextBuffer", "unitBuffer", "footnoteBuffer"):
        try:
            bufLimit = Decimal(streamingAspects.get(bufAspect, "INF"))
            if bufLimit < 1 or (bufLimit.is_finite() and bufLimit % 1 != 0):
                raise InvalidOperation
            elif bufAspect == "contextBuffer":
                contextBufferLimit = bufLimit
            elif bufAspect == "unitBuffer":
                unitBufferLimit = bufLimit
            elif bufAspect == "footnoteBuffer":
                footnoteBufferLimit = bufLimit
        except InvalidOperation:
            modelXbrl.error("streamingExtensions:valueError",
                    _("Streaming %(attrib)s %(value)s, number must be a positive integer or INF"),
                    modelObject=elt, attrib=bufAspect, value=streamingAspects.get(bufAspect))
            foundErrors = True
    if _streamingExtensionsValidate:
        incompatibleValidations = []
        _validateDisclosureSystem = modelXbrl.modelManager.validateDisclosureSystem
        _disclosureSystem = modelXbrl.modelManager.disclosureSystem
        if _validateDisclosureSystem and _disclosureSystem.EFM:
            incompatibleValidations.append("EFM")
        if _validateDisclosureSystem and _disclosureSystem.GFM:
            incompatibleValidations.append("GFM")
        if _validateDisclosureSystem and _disclosureSystem.EBA:
            incompatibleValidations.append("EBA")
        if _validateDisclosureSystem and _disclosureSystem.HMRC:
            incompatibleValidations.append("EBA")
        if modelXbrl.modelManager.validateCalcLB:
            incompatibleValidations.append("calculation LB")
        if incompatibleValidations:
            modelXbrl.error("streamingExtensions:incompatibleValidation",
                    _("Streaming instance validation does not support %(incompatibleValidations)s validation"),
                    modelObject=modelXbrl, incompatibleValidations=', '.join(incompatibleValidations))
            foundErrors = True
    if instInfoContext.error_log:
        foundErrors = True
    logSyntaxErrors(instInfoContext)
    del instInfoContext # dereference

    for pluginMethod in pluginClassMethods("Streaming.BlockStreaming"):
        _blockingPluginName = pluginMethod(modelXbrl)
        if _blockingPluginName: # name of blocking plugin is returned
            modelXbrl.error("streamingExtensions:incompatiblePlugIn",
                    _("Streaming instance not supported by plugin %(blockingPlugin)s"),
                    modelObject=modelXbrl, blockingPlugin=_blockingPluginName)
            foundErrors = True
    
    if foundErrors:
        _file.close()
        return None

    _encoding = XmlUtil.encoding(_file.read(512))
    _file.seek(0,io.SEEK_SET) # allow reparsing

    if _streamingExtensionsValidate:
        validator = Validate(modelXbrl)
        instValidator = validator.instValidator

    eltMdlObjs = {}
    contextBuffer = []
    unitBuffer = []
    footnoteBuffer = []
    factBuffer = []
    numFacts = 1
    
    _streamingValidateFactsPlugin = any(True for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"))

    
    class modelLoaderTarget():
        def __init__(self, element_factory=None, parser=None):
            self.newTree = True
            self.currentMdlObj = None
            self.beforeInstanceStream = True
            self.beforeStartStreamingPlugin = True
            self.numRootFacts = 1
            modelXbrl.streamingParentModelObject = None
            modelXbrl.isStreamingMode = True
        def start(self, tag, attrib, nsmap=None):
            modelXbrl.streamingParentModelObject = self.currentMdlObj # pass parent to makeelement for ModelObjectFactory
            mdlObj = _parser.makeelement(tag, attrib=attrib, nsmap=nsmap)
            mdlObj.sourceline = 1
            if self.newTree:
                self.newTree = False
                self.currentMdlObj = mdlObj
                modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, mdlObj.getroottree())
                modelXbrl.modelDocument = modelDocument # needed for incremental validation
                mdlObj.init(modelDocument)
                modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement 
                modelDocument.parserLookupName = _parserLookupName
                modelDocument.parserLookupClass = _parserLookupClass
                modelDocument.xmlRootElement = mdlObj
                modelDocument.schemaLocationElements.add(mdlObj)
                modelDocument.documentEncoding = _encoding
                modelDocument._creationSoftwareComment = creationSoftwareComment
                modelXbrl.info("streamingExtensions:streaming",
                               _("Stream processing this instance."),
                               modelObject = modelDocument)
            else:
                self.currentMdlObj.append(mdlObj)
                self.currentMdlObj = mdlObj
                mdlObj._init()
                ns = mdlObj.namespaceURI
                ln = mdlObj.localName
                if (self.beforeInstanceStream and (
                    (ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or
                    (ns == XbrlConst.xbrli and ln in ("context", "unit")) or
                    (ns not in (XbrlConst.link, XbrlConst.xbrli)))):
                    self.beforeInstanceStream = False
                    if _streamingExtensionsValidate:
                        instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters())
                    else: # need default dimensions
                        ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
                elif not self.beforeInstanceStream and self.beforeStartStreamingPlugin:
                    for pluginMethod in pluginClassMethods("Streaming.Start"):
                        pluginMethod(modelXbrl)
                    self.beforeStartStreamingPlugin = False
            return mdlObj
        def end(self, tag):
            modelDocument = modelXbrl.modelDocument
            mdlObj = self.currentMdlObj
            parentMdlObj = mdlObj.getparent()
            self.currentMdlObj = parentMdlObj
            ns = mdlObj.namespaceURI
            ln = mdlObj.localName
            if ns == XbrlConst.xbrli:
                if ln == "context":
                    if mdlObj.get("sticky"):
                        del mdlObj.attrib["sticky"]
                        XmlValidate.validate(modelXbrl, mdlObj)
                        modelDocument.contextDiscover(mdlObj)
                    else:
                        if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit:
                            # drop before adding as dropped may have same id as added
                            cntx = contextBuffer.pop(0)
                            dropContext(modelXbrl, cntx)
                            del parentMdlObj[parentMdlObj.index(cntx)]
                            cntx = None
                        XmlValidate.validate(modelXbrl, mdlObj)
                        modelDocument.contextDiscover(mdlObj)
                        if contextBufferLimit.is_finite():
                            contextBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        contextsToCheck = (mdlObj,)
                        instValidator.checkContexts(contextsToCheck)
                        if modelXbrl.hasXDT:
                            instValidator.checkContextsDimensions(contextsToCheck)
                        del contextsToCheck # dereference
                elif ln == "unit":
                    if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit:
                        # drop before additing as dropped may have same id as added
                        unit = unitBuffer.pop(0)
                        dropUnit(modelXbrl, unit)
                        del parentMdlObj[parentMdlObj.index(unit)]
                        unit = None 
                    XmlValidate.validate(modelXbrl, mdlObj)
                    modelDocument.unitDiscover(mdlObj)
                    if unitBufferLimit.is_finite():
                        unitBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        instValidator.checkUnits( (mdlObj,) )
                elif ln == "xbrl": # end of document
                    # check remaining footnote refs
                    for footnoteLink in footnoteBuffer:
                        checkFootnoteHrefs(modelXbrl, footnoteLink)
                    for pluginMethod in pluginClassMethods("Streaming.Finish"):
                        pluginMethod(modelXbrl)
            elif ns == XbrlConst.link:
                if ln == "footnoteLink":
                    XmlValidate.validate(modelXbrl, mdlObj)
                    footnoteLinks = (mdlObj,)
                    modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True)
                    if footnoteBufferLimit.is_finite():
                        footnoteBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        instValidator.checkLinks(footnoteLinks)
                        if len(footnoteBuffer) > footnoteBufferLimit:
                            # check that hrefObjects for locators were all satisfied
                                # drop before addition as dropped may have same id as added
                            footnoteLink = footnoteBuffer.pop(0)
                            checkFootnoteHrefs(modelXbrl, footnoteLink)
                            dropFootnoteLink(modelXbrl, footnoteLink)
                            del parentMdlObj[parentMdlObj.index(footnoteLink)]
                            footnoteLink = None
                    footnoteLinks = None
                elif ln in ("schemaRef", "linkbaseRef"):
                    modelDocument.discoverHref(mdlObj)
                elif not modelXbrl.skipDTS:
                    if ln in ("roleRef", "arcroleRef"):
                        modelDocument.linkbaseDiscover((mdlObj,), inInstance=True)
            elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl:
                self.numRootFacts += 1
                XmlValidate.validate(modelXbrl, mdlObj)
                modelDocument.factDiscover(mdlObj, modelXbrl.facts)
                if _streamingExtensionsValidate or _streamingValidateFactsPlugin:
                    factsToCheck = (mdlObj,)  # validate current fact by itself
                    if _streamingExtensionsValidate:
                        instValidator.checkFacts(factsToCheck)
                        if modelXbrl.hasXDT:
                            instValidator.checkFactsDimensions(factsToCheck)
                    if _streamingValidateFactsPlugin:
                        # plugin attempts to process batch of all root facts not yet processed (not just current one)
                        factsToCheck = modelXbrl.facts.copy()
                        factsHaveBeenProcessed = True
                        # can block facts deletion if required data not yet available, such as numeric unit for DpmDB
                        for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"):
                            if not pluginMethod(modelXbrl, factsToCheck):
                                factsHaveBeenProcessed = False
                        if factsHaveBeenProcessed:
                            for fact in factsToCheck:
                                dropFact(modelXbrl, fact, modelXbrl.facts)
                                del parentMdlObj[parentMdlObj.index(fact)]
                    else:
                        dropFact(modelXbrl, mdlObj, modelXbrl.facts) # single fact has been processed
                        del parentMdlObj[parentMdlObj.index(mdlObj)]
                    del factsToCheck # dereference fact or batch of facts
                if self.numRootFacts % 1000 == 0:
                    modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts, 
                                                                                              100.0 * self.numRootFacts / instInfoNumRootFacts), 
                                              minTimeToShow=20.0)
            return mdlObj
        def data(self, data):
            self.currentMdlObj.text = data
        def comment(self, text):
            pass
        def pi(self, target, data):
            pass
        def close(self):
            del modelXbrl.streamingParentModelObject
            return None
        
    _parser, _parserLookupName, _parserLookupClass = parser(modelXbrl, filepath, target=modelLoaderTarget())
    etree.parse(_file, parser=_parser, base_url=filepath)
    logSyntaxErrors(_parser)
    if _streamingExtensionsValidate and validator is not None:
        _file.close()
        del instValidator
        validator.close()
        # track that modelXbrl has been validated by this streaming extension
        modelXbrl._streamingExtensionValidated = True
        
    modelXbrl.profileStat(_("streaming complete"), time.time() - startedAt)
    return modelXbrl.modelDocument
Exemplo n.º 30
0
 def delay(self, value):
     value = Decimal(value)
     if value <= 0 or not value.is_finite():
         return
     self._delay = value
Exemplo n.º 31
0
 def delay(self, value):
     value = Decimal(value)
     if value <= 0 or not value.is_finite():
         return
     self._delay = value
Exemplo n.º 32
0
def decimals(min_value=None,
             max_value=None,
             allow_nan=None,
             allow_infinity=None,
             places=None):
    """Generates instances of decimals.Decimal, which may be:

    - A finite rational number, between ``min_value`` and ``max_value``.
    - Not a Number, if ``allow_nan`` is True.  None means "allow NaN, unless
      ``min__value`` and ``max_value`` are not None".
    - Positive or negative infinity, if ``max_value`` and ``min_value``
      respectively are None, and ``allow_infinity`` is not False.  None means
      "allow infinity, unless excluded by the min and max values".

    Note that where floats have one `NaN` value, Decimals have four: signed,
    and either *quiet* or *signalling*.  See `the decimal module docs
    <https://docs.python.org/3/library/decimal.html#special-values>`_ for
    more information on special values.

    If ``places`` is not None, all finite values drawn from the strategy will
    have that number of digits after the decimal place.

    """
    # Convert min_value and max_value to Decimal values, and validate args
    check_valid_integer(places)
    if places is not None and places < 0:
        raise InvalidArgument('places=%r may not be negative' % places)

    if min_value is not None:
        min_value = Decimal(min_value)
        if min_value.is_infinite() and min_value < 0:
            if not (allow_infinity or allow_infinity is None):
                raise InvalidArgument('allow_infinity=%r, but min_value=%r' %
                                      (allow_infinity, min_value))
            min_value = None
        elif not min_value.is_finite():
            # This could be positive infinity, quiet NaN, or signalling NaN
            raise InvalidArgument(u'Invalid min_value=%r' % min_value)
    if max_value is not None:
        max_value = Decimal(max_value)
        if max_value.is_infinite() and max_value > 0:
            if not (allow_infinity or allow_infinity is None):
                raise InvalidArgument('allow_infinity=%r, but max_value=%r' %
                                      (allow_infinity, max_value))
            max_value = None
        elif not max_value.is_finite():
            raise InvalidArgument(u'Invalid max_value=%r' % max_value)
    check_valid_bound(min_value, 'min_value')
    check_valid_bound(max_value, 'max_value')
    check_valid_interval(min_value, max_value, 'min_value', 'max_value')

    if allow_infinity and (None not in (min_value, max_value)):
        raise InvalidArgument('Cannot allow infinity between finite bounds')
    # Set up a strategy for finite decimals
    if places is not None:
        # Fixed-point decimals are basically integers with a scale factor

        def try_quantize(d):
            try:
                return d.quantize(factor)
            except InvalidOperation:  # pragma: no cover
                return None

        factor = Decimal(10)**-places
        max_num = max_value / factor if max_value is not None else None
        min_num = min_value / factor if min_value is not None else None
        strat = integers(min_value=min_num, max_value=max_num)\
            .map(lambda d: try_quantize(d * factor))\
            .filter(lambda d: d is not None)
    else:
        # Otherwise, they're like fractions featuring a power of ten
        strat = fractions(min_value=min_value, max_value=max_value).map(
            lambda f: Decimal(f.numerator) / f.denominator)
    # Compose with sampled_from for infinities and NaNs as appropriate
    special = []
    if allow_nan or (allow_nan is None and (None in (min_value, max_value))):
        special.extend(map(Decimal, ('NaN', '-NaN', 'sNaN', '-sNaN')))
    if allow_infinity or (allow_infinity is max_value is None):
        special.append(Decimal('Infinity'))
    if allow_infinity or (allow_infinity is min_value is None):
        special.append(Decimal('-Infinity'))
    return strat | sampled_from(special)
Exemplo n.º 33
0
    def format_value(self, value, format_target):
        """
        Format a value nicely for human-readable output (including rounding).

        @param value: the value to format
        @param format_target the target the value should be formatted for
        @return: a formatted String representation of the given value.
        """
        # Only format counts and measures
        if self.type.type != ColumnType.count and self.type.type != ColumnType.measure:
            return value

        if format_target not in POSSIBLE_FORMAT_TARGETS:
            raise ValueError("Unknown format target")

        if value is None or value == "":
            return ""

        if isinstance(value, str):
            # If the number ends with "s" or another unit, remove it.
            # Units should not occur in table cells, but in the table head.
            number_str = util.remove_unit(value.strip())
            number = Decimal(number_str)
        elif isinstance(value, Decimal):
            number = value
            number_str = util.print_decimal(number)
        else:
            raise TypeError("Unexpected number type " + str(type(value)))

        if number.is_nan():
            return "NaN"
        elif number == inf:
            return "Inf"
        elif number == -inf:
            return "-Inf"

        # Apply the scale factor to the value
        if self.scale_factor is not None:
            number *= self.scale_factor
        assert number.is_finite()

        if (self.number_of_significant_digits is None
                and self.type.type != ColumnType.measure
                and format_target == "tooltip_stochastic"):
            # Column of type count (integral values) without specified sig. digits.
            # However, we need to round values like stdev, so we just round somehow.
            return util.print_decimal(round(number, DEFAULT_TOOLTIP_PRECISION))

        number_of_significant_digits = self.get_number_of_significant_digits(
            format_target)
        max_dec_digits = (self.type.max_decimal_digits
                          if self.type.type == ColumnType.measure else 0)

        if number_of_significant_digits is not None:
            current_significant_digits = _get_significant_digits(number_str)
            return _format_number(
                number,
                current_significant_digits,
                number_of_significant_digits,
                max_dec_digits,
                format_target,
            )
        else:
            return util.print_decimal(number)