def __init__(
self,
underlying: str,
currency: Currency,
optionType: OptionType,
expiration: date,
strike: Decimal,
multiplier: Decimal = Decimal(100),
exchange: Optional[str] = None,
symbol: Optional[str] = None,
):
if not underlying:
raise ValueError("Expected non-empty underlying symbol for Option")
if not strike.is_finite() or strike <= 0:
raise ValueError(f"Expected positive strike price: {strike}")
if not multiplier.is_finite() or multiplier <= 0:
raise ValueError(f"Expected positive multiplier: {multiplier}")
self.underlying = underlying
self.optionType = optionType
self.expiration = expiration
self.strike = self.quantizeStrike(strike)
if symbol is None:
# https://en.wikipedia.org/wiki/Option_symbol#The_OCC_Option_Symbol
symbol = f"{underlying:6}{expiration.strftime('%y%m%d')}{optionType.value}{(strike * 1000):08.0f}"
super().__init__(symbol=symbol,
currency=currency,
multiplier=multiplier,
exchange=exchange)
def holdings(
val: pd.DataFrame, holds: np.ndarray, i: pd.DataFrame, t: int, aum_t: Decimal
) -> Decimal:
open_price = Decimal(val[i].loc["open"][t])
# If the open price is NaN, this instrument's open wasn't recorded at time t.
# So let's use the previous day's calculation.
if not open_price.is_finite():
prev_day: Decimal = holds[t - 1][val.columns.get_loc(i)]
return prev_day
else:
# TODO: make the exchange rate flexible. This should generally be the dollar value of 1 point of instrument i.
exchange_rate = Decimal(1)
# The purpose of the (weights_i_t * sum_of_weights ^ -1) in the holdings calculation is to de-lever the allocations.
sum_of_weights: Decimal = val.loc["weight"].iloc[t].abs().sum()
# If today was t + 1, one way to calculate today's holdings would be by using the opening price, since for a future
# we may not know the closing price of a new contract at roll time.
# If open prices are unavailable, then the last close price at t will work too.
next_open = Decimal(val[i].loc["close"][t])
if not next_open.is_finite():
last_close_price: Decimal = holds[t - 1][val.columns.get_loc(i)]
return last_close_price
weighted_holding: Decimal = Decimal(val[i].loc["weight"][t]) * aum_t / Decimal(
next_open
) * exchange_rate * Decimal(sum_of_weights)
return weighted_holding
def formatNumberPair(value, *errors, precision=2, separator=" +- "): c = Context(prec=precision) largest_error = None for error in errors: e = c.create_decimal_from_float(error) if e.is_finite(): if largest_error: largest_error = largest_error.max_mag(e) else: largest_error = e results = [] value = Decimal(value) if largest_error and value.is_finite(): value = value.quantize(largest_error) results.append(value) for error in errors: error = Decimal(error) if largest_error and error.is_finite(): error = error.quantize(largest_error) results.append(error) return tuple((str(r) for r in results))
def receive_fmi_temperature(
) -> Tuple[Optional[Decimal], Optional[arrow.Arrow]]:
temp, ts = None, None
try:
starttime = arrow.now().shift(
hours=-1).to('UTC').format('YYYY-MM-DDTHH:mm:ss') + 'Z'
result = get_url(
'https://opendata.fmi.fi/wfs?request=getFeature&storedquery_id=fmi::observations::weather'
'::simple&place={place}¶meters=temperature&starttime={starttime}'
.format(place=config.FMI_LOCATION, starttime=starttime))
except Exception as e:
logger.exception(e)
else:
if result.status_code != 200:
logger.error('%d: %s' % (result.status_code, result.content))
else:
try:
wfs_member = xmltodict.parse(result.content).get(
'wfs:FeatureCollection', {}).get('wfs:member')
temp_data = wfs_member[-1].get('BsWfs:BsWfsElement')
if temp_data and 'BsWfs:Time' in temp_data and 'BsWfs:ParameterValue' in temp_data:
ts = arrow.get(temp_data['BsWfs:Time']).to(config.TIMEZONE)
temp = Decimal(temp_data['BsWfs:ParameterValue'])
if not temp.is_finite():
raise TypeError()
except (KeyError, TypeError):
temp, ts = None, None
logger.info('temp:%s ts:%s', temp, ts)
return temp, ts
def _parseFiniteDecimal(input: str) -> Decimal:
with localcontext(ctx=Context(traps=[DivisionByZero, Overflow])):
value = Decimal(input)
if not value.is_finite():
raise ValueError(f"Input is not numeric: {input}")
return value
def _apply(self, value):
allowed_types = (str, int, float, DecimalType,)
if self.allow_tuples:
# Python's Decimal type supports both tuples and lists.
# :py:meth:`decimal.Decimal.__init__`
allowed_types += (list, tuple,)
value = self._filter(value, Type(allowed_types))
if self._has_errors:
return value
try:
d = DecimalType(value)
except (InvalidOperation, TypeError, ValueError):
return self._invalid_value(value, self.CODE_INVALID, exc_info=True)
# Decimal's constructor also accepts values such as 'NaN' or
# '+Inf', which aren't valid in this context.
# :see: decimal.Decimal._parser
if not d.is_finite():
return self._invalid_value(
value=value,
reason=self.CODE_NON_FINITE,
exc_info=True,
)
if self.max_precision is not None:
d = d.quantize(self.max_precision)
return d
def to_python(self, value, timezone_in_use):
if not isinstance(value, Decimal):
try:
value = Decimal(value)
except:
raise ValueError('Invalid value for %s - %r' % (self.__class__.__name__, value))
if not value.is_finite():
raise ValueError('Non-finite value for %s - %r' % (self.__class__.__name__, value))
return self._round(value)
def test_decimal_from_float(f):
d = Decimal(f)
if isfinite(f) and d.is_finite():
py_d = Real(d)
assert isclose(py_d.decode(), d), (d, f.hex())
else:
with pytest.raises(ValueError, message=repr(f)):
Real(f)
with pytest.raises(ValueError, message=repr(d)):
Real(d)
def convert_decimal(value: Decimal) -> Union[float, int]:
if isinstance(value, Decimal):
if not value.is_finite():
return str(value)
if str(value).find('.') > -1:
return float(value.real)
else:
return int(value)
else:
raise Exception('{} is not Decimal'.format(value))
def doUnit(cur):
id=None
y="0"
f=None
symbol=None
for (k, v) in attrs(cur):
if k == "id": id = v
elif k == "name": name = v
elif k == "eval": e=v
# elif k == "ref": pass
elif k == "href": pass
elif k == "factor": f = v
elif k == "y": y = v
elif k == "symbol": symbol = v
elif k == "no": pass
else: raise RuntimeError("Unexpected attribute %r=%r" % (k, v))
no = doCheckNo(cur)
if not symbol:
no_symbol.append((name, no))
if no in conv_map:
raise RuntimeError("no %X in conv_map" % no)
doCheckName(cur)
if f:
f=Decimal(f)
else:
f=dcalc(e)
assert f.is_finite()
assert (1/f).is_finite()
assert Decimal(y).is_finite()
assert no not in conv_map
assert no not in types_map
doCheckEval(cur)
conv_map[no] = (f, Decimal(y), name)
cur = cur.firstChild
while cur:
x = cur
(cur, t) = (x.nextSibling, x.nodeType)
if t == x.ELEMENT_NODE:
t = x.tagName
if t == "same":
iB = doCheckNo(x, False)
vB = dcalc(x.getAttribute("equals"))
vA = Decimal(x.getAttribute("value") or 1)
same_list.append((no, iB, vB, vA))
else:
raise RuntimeError("unexpected element %r" % x.tagName)
elif t == x.TEXT_NODE:
if x.previousSibling:
if x.data.strip():
raise RuntimeError("unexpected TEXT_NODE %r" % x.data.strip())
elif t in (x.COMMENT_NODE, ):
pass
else:
raise RuntimeError("unexpected node %r %r" % (t, x.nodeName))
def next_unstalled_actions(self, time: Decimal):
if time.is_finite() and time > self.current_plan_execution_limit:
return NoAction.ExecutionLimitReached
elif self.plan.empty():
return NoAction.PlanEmpty
elif self.plan.peek().start_time > time:
return NoAction.TooEarly
all_actions = self.plan.get()
unstalled_actions = [action for action in all_actions if self.is_action_available(action, time)]
return unstalled_actions
def parse_number(s):
parsed = None
try:
parsed = Decimal(s)
if not parsed.is_finite() or parsed > Decimal(
"999999999999999999999999"):
parsed = None
except Exception:
pass
return parsed
def _represent_decimal(self: ruamel.yaml.BaseRepresenter,
data: decimal.Decimal) -> ruamel.yaml.ScalarNode:
if data.is_finite():
s = str(
_POINT_ZERO_DECIMAL + data
) # The zero addition is to force float-like string representation
elif data.is_nan():
s = ".nan"
elif data.is_infinite():
s = ".inf" if data > 0 else "-.inf"
else:
assert False
return self.represent_scalar("tag:yaml.org,2002:float", s)
def decimal_validator(v) -> Decimal:
if isinstance(v, Decimal):
return v
elif isinstance(v, (bytes, bytearray)):
v = v.decode()
v = str(v).strip()
with change_exception(errors.DecimalError, DecimalException):
v = Decimal(v)
if not v.is_finite():
raise errors.DecimalIsNotFiniteError()
return v
def from_decimal(cls, dec):
from decimal import Decimal
if isinstance(dec, numbers.Integral):
dec = Decimal(int(dec))
elif not isinstance(dec, Decimal):
raise TypeError('%s.from_decimal() only takes Decimals, not %r (%s)' % (cls.__name__, dec, type(dec).__name__))
if not dec.is_finite():
raise TypeError('Cannot convert %s to %s.' % (dec, cls.__name__))
(sign, digits, exp) = dec.as_tuple()
digits = int(''.join(map(str, digits)))
if sign:
digits = -digits
if exp >= 0:
return cls(digits*10**exp)
return cls(digits, 10**(-exp))
def xrp_to_drops(xrp: Union[int, float, Decimal]) -> str:
"""
Convert a numeric XRP amount to drops of XRP.
Args:
xrp: Numeric representation of whole XRP
Returns:
Equivalent amount in drops of XRP
Raises:
TypeError: if ``xrp`` is given as a string
XRPRangeException: if the given amount of XRP is invalid
"""
if type(xrp) == str: # type: ignore
# This protects people from passing drops to this function and getting
# a million times as many drops back.
raise TypeError(
"XRP provided as a string. Use a number format" "like Decimal or int."
)
setcontext(_DROPS_CONTEXT)
try:
xrp_d = Decimal(xrp)
except InvalidOperation:
raise XRPRangeException(f"Not a valid amount of XRP: '{xrp}'")
if not xrp_d.is_finite(): # NaN or an Infinity
raise XRPRangeException(f"Not a valid amount of XRP: '{xrp}'")
if xrp_d < ONE_DROP and xrp_d != 0:
raise XRPRangeException(f"XRP amount {xrp} is too small.")
if xrp_d > MAX_XRP:
raise XRPRangeException(f"XRP amount {xrp} is too large.")
drops_amount = (xrp_d / ONE_DROP).quantize(Decimal(1))
drops_str = str(drops_amount).strip()
# This should never happen, but is a precaution against Decimal doing
# something unexpected.
if not fullmatch(_DROPS_REGEX, drops_str):
raise XRPRangeException(
f"xrp_to_drops failed sanity check. Value "
f"'{drops_str}' does not match the drops regex"
)
return drops_str
def from_decimal(cls, dec):
"""Converts a finite Decimal instance to a rational number, exactly."""
from decimal import Decimal
if isinstance(dec, numbers.Integral):
dec = Decimal(int(dec))
elif not isinstance(dec, Decimal):
raise TypeError('%s.from_decimal() only takes Decimals, not %r (%s)' % (cls.__name__, dec, type(dec).__name__))
if not dec.is_finite():
raise TypeError('Cannot convert %s to %s.' % (dec, cls.__name__))
sign, digits, exp = dec.as_tuple()
digits = int(''.join(map(str, digits)))
if sign:
digits = -digits
if exp >= 0:
return cls(digits * 10 ** exp)
else:
return cls(digits, 10 ** (-exp))
def test_decimal_from_float(f):
d = Decimal(f)
if isfinite(f) and d.is_finite():
try:
# PDF is limited to ~5 sig figs
decstr = str(d.quantize(Decimal('1.000000')))
except InvalidOperation:
return # PDF doesn't support exponential notation
try:
py_d = Object.parse(decstr)
except RuntimeError as e:
if 'overflow' in str(e) or 'underflow' in str(e):
py_d = Object.parse(str(f))
assert isclose(py_d, d, abs_tol=1e-5), (d, f.hex())
else:
with pytest.raises(PdfError):
Object.parse(str(d))
def decimal_validator(v) -> Decimal:
if isinstance(v, Decimal):
return v
elif isinstance(v, (bytes, bytearray)):
v = v.decode()
v = str(v).strip()
try:
v = Decimal(v)
except DecimalException as e:
raise TypeError(
f'value is not a valid decimal, got {display_as_type(v)}') from e
if not v.is_finite():
raise TypeError(
f'value is not a valid decimal, got {display_as_type(v)}')
return v
def from_decimal(cls, dec):
"""Converts a finite Decimal instance to a rational number, exactly."""
from decimal import Decimal
if isinstance(dec, numbers.Integral):
dec = Decimal(int(dec))
elif not isinstance(dec, Decimal):
raise TypeError(
"%s.from_decimal() only takes Decimals, not %r (%s)" %
(cls.__name__, dec, type(dec).__name__))
if not dec.is_finite():
# Catches infinities and nans.
raise TypeError("Cannot convert %s to %s." % (dec, cls.__name__))
sign, digits, exp = dec.as_tuple()
digits = int(''.join(map(str, digits)))
if sign:
digits = -digits
if exp >= 0:
return cls(digits * 10 ** exp)
else:
return cls(digits, 10 ** -exp)
def validate(self, value: Decimal) -> Iterable[ValidationError]:
if not value.is_finite(): # check for Inf/NaN/sNaN/qNaN
yield self.validation_error(f"Illegal value in decimal: {value!r}")
decimal_tuple: Optional[DecimalTuple] = None
mdp = self.max_decimal_places
if mdp:
decimal_tuple = value.as_tuple()
if abs(decimal_tuple.exponent) > mdp:
yield self.validation_error(
f"{self.field} must have less than {mdp} decimal places.")
max_digits = self.max_digits
if max_digits:
if decimal_tuple is None:
decimal_tuple = value.as_tuple()
digits = len(decimal_tuple.digits[:decimal_tuple.exponent])
if digits > max_digits:
yield self.validation_error(
f"{self.field} must have less than {max_digits} digits.")
def str_to_decimal(s: str, maxlen: int = DECIMAL_MAXLEN) -> Optional[Decimal]:
"""Convert string to :class:`~decimal.Decimal`.
Args:
s (str): Number to convert.
maxlen (int): Max length of string. Default is 100.
Raises:
ValueError: if length exceeds maximum length, or if value is not
a valid number (e.g. Inf, NaN or sNaN).
Returns:
Decimal: Converted number.
"""
if s is None:
return None
if len(s) > maxlen:
raise ValueError(
f'string of length {len(s)} is longer than limit ({maxlen})')
v = Decimal(s)
if not v.is_finite(): # check for Inf/NaN/sNaN/qNaN
raise ValueError(f'Illegal value in decimal: {s!r}')
return v
def streamingExtensionsLoader(modelXbrl, mappedUri, filepath, **kwargs):
# check if big instance and has header with an initial incomplete tree walk (just 2 elements
if not _streamingExtensionsCheck:
return None
# track whether modelXbrl has been validated by this streaming extension
modelXbrl._streamingExtensionValidated = False
def logSyntaxErrors(parsercontext):
for error in parsercontext.error_log:
modelXbrl.error("xmlSchema:syntax",
_("%(error)s, %(fileName)s, line %(line)s, column %(column)s, %(sourceAction)s source element"),
modelObject=modelXbrl, fileName=os.path.basename(filepath),
error=error.message, line=error.line, column=error.column, sourceAction="streaming")
#### note: written for iterparse of lxml prior to version 3.3, otherwise rewrite to use XmlPullParser ###
#### note: iterparse wants a binary file, but file is text mode
_file, = modelXbrl.fileSource.file(filepath, binary=True)
startedAt = time.time()
modelXbrl.profileActivity()
''' this seems twice as slow as iterparse
class instInfoTarget():
def __init__(self, element_factory=None, parser=None):
self.newTree = True
self.streamingAspects = None
self.foundInstance = False
self.creationSoftwareComment = ''
self.currentEltTag = "(before xbrli:xbrl)"
self.numRootFacts = 0
def start(self, tag, attrib, nsmap=None):
if self.newTree:
if tag == "{http://www.xbrl.org/2003/instance}xbrl":
self.foundInstance = True
self.newTree = False
else: # break
raise NotInstanceDocumentException()
elif not tag.startswith("{http://www.xbrl.org/"):
self.numRootFacts += 1
if self.numRootFacts % 1000 == 0:
modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
self.currentEltTag = tag
def end(self, tag):
pass
def data(self, data):
pass
def comment(self, text):
if not self.foundInstance: # accumulate comments before xbrli:xbrl
self.creationSoftwareComment += ('\n' if self.creationSoftwareComment else '') + text
elif not self.creationSoftwareComment:
self.creationSoftwareComment = text # or first comment after xbrli:xbrl
def pi(self, target, data):
if target == "xbrl-streamable-instance":
if self.currentEltTag == "{http://www.xbrl.org/2003/instance}xbrl":
self.streamingAspects = dict(etree.PI(target,data).attrib.copy()) # dereference target results
else:
modelXbrl.error("streamingExtensions:headerMisplaced",
_("Header is misplaced: %(target)s, must follow xbrli:xbrl element but was found at %(element)s"),
modelObject=modelXbrl, target=target, element=self.currentEltTag)
def close(self):
if not self.creationSoftwareComment:
self.creationSoftwareComment = None
return True
instInfo = instInfoTarget()
infoParser = etree.XMLParser(recover=True, huge_tree=True, target=instInfo)
try:
etree.parse(_file, parser=infoParser, base_url=filepath)
except NotInstanceDocumentException:
pass
'''
foundErrors = False
foundInstance = False
streamingAspects = None
creationSoftwareComment = None
instInfoNumRootFacts = 0
numElts = 0
elt = None
instInfoContext = etree.iterparse(_file, events=("start","end"), huge_tree=True)
for event, elt in instInfoContext:
if event == "start":
if elt.getparent() is not None:
if elt.getparent().tag == "{http://www.xbrl.org/2003/instance}xbrl":
if not foundInstance:
foundInstance = True
pi = precedingProcessingInstruction(elt, "xbrl-streamable-instance")
if pi is None:
break
else:
streamingAspects = dict(pi.attrib.copy())
if creationSoftwareComment is None:
creationSoftwareComment = precedingComment(elt)
if not elt.tag.startswith("{http://www.xbrl.org/"):
instInfoNumRootFacts += 1
if instInfoNumRootFacts % 1000 == 0:
modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
elif not foundInstance:
break
elif elt.tag == "{http://www.xbrl.org/2003/instance}xbrl":
creationSoftwareComment = precedingComment(elt)
if precedingProcessingInstruction(elt, "xbrl-streamable-instance") is not None:
modelXbrl.error("streamingExtensions:headerMisplaced",
_("Header is misplaced: %(error)s, must follow xbrli:xbrl element"),
modelObject=elt)
elif event == "end":
elt.clear()
numElts += 1
if numElts % 1000 == 0 and elt.getparent() is not None:
while elt.getprevious() is not None and elt.getparent() is not None:
del elt.getparent()[0]
if elt is not None:
elt.clear()
_file.seek(0,io.SEEK_SET) # allow reparsing
if not foundInstance or streamingAspects is None:
del elt
_file.close()
return None
modelXbrl.profileStat(_("streaming tree check"), time.time() - startedAt)
startedAt = time.time()
try:
version = Decimal(streamingAspects.get("version"))
if int(version) != 1:
modelXbrl.error("streamingExtensions:unsupportedVersion",
_("Streaming version %(version)s, major version number must be 1"),
modelObject=elt, version=version)
foundErrors = True
except (InvalidOperation, OverflowError):
modelXbrl.error("streamingExtensions:versionError",
_("Version %(version)s, number must be 1.n"),
modelObject=elt, version=streamingAspects.get("version", "(none)"))
foundErrors = True
for bufAspect in ("contextBuffer", "unitBuffer", "footnoteBuffer"):
try:
bufLimit = Decimal(streamingAspects.get(bufAspect, "INF"))
if bufLimit < 1 or (bufLimit.is_finite() and bufLimit % 1 != 0):
raise InvalidOperation
elif bufAspect == "contextBuffer":
contextBufferLimit = bufLimit
elif bufAspect == "unitBuffer":
unitBufferLimit = bufLimit
elif bufAspect == "footnoteBuffer":
footnoteBufferLimit = bufLimit
except InvalidOperation:
modelXbrl.error("streamingExtensions:valueError",
_("Streaming %(attrib)s %(value)s, number must be a positive integer or INF"),
modelObject=elt, attrib=bufAspect, value=streamingAspects.get(bufAspect))
foundErrors = True
if _streamingExtensionsValidate:
incompatibleValidations = []
_validateDisclosureSystem = modelXbrl.modelManager.validateDisclosureSystem
_disclosureSystem = modelXbrl.modelManager.disclosureSystem
if _validateDisclosureSystem and _disclosureSystem.EFM:
incompatibleValidations.append("EFM")
if _validateDisclosureSystem and _disclosureSystem.GFM:
incompatibleValidations.append("GFM")
if _validateDisclosureSystem and _disclosureSystem.EBA:
incompatibleValidations.append("EBA")
if _validateDisclosureSystem and _disclosureSystem.HMRC:
incompatibleValidations.append("EBA")
if modelXbrl.modelManager.validateCalcLB:
incompatibleValidations.append("calculation LB")
if incompatibleValidations:
modelXbrl.error("streamingExtensions:incompatibleValidation",
_("Streaming instance validation does not support %(incompatibleValidations)s validation"),
modelObject=modelXbrl, incompatibleValidations=', '.join(incompatibleValidations))
foundErrors = True
if instInfoContext.error_log:
foundErrors = True
logSyntaxErrors(instInfoContext)
del instInfoContext # dereference
if foundErrors:
_file.close()
return None
_encoding = XmlUtil.encoding(_file.read(512))
_file.seek(0,io.SEEK_SET) # allow reparsing
if _streamingExtensionsValidate:
validator = Validate(modelXbrl)
instValidator = validator.instValidator
eltMdlObjs = {}
contextBuffer = []
unitBuffer = []
footnoteBuffer = []
factBuffer = []
numFacts = 1
class modelLoaderTarget():
def __init__(self, element_factory=None, parser=None):
self.newTree = True
self.currentMdlObj = None
self.beforeInstanceStream = True
self.numRootFacts = 1
def start(self, tag, attrib, nsmap=None):
mdlObj = _parser.makeelement(tag, attrib=attrib, nsmap=nsmap)
mdlObj.sourceline = 1
if self.newTree:
self.newTree = False
self.currentMdlObj = mdlObj
modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, mdlObj.getroottree())
modelXbrl.modelDocument = modelDocument # needed for incremental validation
mdlObj.init(modelDocument)
modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement
modelDocument.parserLookupName = _parserLookupName
modelDocument.parserLookupClass = _parserLookupClass
modelDocument.xmlRootElement = mdlObj
modelDocument.schemaLocationElements.add(mdlObj)
modelDocument.documentEncoding = _encoding
modelDocument._creationSoftwareComment = creationSoftwareComment
modelXbrl.info("streamingExtensions:streaming",
_("Stream processing this instance."),
modelObject = modelDocument)
else:
self.currentMdlObj.append(mdlObj)
self.currentMdlObj = mdlObj
mdlObj._init()
ns = mdlObj.namespaceURI
ln = mdlObj.localName
if (self.beforeInstanceStream and (
(ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or
(ns == XbrlConst.xbrli and ln in ("context", "unit")) or
(ns not in (XbrlConst.link, XbrlConst.xbrli)))):
self.beforeInstanceStream = False
if _streamingExtensionsValidate:
instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters())
else: # need default dimensions
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
return mdlObj
def end(self, tag):
modelDocument = modelXbrl.modelDocument
mdlObj = self.currentMdlObj
parentMdlObj = mdlObj.getparent()
self.currentMdlObj = parentMdlObj
ns = mdlObj.namespaceURI
ln = mdlObj.localName
if ns == XbrlConst.xbrli:
if ln == "context":
if mdlObj.get("sticky"):
del mdlObj.attrib["sticky"]
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.contextDiscover(mdlObj)
else:
if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit:
# drop before adding as dropped may have same id as added
cntx = contextBuffer.pop(0)
dropContext(modelXbrl, cntx)
del parentMdlObj[parentMdlObj.index(cntx)]
cntx = None
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.contextDiscover(mdlObj)
if contextBufferLimit.is_finite():
contextBuffer.append(mdlObj)
if _streamingExtensionsValidate:
contextsToCheck = (mdlObj,)
instValidator.checkContexts(contextsToCheck)
if modelXbrl.hasXDT:
instValidator.checkContextsDimensions(contextsToCheck)
del contextsToCheck # dereference
elif ln == "unit":
if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit:
# drop before additing as dropped may have same id as added
unit = unitBuffer.pop(0)
dropUnit(modelXbrl, unit)
del parentMdlObj[parentMdlObj.index(unit)]
unit = None
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.unitDiscover(mdlObj)
if unitBufferLimit.is_finite():
unitBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkUnits( (mdlObj,) )
elif ln == "xbrl": # end of document
# check remaining footnote refs
for footnoteLink in footnoteBuffer:
checkFootnoteHrefs(modelXbrl, footnoteLink)
elif ns == XbrlConst.link:
if ln == "footnoteLink":
XmlValidate.validate(modelXbrl, mdlObj)
footnoteLinks = (mdlObj,)
modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True)
if footnoteBufferLimit.is_finite():
footnoteBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkLinks(footnoteLinks)
if len(footnoteBuffer) > footnoteBufferLimit:
# check that hrefObjects for locators were all satisfied
# drop before addition as dropped may have same id as added
footnoteLink = footnoteBuffer.pop(0)
checkFootnoteHrefs(modelXbrl, footnoteLink)
dropFootnoteLink(modelXbrl, footnoteLink)
del parentMdlObj[parentMdlObj.index(footnoteLink)]
footnoteLink = None
footnoteLinks = None
elif ln in ("schemaRef", "linkbaseRef"):
modelDocument.discoverHref(mdlObj)
elif not modelXbrl.skipDTS:
if ln in ("roleRef", "arcroleRef"):
modelDocument.linkbaseDiscover((mdlObj,), inInstance=True)
elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl:
self.numRootFacts += 1
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.factDiscover(mdlObj, modelXbrl.facts)
if _streamingExtensionsValidate:
factsToCheck = (mdlObj,)
instValidator.checkFacts(factsToCheck)
if modelXbrl.hasXDT:
instValidator.checkFactsDimensions(factsToCheck)
del factsToCheck
dropFact(modelXbrl, mdlObj, modelXbrl.facts)
del parentMdlObj[parentMdlObj.index(mdlObj)]
if self.numRootFacts % 1000 == 0:
modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts,
100.0 * self.numRootFacts / instInfoNumRootFacts),
minTimeToShow=20.0)
return mdlObj
def data(self, data):
self.currentMdlObj.text = data
def comment(self, text):
pass
def pi(self, target, data):
pass
def close(self):
return None
_parser, _parserLookupName, _parserLookupClass = parser(modelXbrl, filepath, target=modelLoaderTarget())
etree.parse(_file, parser=_parser, base_url=filepath)
logSyntaxErrors(_parser)
_file.close()
if _streamingExtensionsValidate and validator is not None:
del instValidator
validator.close()
# track that modelXbrl has been validated by this streaming extension
modelXbrl._streamingExtensionValidated = True
modelXbrl.profileStat(_("streaming complete"), time.time() - startedAt)
return modelXbrl.modelDocument
def streamingExtensionsLoader(modelXbrl, mappedUri, filepath, *args, **kwargs):
# check if big instance and has header with an initial incomplete tree walk (just 2 elements
if not _streamingExtensionsCheck:
return None
# track whether modelXbrl has been validated by this streaming extension
modelXbrl._streamingExtensionValidated = False
def logSyntaxErrors(parsercontext):
for error in parsercontext.error_log:
modelXbrl.error(
"xmlSchema:syntax",
_("%(error)s, %(fileName)s, line %(line)s, column %(column)s, %(sourceAction)s source element"
),
modelObject=modelXbrl,
fileName=os.path.basename(filepath),
error=error.message,
line=error.line,
column=error.column,
sourceAction="streaming")
#### note: written for iterparse of lxml prior to version 3.3, otherwise rewrite to use XmlPullParser ###
#### note: iterparse wants a binary file, but file is text mode
_file, = modelXbrl.fileSource.file(filepath, binary=True)
startedAt = time.time()
modelXbrl.profileActivity()
''' this seems twice as slow as iterparse
class instInfoTarget():
def __init__(self, element_factory=None, parser=None):
self.newTree = True
self.streamingAspects = None
self.foundInstance = False
self.creationSoftwareComment = ''
self.currentEltTag = "(before xbrli:xbrl)"
self.numRootFacts = 0
def start(self, tag, attrib, nsmap=None):
if self.newTree:
if tag == "{http://www.xbrl.org/2003/instance}xbrl":
self.foundInstance = True
self.newTree = False
else: # break
raise NotInstanceDocumentException()
elif not tag.startswith("{http://www.xbrl.org/"):
self.numRootFacts += 1
if self.numRootFacts % 1000 == 0:
modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
self.currentEltTag = tag
def end(self, tag):
pass
def data(self, data):
pass
def comment(self, text):
if not self.foundInstance: # accumulate comments before xbrli:xbrl
self.creationSoftwareComment += ('\n' if self.creationSoftwareComment else '') + text
elif not self.creationSoftwareComment:
self.creationSoftwareComment = text # or first comment after xbrli:xbrl
def pi(self, target, data):
if target == "xbrl-streamable-instance":
if self.currentEltTag == "{http://www.xbrl.org/2003/instance}xbrl":
self.streamingAspects = dict(etree.PI(target,data).attrib.copy()) # dereference target results
else:
modelXbrl.error("streamingExtensions:headerMisplaced",
_("Header is misplaced: %(target)s, must follow xbrli:xbrl element but was found at %(element)s"),
modelObject=modelXbrl, target=target, element=self.currentEltTag)
def close(self):
if not self.creationSoftwareComment:
self.creationSoftwareComment = None
return True
instInfo = instInfoTarget()
infoParser = etree.XMLParser(recover=True, huge_tree=True, target=instInfo)
try:
etree.parse(_file, parser=infoParser, base_url=filepath)
except NotInstanceDocumentException:
pass
'''
foundErrors = False
foundInstance = False
streamingAspects = None
creationSoftwareComment = None
instInfoNumRootFacts = 0
numElts = 0
elt = None
instInfoContext = etree.iterparse(_file,
events=("start", "end"),
huge_tree=True)
try:
for event, elt in instInfoContext:
if event == "start":
if elt.getparent() is not None:
if elt.getparent(
).tag == "{http://www.xbrl.org/2003/instance}xbrl":
if not foundInstance:
foundInstance = True
pi = precedingProcessingInstruction(
elt, "xbrl-streamable-instance")
if pi is None:
break
else:
streamingAspects = dict(pi.attrib.copy())
if creationSoftwareComment is None:
creationSoftwareComment = precedingComment(
elt)
if not elt.tag.startswith("{http://www.xbrl.org/"):
instInfoNumRootFacts += 1
if instInfoNumRootFacts % 1000 == 0:
modelXbrl.profileActivity(
"... streaming tree check",
minTimeToShow=20.0)
elif not foundInstance:
break
elif elt.tag == "{http://www.xbrl.org/2003/instance}xbrl":
creationSoftwareComment = precedingComment(elt)
if precedingProcessingInstruction(
elt, "xbrl-streamable-instance") is not None:
modelXbrl.error(
"streamingExtensions:headerMisplaced",
_("Header is misplaced: %(error)s, must follow xbrli:xbrl element"
),
modelObject=elt)
elif event == "end":
elt.clear()
numElts += 1
if numElts % 1000 == 0 and elt.getparent() is not None:
while elt.getprevious() is not None and elt.getparent(
) is not None:
del elt.getparent()[0]
except etree.XMLSyntaxError as err:
modelXbrl.error("xmlSchema:syntax",
_("Unrecoverable error: %(error)s"),
error=err)
_file.close()
return err
_file.seek(0, io.SEEK_SET) # allow reparsing
if not foundInstance or streamingAspects is None:
del elt
_file.close()
return None
modelXbrl.profileStat(_("streaming tree check"), time.time() - startedAt)
startedAt = time.time()
try:
version = Decimal(streamingAspects.get("version"))
if int(version) != 1:
modelXbrl.error(
"streamingExtensions:unsupportedVersion",
_("Streaming version %(version)s, major version number must be 1"
),
modelObject=elt,
version=version)
foundErrors = True
except (InvalidOperation, OverflowError):
modelXbrl.error("streamingExtensions:versionError",
_("Version %(version)s, number must be 1.n"),
modelObject=elt,
version=streamingAspects.get("version", "(none)"))
foundErrors = True
for bufAspect in ("contextBuffer", "unitBuffer", "footnoteBuffer"):
try:
bufLimit = Decimal(streamingAspects.get(bufAspect, "INF"))
if bufLimit < 1 or (bufLimit.is_finite() and bufLimit % 1 != 0):
raise InvalidOperation
elif bufAspect == "contextBuffer":
contextBufferLimit = bufLimit
elif bufAspect == "unitBuffer":
unitBufferLimit = bufLimit
elif bufAspect == "footnoteBuffer":
footnoteBufferLimit = bufLimit
except InvalidOperation:
modelXbrl.error(
"streamingExtensions:valueError",
_("Streaming %(attrib)s %(value)s, number must be a positive integer or INF"
),
modelObject=elt,
attrib=bufAspect,
value=streamingAspects.get(bufAspect))
foundErrors = True
if _streamingExtensionsValidate:
incompatibleValidations = []
_validateDisclosureSystem = modelXbrl.modelManager.validateDisclosureSystem
_disclosureSystem = modelXbrl.modelManager.disclosureSystem
if _validateDisclosureSystem and _disclosureSystem.validationType == "EFM":
incompatibleValidations.append("EFM")
if _validateDisclosureSystem and _disclosureSystem.validationType == "GFM":
incompatibleValidations.append("GFM")
if _validateDisclosureSystem and _disclosureSystem.validationType == "HMRC":
incompatibleValidations.append("HMRC")
if modelXbrl.modelManager.validateCalcLB:
incompatibleValidations.append("calculation LB")
if incompatibleValidations:
modelXbrl.error(
"streamingExtensions:incompatibleValidation",
_("Streaming instance validation does not support %(incompatibleValidations)s validation"
),
modelObject=modelXbrl,
incompatibleValidations=', '.join(incompatibleValidations))
foundErrors = True
if instInfoContext.error_log:
foundErrors = True
logSyntaxErrors(instInfoContext)
del instInfoContext # dereference
for pluginMethod in pluginClassMethods("Streaming.BlockStreaming"):
_blockingPluginName = pluginMethod(modelXbrl)
if _blockingPluginName: # name of blocking plugin is returned
modelXbrl.error(
"streamingExtensions:incompatiblePlugIn",
_("Streaming instance not supported by plugin %(blockingPlugin)s"
),
modelObject=modelXbrl,
blockingPlugin=_blockingPluginName)
foundErrors = True
if foundErrors:
_file.close()
return None
_encoding = XmlUtil.encoding(_file.read(512))
_file.seek(0, io.SEEK_SET) # allow reparsing
if _streamingExtensionsValidate:
validator = Validate(modelXbrl)
instValidator = validator.instValidator
contextBuffer = []
contextsToDrop = []
unitBuffer = []
unitsToDrop = []
footnoteBuffer = []
footnoteLinksToDrop = []
_streamingFactsPlugin = any(
True for pluginMethod in pluginClassMethods("Streaming.Facts"))
_streamingValidateFactsPlugin = (_streamingExtensionsValidate and any(
True
for pluginMethod in pluginClassMethods("Streaming.ValidateFacts")))
''' this is very much slower than iterparse
class modelLoaderTarget():
def __init__(self, element_factory=None, parser=None):
self.newTree = True
self.currentMdlObj = None
self.beforeInstanceStream = True
self.beforeStartStreamingPlugin = True
self.numRootFacts = 1
modelXbrl.makeelementParentModelObject = None
modelXbrl.isStreamingMode = True
self.factsCheckVersion = None
self.factsCheckMd5s = Md5Sum()
def start(self, tag, attrib, nsmap=None):
modelXbrl.makeelementParentModelObject = self.currentMdlObj # pass parent to makeelement for ModelObjectFactory
mdlObj = _parser.makeelement(tag, attrib=attrib, nsmap=nsmap)
mdlObj.sourceline = 1
if self.newTree:
self.newTree = False
self.currentMdlObj = mdlObj
modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, mdlObj.getroottree())
modelXbrl.modelDocument = modelDocument # needed for incremental validation
mdlObj.init(modelDocument)
modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement
modelDocument.parserLookupName = _parserLookupName
modelDocument.parserLookupClass = _parserLookupClass
modelDocument.xmlRootElement = mdlObj
modelDocument.schemaLocationElements.add(mdlObj)
modelDocument.documentEncoding = _encoding
modelDocument._creationSoftwareComment = creationSoftwareComment
modelXbrl.info("streamingExtensions:streaming",
_("Stream processing this instance."),
modelObject = modelDocument)
else:
self.currentMdlObj.append(mdlObj)
self.currentMdlObj = mdlObj
mdlObj._init()
ns = mdlObj.namespaceURI
ln = mdlObj.localName
if (self.beforeInstanceStream and (
(ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or
(ns == XbrlConst.xbrli and ln in ("context", "unit")) or
(ns not in (XbrlConst.link, XbrlConst.xbrli)))):
self.beforeInstanceStream = False
if _streamingExtensionsValidate:
instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters(modelXbrl.prefixedNamespaces))
else: # need default dimensions
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
elif not self.beforeInstanceStream and self.beforeStartStreamingPlugin:
for pluginMethod in pluginClassMethods("Streaming.Start"):
pluginMethod(modelXbrl)
self.beforeStartStreamingPlugin = False
return mdlObj
def end(self, tag):
modelDocument = modelXbrl.modelDocument
mdlObj = self.currentMdlObj
parentMdlObj = mdlObj.getparent()
self.currentMdlObj = parentMdlObj
ns = mdlObj.namespaceURI
ln = mdlObj.localName
if ns == XbrlConst.xbrli:
if ln == "context":
if mdlObj.get("sticky"):
del mdlObj.attrib["sticky"]
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.contextDiscover(mdlObj)
else:
if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit:
# drop before adding as dropped may have same id as added
cntx = contextBuffer.pop(0)
if _streamingValidateFactsPlugin:
contextsToDrop.append(cntx)
else:
dropContext(modelXbrl, cntx)
del parentMdlObj[parentMdlObj.index(cntx)]
cntx = None
#>>XmlValidate.validate(modelXbrl, mdlObj)
#>>modelDocument.contextDiscover(mdlObj)
if contextBufferLimit.is_finite():
contextBuffer.append(mdlObj)
if _streamingExtensionsValidate:
contextsToCheck = (mdlObj,)
instValidator.checkContexts(contextsToCheck)
if modelXbrl.hasXDT:
instValidator.checkContextsDimensions(contextsToCheck)
del contextsToCheck # dereference
elif ln == "unit":
if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit:
# drop before adding as dropped may have same id as added
unit = unitBuffer.pop(0)
if _streamingValidateFactsPlugin:
unitsToDrop.append(unit)
else:
dropUnit(modelXbrl, unit)
del parentMdlObj[parentMdlObj.index(unit)]
unit = None
#>>XmlValidate.validate(modelXbrl, mdlObj)
#>>modelDocument.unitDiscover(mdlObj)
if unitBufferLimit.is_finite():
unitBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkUnits( (mdlObj,) )
elif ln == "xbrl": # end of document
# check remaining batched facts if any
if _streamingValidateFactsPlugin:
# plugin attempts to process batch of all root facts not yet processed (not just current one)
# finish any final batch of facts
if len(modelXbrl.facts) > 0:
factsToCheck = modelXbrl.facts.copy()
factsHaveBeenProcessed = True
# can block facts deletion if required data not yet available, such as numeric unit for DpmDB
for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"):
if not pluginMethod(modelXbrl, factsToCheck):
factsHaveBeenProcessed = False
if factsHaveBeenProcessed:
for fact in factsToCheck:
dropFact(modelXbrl, fact, modelXbrl.facts)
del parentMdlObj[parentMdlObj.index(fact)]
for cntx in contextsToDrop:
dropContext(modelXbrl, cntx)
del parentMdlObj[parentMdlObj.index(cntx)]
for unit in unitsToDrop:
dropUnit(modelXbrl, unit)
del parentMdlObj[parentMdlObj.index(unit)]
for footnoteLink in footnoteLinksToDrop:
dropFootnoteLink(modelXbrl, footnoteLink)
del parentMdlObj[parentMdlObj.index(footnoteLink)]
fact = cntx = unit = footnoteLink = None
del contextsToDrop[:]
del unitsToDrop[:]
del footnoteLinksToDrop[:]
del factsToCheck
# check remaining footnote refs
for footnoteLink in footnoteBuffer:
checkFootnoteHrefs(modelXbrl, footnoteLink)
for pluginMethod in pluginClassMethods("Streaming.Finish"):
pluginMethod(modelXbrl)
elif ns == XbrlConst.link:
if ln == "footnoteLink":
XmlValidate.validate(modelXbrl, mdlObj)
footnoteLinks = (mdlObj,)
modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True)
if footnoteBufferLimit.is_finite():
footnoteBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkLinks(footnoteLinks)
if len(footnoteBuffer) > footnoteBufferLimit:
# check that hrefObjects for locators were all satisfied
# drop before addition as dropped may have same id as added
footnoteLink = footnoteBuffer.pop(0)
checkFootnoteHrefs(modelXbrl, footnoteLink)
if _streamingValidateFactsPlugin:
footnoteLinksToDrop.append(footnoteLink)
else:
dropFootnoteLink(modelXbrl, footnoteLink)
del parentMdlObj[parentMdlObj.index(footnoteLink)]
footnoteLink = None
footnoteLinks = None
elif ln in ("schemaRef", "linkbaseRef"):
modelDocument.discoverHref(mdlObj)
elif not modelXbrl.skipDTS:
if ln in ("roleRef", "arcroleRef"):
modelDocument.linkbaseDiscover((mdlObj,), inInstance=True)
elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl:
self.numRootFacts += 1
#>>XmlValidate.validate(modelXbrl, mdlObj)
#>>modelDocument.factDiscover(mdlObj, modelXbrl.facts)
if self.factsCheckVersion:
self.factCheckFact(mdlObj)
if _streamingExtensionsValidate or _streamingValidateFactsPlugin:
factsToCheck = (mdlObj,) # validate current fact by itself
if _streamingExtensionsValidate:
instValidator.checkFacts(factsToCheck)
if modelXbrl.hasXDT:
instValidator.checkFactsDimensions(factsToCheck)
if _streamingValidateFactsPlugin:
# plugin attempts to process batch of all root facts not yet processed (not just current one)
# use batches of 1000 facts
if len(modelXbrl.facts) > 1000:
factsToCheck = modelXbrl.facts.copy()
factsHaveBeenProcessed = True
# can block facts deletion if required data not yet available, such as numeric unit for DpmDB
for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"):
if not pluginMethod(modelXbrl, factsToCheck):
factsHaveBeenProcessed = False
if factsHaveBeenProcessed:
for fact in factsToCheck:
dropFact(modelXbrl, fact, modelXbrl.facts)
del parentMdlObj[parentMdlObj.index(fact)]
for cntx in contextsToDrop:
dropContext(modelXbrl, cntx)
del parentMdlObj[parentMdlObj.index(cntx)]
for unit in unitsToDrop:
dropUnit(modelXbrl, unit)
del parentMdlObj[parentMdlObj.index(unit)]
for footnoteLink in footnoteLinksToDrop:
dropFootnoteLink(modelXbrl, footnoteLink)
del parentMdlObj[parentMdlObj.index(footnoteLink)]
fact = cntx = unit = footnoteLink = None
del contextsToDrop[:]
del unitsToDrop[:]
del footnoteLinksToDrop[:]
del factsToCheck # dereference fact or batch of facts
else:
dropFact(modelXbrl, mdlObj, modelXbrl.facts) # single fact has been processed
del parentMdlObj[parentMdlObj.index(mdlObj)]
if self.numRootFacts % 1000 == 0:
pass
#modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts,
# 100.0 * self.numRootFacts / instInfoNumRootFacts),
# minTimeToShow=20.0)
gc.collect()
sys.stdout.write ("\rAt fact {} of {} mem {}".format(self.numRootFacts, instInfoNumRootFacts, modelXbrl.modelManager.cntlr.memoryUsed))
return mdlObj
def data(self, data):
self.currentMdlObj.text = data
def comment(self, text):
pass
def pi(self, target, data):
if target == "xbrl-facts-check":
_match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", data)
if _match:
_matchGroups = _match.groups()
if len(_matchGroups) == 2:
if _matchGroups[0] == "version":
self.factsCheckVersion = _matchGroups[1]
elif _matchGroups[0] == "sum-of-fact-md5s":
try:
expectedMd5 = Md5Sum(_matchGroups[1])
if self.factsCheckMd5s != expectedMd5:
modelXbrl.warning("streamingExtensions:xbrlFactsCheckWarning",
_("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"),
modelObject=modelXbrl, expectedMd5=expectedMd5, actualMd5Sum=self.factsCheckMd5s)
else:
modelXbrl.info("info",
_("Successful XBRL facts sum of md5s."),
modelObject=modelXbrl)
except ValueError:
modelXbrl.error("streamingExtensions:xbrlFactsCheckError",
_("Invalid sum-of-md5s %(sumOfMd5)s"),
modelObject=modelXbrl, sumOfMd5=_matchGroups[1])
def close(self):
del modelXbrl.makeelementParentModelObject
return None
def factCheckFact(self, fact):
self.factsCheckMd5s += fact.md5sum
for _tupleFact in fact.modelTupleFacts:
self.factCheckFact(_tupleFact)
_parser, _parserLookupName, _parserLookupClass = parser(modelXbrl, filepath, target=modelLoaderTarget())
etree.parse(_file, parser=_parser, base_url=filepath)
logSyntaxErrors(_parser)
'''
# replace modelLoaderTarget with iterparse (as it now supports CustomElementClassLookup)
streamingParserContext = etree.iterparse(_file,
events=("start", "end"),
huge_tree=True)
from arelle.ModelObjectFactory import setParserElementClassLookup
modelXbrl.isStreamingMode = True # must be set before setting element class lookup
(_parser, _parserLookupName,
_parserLookupClass) = setParserElementClassLookup(streamingParserContext,
modelXbrl)
foundInstance = False
beforeInstanceStream = beforeStartStreamingPlugin = True
numRootFacts = 0
factsCheckVersion = None
def factCheckFact(fact):
modelDocument._factsCheckMd5s += fact.md5sum
for _tupleFact in fact.modelTupleFacts:
factCheckFact(_tupleFact)
for event, mdlObj in streamingParserContext:
if event == "start":
if mdlObj.tag == "{http://www.xbrl.org/2003/instance}xbrl":
modelDocument = ModelDocument(modelXbrl, Type.INSTANCE,
mappedUri, filepath,
mdlObj.getroottree())
modelXbrl.modelDocument = modelDocument # needed for incremental validation
mdlObj.init(modelDocument)
modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement
modelDocument.parserLookupName = _parserLookupName
modelDocument.parserLookupClass = _parserLookupClass
modelDocument.xmlRootElement = mdlObj
modelDocument.schemaLocationElements.add(mdlObj)
modelDocument.documentEncoding = _encoding
modelDocument._creationSoftwareComment = precedingComment(
mdlObj)
modelDocument._factsCheckMd5s = Md5Sum()
modelXbrl.info("streamingExtensions:streaming",
_("Stream processing this instance."),
modelObject=modelDocument)
elif mdlObj.getparent() is not None:
mdlObj._init() # requires discovery as part of start elements
if mdlObj.getparent(
).tag == "{http://www.xbrl.org/2003/instance}xbrl":
if not foundInstance:
foundInstance = True
pi = precedingProcessingInstruction(
mdlObj, "xbrl-facts-check")
if pi is not None:
factsCheckVersion = pi.attrib.get("version", None)
elif not foundInstance:
break
ns = mdlObj.qname.namespaceURI
ln = mdlObj.qname.localName
if beforeInstanceStream:
if ((ns == XbrlConst.link
and ln not in ("schemaRef", "linkbaseRef")) or
(ns == XbrlConst.xbrli and ln in ("context", "unit"))
or (ns not in (XbrlConst.link, XbrlConst.xbrli))):
beforeInstanceStream = False
if _streamingExtensionsValidate:
instValidator.validate(
modelXbrl,
modelXbrl.modelManager.formulaOptions.
typedParameters(modelXbrl.prefixedNamespaces))
else: # need default dimensions
ValidateXbrlDimensions.loadDimensionDefaults(
modelXbrl)
elif not beforeInstanceStream and beforeStartStreamingPlugin:
for pluginMethod in pluginClassMethods("Streaming.Start"):
pluginMethod(modelXbrl)
beforeStartStreamingPlugin = False
elif event == "end":
parentMdlObj = mdlObj.getparent()
ns = mdlObj.namespaceURI
ln = mdlObj.localName
if ns == XbrlConst.xbrli:
if ln == "context":
if mdlObj.get("sticky"):
del mdlObj.attrib["sticky"]
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.contextDiscover(mdlObj)
else:
if len(contextBuffer) >= contextBufferLimit:
# drop before adding as dropped may have same id as added
cntx = contextBuffer.pop(0)
if _streamingFactsPlugin or _streamingValidateFactsPlugin:
contextsToDrop.append(cntx)
else:
dropContext(modelXbrl, cntx)
#>>del parentMdlObj[parentMdlObj.index(cntx)]
cntx = None
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.contextDiscover(mdlObj)
if contextBufferLimit.is_finite():
contextBuffer.append(mdlObj)
if _streamingExtensionsValidate:
contextsToCheck = (mdlObj, )
instValidator.checkContexts(contextsToCheck)
if modelXbrl.hasXDT:
instValidator.checkContextsDimensions(
contextsToCheck)
del contextsToCheck # dereference
elif ln == "unit":
if len(unitBuffer) >= unitBufferLimit:
# drop before additing as dropped may have same id as added
unit = unitBuffer.pop(0)
if _streamingFactsPlugin or _streamingValidateFactsPlugin:
unitsToDrop.append(unit)
else:
dropUnit(modelXbrl, unit)
#>>del parentMdlObj[parentMdlObj.index(unit)]
unit = None
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.unitDiscover(mdlObj)
if unitBufferLimit.is_finite():
unitBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkUnits((mdlObj, ))
elif ln == "xbrl": # end of document
# check remaining batched facts if any
if _streamingFactsPlugin or _streamingValidateFactsPlugin:
# plugin attempts to process batch of all root facts not yet processed (not just current one)
# finish any final batch of facts
if len(modelXbrl.facts) > 0:
factsToCheck = modelXbrl.facts.copy()
# can block facts deletion if required data not yet available, such as numeric unit for DpmDB
if _streamingValidateFactsPlugin:
for pluginMethod in pluginClassMethods(
"Streaming.ValidateFacts"):
pluginMethod(instValidator, factsToCheck)
if _streamingFactsPlugin:
for pluginMethod in pluginClassMethods(
"Streaming.Facts"):
pluginMethod(modelXbrl, factsToCheck)
for fact in factsToCheck:
dropFact(modelXbrl, fact, modelXbrl.facts)
#>>del parentMdlObj[parentMdlObj.index(fact)]
for cntx in contextsToDrop:
dropContext(modelXbrl, cntx)
#>>del parentMdlObj[parentMdlObj.index(cntx)]
for unit in unitsToDrop:
dropUnit(modelXbrl, unit)
#>>del parentMdlObj[parentMdlObj.index(unit)]
for footnoteLink in footnoteLinksToDrop:
dropFootnoteLink(modelXbrl, footnoteLink)
#>>del parentMdlObj[parentMdlObj.index(footnoteLink)]
fact = cntx = unit = footnoteLink = None
del contextsToDrop[:]
del unitsToDrop[:]
del footnoteLinksToDrop[:]
del factsToCheck
# check remaining footnote refs
for footnoteLink in footnoteBuffer:
checkFootnoteHrefs(modelXbrl, footnoteLink)
pi = childProcessingInstruction(mdlObj,
"xbrl-facts-check",
reversed=True)
if pi is not None: # attrib is in .text, not attrib, no idea why!!!
_match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']",
pi.text)
if _match:
_matchGroups = _match.groups()
if len(_matchGroups) == 2:
if _matchGroups[0] == "sum-of-fact-md5s":
try:
expectedMd5 = Md5Sum(_matchGroups[1])
if modelDocument._factsCheckMd5s != expectedMd5:
modelXbrl.warning(
"streamingExtensions:xbrlFactsCheckWarning",
_("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"
),
modelObject=modelXbrl,
expectedMd5=expectedMd5,
actualMd5Sum=modelDocument.
_factsCheckMd5s)
else:
modelXbrl.info(
"info",
_("Successful XBRL facts sum of md5s."
),
modelObject=modelXbrl)
except ValueError:
modelXbrl.error(
"streamingExtensions:xbrlFactsCheckError",
_("Invalid sum-of-md5s %(sumOfMd5)s"
),
modelObject=modelXbrl,
sumOfMd5=_matchGroups[1])
if _streamingValidateFactsPlugin:
for pluginMethod in pluginClassMethods(
"Streaming.ValidateFinish"):
pluginMethod(instValidator)
if _streamingFactsPlugin:
for pluginMethod in pluginClassMethods(
"Streaming.Finish"):
pluginMethod(modelXbrl)
elif ns == XbrlConst.link:
if ln in ("schemaRef", "linkbaseRef"):
modelDocument.discoverHref(
mdlObj,
urlRewritePluginClass=
"ModelDocument.InstanceSchemaRefRewriter")
elif ln in ("roleRef", "arcroleRef"):
modelDocument.linkbaseDiscover((mdlObj, ), inInstance=True)
elif ln == "footnoteLink":
XmlValidate.validate(modelXbrl, mdlObj)
footnoteLinks = (mdlObj, )
modelDocument.linkbaseDiscover(footnoteLinks,
inInstance=True)
if footnoteBufferLimit.is_finite():
footnoteBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkLinks(footnoteLinks)
if len(footnoteBuffer) > footnoteBufferLimit:
# check that hrefObjects for locators were all satisfied
# drop before addition as dropped may have same id as added
footnoteLink = footnoteBuffer.pop(0)
checkFootnoteHrefs(modelXbrl, footnoteLink)
if _streamingValidateFactsPlugin:
footnoteLinksToDrop.append(footnoteLink)
else:
dropFootnoteLink(modelXbrl, footnoteLink)
#>>del parentMdlObj[parentMdlObj.index(footnoteLink)]
footnoteLink = None
footnoteLinks = None
elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl and isinstance(
mdlObj, ModelFact):
numRootFacts += 1
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.factDiscover(mdlObj, modelXbrl.facts)
if factsCheckVersion:
factCheckFact(mdlObj)
if _streamingExtensionsValidate or _streamingFactsPlugin or _streamingValidateFactsPlugin:
factsToCheck = (mdlObj,
) # validate current fact by itself
if _streamingExtensionsValidate:
instValidator.checkFacts(factsToCheck)
if modelXbrl.hasXDT:
instValidator.checkFactsDimensions(factsToCheck)
if _streamingFactsPlugin or _streamingValidateFactsPlugin:
# plugin attempts to process batch of all root facts not yet processed (not just current one)
# use batches of 1000 facts
if len(modelXbrl.facts) > 1000:
factsToCheck = modelXbrl.facts.copy()
# can block facts deletion if required data not yet available, such as numeric unit for DpmDB
if _streamingValidateFactsPlugin:
for pluginMethod in pluginClassMethods(
"Streaming.ValidateFacts"):
pluginMethod(instValidator, factsToCheck)
if _streamingFactsPlugin:
for pluginMethod in pluginClassMethods(
"Streaming.Facts"):
pluginMethod(modelXbrl, factsToCheck)
for fact in factsToCheck:
dropFact(modelXbrl, fact, modelXbrl.facts)
#>>del parentMdlObj[parentMdlObj.index(fact)]
for cntx in contextsToDrop:
dropContext(modelXbrl, cntx)
#>>del parentMdlObj[parentMdlObj.index(cntx)]
for unit in unitsToDrop:
dropUnit(modelXbrl, unit)
#>>del parentMdlObj[parentMdlObj.index(unit)]
for footnoteLink in footnoteLinksToDrop:
dropFootnoteLink(modelXbrl, footnoteLink)
#>>del parentMdlObj[parentMdlObj.index(footnoteLink)]
fact = cntx = unit = footnoteLink = None
del contextsToDrop[:]
del unitsToDrop[:]
del footnoteLinksToDrop[:]
del factsToCheck # dereference fact or batch of facts
else:
dropFact(
modelXbrl, mdlObj,
modelXbrl.facts) # single fact has been processed
#>>del parentMdlObj[parentMdlObj.index(mdlObj)]
if numRootFacts % 1000 == 0:
pass
#modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts,
# 100.0 * self.numRootFacts / instInfoNumRootFacts),
# minTimeToShow=20.0)
#gc.collect()
#sys.stdout.write ("\rAt fact {} of {} mem {}".format(numRootFacts, instInfoNumRootFacts, modelXbrl.modelManager.cntlr.memoryUsed))
if mdlObj is not None:
mdlObj.clear()
del _parser, _parserLookupName, _parserLookupClass
if _streamingExtensionsValidate and validator is not None:
_file.close()
del instValidator
validator.close()
# track that modelXbrl has been validated by this streaming extension
modelXbrl._streamingExtensionValidated = True
modelXbrl.profileStat(_("streaming complete"), time.time() - startedAt)
return modelXbrl.modelDocument
def patch(self, request, *args, **kwargs): # pylint: disable=unused-argument
"""
Take PATCH requests, check permissions, and alter state based on them.
"""
uuid = self.kwargs['uuid']
data = request.data
user = request.user
# We shouldn't save any data until all validation checks out
course_modified = False
course = AuthorizationHelpers.get_course(uuid, user)
if course is None:
raise Http404
if "live" in data:
live = data['live']
if not isinstance(live, bool):
raise ValidationError("live must be a bool")
if not AuthorizationHelpers.can_edit_own_liveness(course, user):
raise ValidationError(
"User doesn't have permission to edit course liveness")
course.live = live
course_modified = True
if "title" in data:
title = data['title']
if not isinstance(title, string_types) or title == '':
raise ValidationError("title must be a non-empty string")
if not AuthorizationHelpers.can_edit_own_content(course, user):
raise ValidationError(
"User doesn't have permission to edit course descriptions or titles"
)
course.title = title
course_modified = True
if "description" in data:
description = data['description']
if not isinstance(description, string_types):
raise ValidationError("description must be a string")
if not AuthorizationHelpers.can_edit_own_content(course, user):
raise ValidationError(
"User doesn't have permission to edit course descriptions or titles"
)
course.description = description
course_modified = True
modules_to_save = {}
if "modules" in data:
if not isinstance(data['modules'], list):
raise ValidationError("modules must be a list of modules")
for module_data in data['modules']:
if not isinstance(module_data, dict):
raise ValidationError("Each module must be an object")
if 'uuid' not in module_data:
raise ValidationError("Missing key uuid")
if module_data['uuid'] in modules_to_save:
raise ValidationError("Duplicate module")
try:
# We've already checked permissions for course above
# so this is a simple object lookup
module = Module.objects.get(uuid=module_data['uuid'],
course=course)
except Module.DoesNotExist:
raise ValidationError(
"Unable to find module {uuid}".format(
uuid=module_data['uuid']))
if 'title' in module_data:
title = module_data['title']
if not isinstance(title, string_types) or title == '':
raise ValidationError(
"title must be a non-empty string")
if not AuthorizationHelpers.can_edit_own_content(
course, user):
raise ValidationError(
"User doesn't have permission to edit course descriptions or titles"
)
module.title = title
modules_to_save[module.uuid] = module
if 'price_without_tax' in module_data:
price_without_tax = module_data['price_without_tax']
if price_without_tax is not None:
if not isinstance(price_without_tax, string_types):
raise ValidationError(
'price_without_tax must be a string')
try:
price_without_tax = Decimal(price_without_tax)
except DecimalException:
raise ValidationError(
'price_without_tax is not a valid number')
if not price_without_tax.is_finite():
raise ValidationError(
'price_without_tax is not a valid number')
if price_without_tax < 0:
raise ValidationError(
'price_without_tax is not a valid number')
if not AuthorizationHelpers.can_edit_own_price(
course, user):
raise ValidationError(
"User doesn't have permission to edit module price"
)
module.price_without_tax = price_without_tax
modules_to_save[module.uuid] = module
if course_modified:
course.save()
for module in modules_to_save.values():
module.save()
return Response(status=200)
class ComplexDecimal(object):
def __init__(self,real,i="0.0"):
if type(real) != Decimal: self.real = Decimal(real)
else: self.real = real
if type(i) != Decimal: self.imaginary = Decimal(i)
else: self.imaginary = i
self.e = constants.e
self.pi = constants.pi
#Basic mathamtical functions ----------------------------------------
def conj(self):
"""Returns the conjugate for a number"""
return ComplexDecimal(self.real,-self.imaginary)
def re(self):
"""Returns real portion"""
return self.real
def im(self):
"""Returns imaginary portion"""
return self.imaginary
def floor(self):
"""Floors a number"""
return ComplexDecimal(self.real.to_integral(), self.imaginary.to_integral())
def ceil(self):
"""Ceilings a number"""
return ComplexDecimal(math.ceil(self.real),math.ceil(self.imaginary))
def round(self,prec=0):
"""Rounds a number to prec decimal places"""
return ComplexDecimal(round(self.real,prec),round(self.imaginary,prec))
def exp(self):
"""Returns e^self"""
if self.imaginary == 0:
return ComplexDecimal(str(self.e ** self.real ))
return ComplexDecimal(self.e**(self.real) * Decimal(self.cos_raw(self.imaginary)), self.e**(self.real) * Decimal(self.sin_raw(self.imaginary)))
def isPrime_raw(self,x):
if x<=1: return False
for i in range(2,int(math.floor(math.sqrt(x))) + 1):
if x%i==0: return False
return True
def isPrime(self):
"""Gets if self is prime"""
if self.imaginary == 0:
return self.isPrime_raw(self.real)
if self.real == 0:
return self.isPrime_raw(abs(self.imaginary))
return self.isPrime_raw(self.real * self.real + self.imaginary*self.imaginary)
def phase(self):
"""Returns angle between x axis"""
return Decimal(math.atan2(self.imaginary , self.real))
def polar(self): #Converts to polar
return (abs(self), self.phase())
def toRect(self,r,phi):
return ComplexDecimal(r) * ComplexDecimal(self.cos_raw(phi) , self.sin_raw(phi))
#Trig --------------------------------------------
#1/functions, log,
def cos_raw(self,x,prec=Decimal("0.0000000000000000000001")):
"""Raw cosine function for real numbers only"""
p = Decimal("0")
s = t = Decimal("1.0")
while abs(t/s) > prec:
p+=Decimal("1")
t = (-t * x * x) / ((Decimal("2") * p - Decimal("1")) * (Decimal("2") * p))
s += t
if round(s,50) in [1,0,-1]: #If it's close enough round to get perfect answers (No one likes 1E-70 as cos(pi))
return Decimal(round(s,50))
return s
def sin_raw(self,x,prec=Decimal("0.00000000000000000000001")):
return self.cos_raw(x+self.pi/Decimal("2"))
def tan_raw(self,x,prec=Decimal("0.00000000000000000000001")):
return self.sin_raw(x,prec) / self.cos_raw(x,prec)
def atan_raw(self,x): #Not relatively accurate
returned = Decimal("0")
for j in range(0,50):
i = Decimal(j)
a = Decimal("-1")**i * x ** (Decimal("2")*i + Decimal("1"))
b = Decimal("2")*i + Decimal("1")
returned += a/b
return returned
def cos(self):
"""Compute cos of self"""
if self.imaginary == 0:
return ComplexDecimal(self.cos_raw(self.real))
try:
returned = ComplexDecimal(
self.cos_raw(self.real) * ComplexDecimal(self.imaginary).cosh().real,
-self.sin_raw(self.real) * ComplexDecimal(self.imaginary).sinh().real
)
returned.imaginary = -returned.imaginary
return returned
except: return ComplexDecimal("inf")
def sin(self):
"""Compute sin of self"""
if self.imaginary == 0:
return ComplexDecimal(self.sin_raw(self.real))
try:
returned = ComplexDecimal(
self.sin_raw(self.real) * ComplexDecimal(self.imaginary).cosh().real,
-self.cos_raw(self.real) * ComplexDecimal(self.imaginary).sinh().real
)
returned.imaginary = -returned.imaginary
return returned
except: return ComplexDecimal("inf")
def tan(self):
"""Compute tangent of self"""
try: return self.sin() / self.cos()
except: return ComplexDecimal("inf")
def acos(self):
"""Attempt to compute arcosine"""
if self.imaginary == 0:
return ComplexDecimal(math.acos(self))
A = ComplexDecimal(((Decimal(1) + self.real)**Decimal(2) + self.imaginary**Decimal(2))**Decimal(0.5) - ((Decimal(1) - self.real)**Decimal(0.5) + self.imaginary**Decimal(2))**Decimal(0.5)) / ComplexDecimal(2)
B = ComplexDecimal(((Decimal(1) + self.real)**Decimal(2) + self.imaginary**Decimal(2))**Decimal(0.5) + ((Decimal(1) - self.real)**Decimal(0.5) + self.imaginary**Decimal(2))**Decimal(0.5)) / ComplexDecimal(2)
return ComplexDecimal(A.acos(), -(B+(B*B - ComplexDecimal(1))**ComplexDecimal(0.5)).ln() )
def asin(self):
"""Attempt to compute arcsine"""
if self.imaginary == 0:
return ComplexDecimal(math.asin(self))
A = ComplexDecimal(((Decimal(1) + self.real)**Decimal(2) + self.imaginary**Decimal(2))**Decimal(0.5) - ((Decimal(1) - self.real)**Decimal(0.5) + self.imaginary**Decimal(2))**Decimal(0.5)) / ComplexDecimal(2)
B = ComplexDecimal(((Decimal(1) + self.real)**Decimal(2) + self.imaginary**Decimal(2))**Decimal(0.5) + ((Decimal(1) - self.real)**Decimal(0.5) + self.imaginary**Decimal(2))**Decimal(0.5)) / ComplexDecimal(2)
return ComplexDecimal(A.asin(), (B+(B*B - ComplexDecimal(1))**ComplexDecimal(0.5)).ln() )
def atan(self):
"""Attempt to compute arctangent"""
result = (ComplexDecimal(0,1)+self) / (ComplexDecimal(0,1) - self)
return ComplexDecimal(0,0.5) * result.ln()
def sinh(self):
"""Hyperbolic sine of self"""
returned = (self.exp() - (-self).exp()) / ComplexDecimal("2")
returned.imaginary = -returned.imaginary
return returned
def cosh(self):
"""Hyperbolic cosine of self"""
returned = (self.exp() + (-self).exp()) / ComplexDecimal("2")
returned.imaginary = -returned.imaginary
return returned
def tanh(self):
"""Hyperbolic tangent of self"""
return self.sinh()/self.cosh()
def acosh(self):
"""Arc hyperbolic cosine"""
returned = self + (self*self - ComplexDecimal(1))**0.5
return returned.ln()
def asinh(self):
"""Arc hyperbolic sine"""
returned = self + (self*self + ComplexDecimal(1))**0.5
return returned.ln()
def atanh(self):
"""Arc hyperbolic tan"""
a = (ComplexDecimal(1)+self).ln() - (ComplexDecimal(1)-self).ln()
return ComplexDecimal(0.5) * a
def ln_raw(self,x):
returned = Decimal("0")
for i in range(0,25):
a = Decimal("1") / (Decimal(i) * Decimal("2") + Decimal("1"))
b = ( (x - Decimal("1"))/(x + Decimal("1")) ) ** (Decimal(i) * Decimal("2") + Decimal("1"))
returned += a*b
return returned*2
def ln(self):
"""Natural logarithim of self"""
if self.imaginary == 0:
try: return ComplexDecimal(self.ln_raw(self.real))
except: pass
p = self.polar()
return ComplexDecimal(self.ln_raw(p[0]), p[1])
def log(self,x=None):
"""Compute log of number in base x"""
if not x: x = self.e
if self.imaginary == 0:
try: return ComplexDecimal(math.log(self.real,x))
except: pass
p = self.polar()
return ComplexDecimal(math.log(p[0],x), p[1])
def log10(self):
"""Compute log base 10"""
if self.imaginary == 0:
return ComplexDecimal(math.log10(self.real))
return self.ln() / ComplexDecimal("10").ln()
#Replace builtins --------------------------------
# power, comparasion,
#To float, int, complex
def __str__(self):
"""Converts to str"""
if self.imaginary == 0:
return str(self.real)
return (str(self.real)+"+"+str(self.imaginary)+"i").replace("+-","-")
def __abs__(self):
"""Absolute value"""
return (self.real**2 + self.imaginary**2).sqrt()
def __add__(self,other):
"""Add 2 numbers together"""
return ComplexDecimal(self.real+other.real, self.imaginary+other.imaginary)
def __sub__(self,other):
"""Subtract 2 numbers"""
return ComplexDecimal(self.real-other.real, self.imaginary-other.imaginary)
def __mul__(self,other):
"""Multiply 2 numbers"""
if self.imaginary == 0 and other.imaginary == 0:
return ComplexDecimal(self.real*other.real,"0")
return ComplexDecimal( self.real*other.real - self.imaginary*other.imaginary,self.real*other.imaginary + self.imaginary*other.real )
def __div__(self,other):
"""Divide 2 numbers"""
if self.imaginary == 0 and other.imaginary == 0:
return ComplexDecimal(self.real/other.real,"0")
a = (self.real*other.real + self.imaginary*other.imaginary)/(other.real*other.real + other.imaginary*other.imaginary)
b = (self.imaginary*other.real - self.real*other.imaginary)/(other.real*other.real + other.imaginary*other.imaginary)
return ComplexDecimal(a,b)
def __truediv__(self,other):
"""Divide 2 numbers"""
if self.imaginary == 0 and other.imaginary == 0:
return ComplexDecimal(self.real/other.real,"0")
a = (self.real*other.real + self.imaginary*other.imaginary)/(other.real*other.real + other.imaginary*other.imaginary)
b = (self.imaginary*other.real - self.real*other.imaginary)/(other.real*other.real + other.imaginary*other.imaginary)
return ComplexDecimal(a,b)
def __neg__(self):
"""Negates a number"""
return ComplexDecimal(-self.real,-self.imaginary)
def __pos__(self):
"""Positive number"""
return ComplexDecimal(abs(self.real),abs(self.imaginary))
def __inverse__(self):
"""1/ number"""
return ComplexDecimal(1)/self
def __mod__(self,other):#a%b = a + b * ciel(-a/b)
"""Modolus"""
return self+ other* ((-self/other).ceil())
#Powers
def __pow__(self,other):
"""Powers :D"""
if self.imaginary == 0 and other.imaginary == 0:
return ComplexDecimal(self.real**other.real)
if other.imaginary == 0:
polar = self.polar()
return self.toRect(polar[0]**other.real, polar[1]*other.real)
elif other.real == 0:
a = ComplexDecimal(self.real); b = ComplexDecimal(self.imaginary)
c = ComplexDecimal(other.real); d = ComplexDecimal(other.imaginary)
x = (-d * (b/a).atan()).exp() * ((a*a+b*b).ln() * d / ComplexDecimal(2)).cos()
y = ComplexDecimal(0,1) * (-d * (b/a).atan()).exp() * ((a*a+b*b).ln() * d / ComplexDecimal(2)).sin()
return x+y
b = other.real; c = other.imaginary
returned = self**(ComplexDecimal(b)) * self**(ComplexDecimal(0,1) * ComplexDecimal(c))
returned.imaginary = -returned.imaginary
return returned
#Additional conversions
def __complex__(self):
"""Convert to complex"""
return complex(float(self.real),float(self.imaginary))
def __int__(self):
"""Convert to int"""
return int(self.real)
def __float__(self):
"""Convert to float"""
return float(self.real)
#Comparasions
def __lt__(self,other): #>
if self.imaginary != 0 or other.imaginary != 0:
raise errors.ComparasionError("Complex comparasion is not supported")
return self.real < other.real
def __le__(self,other): #>=
if self.imaginary != 0 or other.imaginary != 0:
raise errors.ComparasionError("Complex comparasion is not supported")
return self.real <= other.real
def __eq__(self,other): #==
if self.real == other.real and self.imaginary == other.imaginary:
return True
return False
def __ne__(self,other): #!=
return not self.__eq__(other)
def __gt__(self,other): #<
if self.imaginary != 0 or other.imaginary != 0:
raise errors.ComparasionError("Complex comparasion is not supported")
return self.real > other.real
def __ge__(self,other): #<=
if self.imaginary != 0 or other.imaginary != 0:
raise errors.ComparasionError("Complex comparasion is not supported")
return self.real >= other.real
#Some things reimplemented from decimal class for complex ComplexDecimals
#================================================================
def copy_abs(self): return abs(self)
def copy_negate(self): return -self
def copy_sign(self,other):
if other < 0:
return -abs(self)
return abs(self)
def is_finite(self):
return self.real.is_finite() and self.imaginary.is_finite()
def is_infinite(self):
return not self.is_finite()
def is_nan(self):
return self.real.is_nan() or self.imaginary.is_nan()
def is_signed(self):
return self < ComplexDecimal(0)
def is_zero(self):
return self.real.is_zero() and self.imaginary.is_zero()
def radix(self): return Decimal(10)
def sqrt(self):
if self.imaginary.is_zero():
return ComplexDecimal(self.real.sqrt())
return self ** ComplexDecimal("0.5")
def to_eng_string(self):
returned = self.real.to_eng_string() + " + " + self.imaginary.to_eng_string + "i"
return returned.replace(" + -"," - ")
class ComplexDecimal(object):
def __init__(self, real, i="0.0"):
if type(real) != Decimal: self.real = Decimal(real)
else: self.real = real
if type(i) != Decimal: self.imaginary = Decimal(i)
else: self.imaginary = i
self.e = constants.e
self.pi = constants.pi
#Basic mathamtical functions ----------------------------------------
def conj(self):
"""Returns the conjugate for a number"""
return ComplexDecimal(self.real, -self.imaginary)
def re(self):
"""Returns real portion"""
return self.real
def im(self):
"""Returns imaginary portion"""
return self.imaginary
def floor(self):
"""Floors a number"""
return ComplexDecimal(self.real.to_integral(),
self.imaginary.to_integral())
def ceil(self):
"""Ceilings a number"""
return ComplexDecimal(math.ceil(self.real), math.ceil(self.imaginary))
def round(self, prec=0):
"""Rounds a number to prec decimal places"""
return ComplexDecimal(round(self.real, prec),
round(self.imaginary, prec))
def exp(self):
"""Returns e^self"""
if self.imaginary == 0:
return ComplexDecimal(str(self.e**self.real))
return ComplexDecimal(
self.e**(self.real) * Decimal(self.cos_raw(self.imaginary)),
self.e**(self.real) * Decimal(self.sin_raw(self.imaginary)))
def isPrime_raw(self, x):
if x <= 1: return False
for i in range(2, int(math.floor(math.sqrt(x))) + 1):
if x % i == 0: return False
return True
def isPrime(self):
"""Gets if self is prime"""
if self.imaginary == 0:
return self.isPrime_raw(self.real)
if self.real == 0:
return self.isPrime_raw(abs(self.imaginary))
return self.isPrime_raw(self.real * self.real +
self.imaginary * self.imaginary)
def phase(self):
"""Returns angle between x axis"""
return Decimal(math.atan2(self.imaginary, self.real))
def polar(self): #Converts to polar
return (Decimal(abs(self).real), self.phase())
def toRect(self, r, phi):
return ComplexDecimal(r) * ComplexDecimal(self.cos_raw(phi),
self.sin_raw(phi))
#Trig --------------------------------------------
#1/functions, log,
def cos_raw(self, x, prec=Decimal("0.0000000000000000000001")):
"""Raw cosine function for real numbers only"""
# p = Decimal("0")
# s = t = Decimal("1.0")
# while abs(t/s) > prec:
# p+=Decimal("1")
# t = (-t * x * x) / ((Decimal("2") * p - Decimal("1")) * (Decimal("2") * p))
# s += t
# if round(s,50) in [1,0,-1]: #If it's close enough round to get perfect answers (No one likes 1E-70 as cos(pi))
# return Decimal(round(s,50))
# return s
return Decimal(math.cos(x))
def sin_raw(self, x, prec=Decimal("0.00000000000000000000001")):
#return self.cos_raw(x+self.pi/Decimal("2"))
return Decimal(math.sin(x))
def tan_raw(self, x, prec=Decimal("0.00000000000000000000001")):
return self.sin_raw(x, prec) / self.cos_raw(x, prec)
def atan_raw(self, x): #Not relatively accurate
returned = Decimal("0")
for j in range(0, 50):
i = Decimal(j)
a = Decimal("-1")**i * x**(Decimal("2") * i + Decimal("1"))
b = Decimal("2") * i + Decimal("1")
returned += a / b
return returned
def cos(self):
"""Compute cos of self"""
if self.imaginary == 0:
return ComplexDecimal(self.cos_raw(self.real))
try:
returned = ComplexDecimal(
self.cos_raw(self.real) *
ComplexDecimal(self.imaginary).cosh().real,
-self.sin_raw(self.real) *
ComplexDecimal(self.imaginary).sinh().real)
returned.imaginary = -returned.imaginary
return returned
except:
return ComplexDecimal("inf")
def sin(self):
"""Compute sin of self"""
if self.imaginary == 0:
return ComplexDecimal(self.sin_raw(self.real))
try:
returned = ComplexDecimal(
self.sin_raw(self.real) *
ComplexDecimal(self.imaginary).cosh().real,
-self.cos_raw(self.real) *
ComplexDecimal(self.imaginary).sinh().real)
returned.imaginary = -returned.imaginary
return returned
except:
return ComplexDecimal("inf")
def tan(self):
"""Compute tangent of self"""
try:
return self.sin() / self.cos()
except:
return ComplexDecimal("inf")
def acos(self):
"""Attempt to compute arcosine"""
if self.imaginary == 0:
return ComplexDecimal(math.acos(self))
A = ComplexDecimal((
(Decimal(1) + self.real)**Decimal(2) +
self.imaginary**Decimal(2))**Decimal(0.5) - (
(Decimal(1) - self.real)**Decimal(0.5) +
self.imaginary**Decimal(2))**Decimal(0.5)) / ComplexDecimal(2)
B = ComplexDecimal((
(Decimal(1) + self.real)**Decimal(2) +
self.imaginary**Decimal(2))**Decimal(0.5) + (
(Decimal(1) - self.real)**Decimal(0.5) +
self.imaginary**Decimal(2))**Decimal(0.5)) / ComplexDecimal(2)
return ComplexDecimal(
A.acos(),
-(B + (B * B - ComplexDecimal(1))**ComplexDecimal(0.5)).ln())
def asin(self):
"""Attempt to compute arcsine"""
if self.imaginary == 0:
return ComplexDecimal(math.asin(self))
A = ComplexDecimal((
(Decimal(1) + self.real)**Decimal(2) +
self.imaginary**Decimal(2))**Decimal(0.5) - (
(Decimal(1) - self.real)**Decimal(0.5) +
self.imaginary**Decimal(2))**Decimal(0.5)) / ComplexDecimal(2)
B = ComplexDecimal((
(Decimal(1) + self.real)**Decimal(2) +
self.imaginary**Decimal(2))**Decimal(0.5) + (
(Decimal(1) - self.real)**Decimal(0.5) +
self.imaginary**Decimal(2))**Decimal(0.5)) / ComplexDecimal(2)
return ComplexDecimal(
A.asin(),
(B + (B * B - ComplexDecimal(1))**ComplexDecimal(0.5)).ln())
def atan(self):
"""Attempt to compute arctangent"""
result = (ComplexDecimal(0, 1) + self) / (ComplexDecimal(0, 1) - self)
return ComplexDecimal(0, 0.5) * result.ln()
def sinh(self):
"""Hyperbolic sine of self"""
returned = (self.exp() - (-self).exp()) / ComplexDecimal("2")
returned.imaginary = -returned.imaginary
return returned
def cosh(self):
"""Hyperbolic cosine of self"""
returned = (self.exp() + (-self).exp()) / ComplexDecimal("2")
returned.imaginary = -returned.imaginary
return returned
def tanh(self):
"""Hyperbolic tangent of self"""
return self.sinh() / self.cosh()
def acosh(self):
"""Arc hyperbolic cosine"""
returned = self + (self * self - ComplexDecimal(1))**0.5
return returned.ln()
def asinh(self):
"""Arc hyperbolic sine"""
returned = self + (self * self + ComplexDecimal(1))**0.5
return returned.ln()
def atanh(self):
"""Arc hyperbolic tan"""
a = (ComplexDecimal(1) + self).ln() - (ComplexDecimal(1) - self).ln()
return ComplexDecimal(0.5) * a
def ln_raw(self, x):
returned = Decimal("0")
for i in range(0, 25):
a = Decimal("1") / (Decimal(i) * Decimal("2") + Decimal("1"))
b = ((x - Decimal("1")) /
(x + Decimal("1")))**(Decimal(i) * Decimal("2") +
Decimal("1"))
returned += a * b
return returned * 2
def ln(self):
"""Natural logarithim of self"""
if self.imaginary == 0:
try:
return ComplexDecimal(self.ln_raw(self.real))
except:
pass
p = self.polar()
return ComplexDecimal(self.ln_raw(p[0]), p[1])
def log(self, x=None):
"""Compute log of number in base x"""
if not x: x = self.e
if self.imaginary == 0:
try:
return ComplexDecimal(math.log(self.real, x))
except:
pass
p = self.polar()
return ComplexDecimal(math.log(p[0], x), p[1])
def log10(self):
"""Compute log base 10"""
if self.imaginary == 0:
return ComplexDecimal(math.log10(self.real))
return self.ln() / ComplexDecimal("10").ln()
#Replace builtins --------------------------------
# power, comparasion,
#To float, int, complex
def __str__(self):
"""Converts to str"""
if self.imaginary == 0:
return str(self.real)
elif self.real == 0:
if self.imaginary == 1: return "i"
if self.imaginary == -1: return "-i"
return str(self.imaginary) + "i"
return (str(self.real) + "+" + str(self.imaginary) + "i").replace(
"+-", "-")
def __abs__(self):
"""Absolute value"""
return ComplexDecimal((self.real**2 + self.imaginary**2).sqrt())
def __add__(self, other):
"""Add 2 numbers together"""
return ComplexDecimal(self.real + other.real,
self.imaginary + other.imaginary)
def __sub__(self, other):
"""Subtract 2 numbers"""
return ComplexDecimal(self.real - other.real,
self.imaginary - other.imaginary)
def __mul__(self, other):
"""Multiply 2 numbers"""
if self.imaginary == 0 and other.imaginary == 0:
return ComplexDecimal(self.real * other.real, "0")
return ComplexDecimal(
self.real * other.real - self.imaginary * other.imaginary,
self.real * other.imaginary + self.imaginary * other.real)
def __div__(self, other):
"""Divide 2 numbers"""
if self.imaginary == 0 and other.imaginary == 0:
return ComplexDecimal(self.real / other.real, "0")
a = (self.real * other.real + self.imaginary * other.imaginary) / (
other.real * other.real + other.imaginary * other.imaginary)
b = (self.imaginary * other.real - self.real * other.imaginary) / (
other.real * other.real + other.imaginary * other.imaginary)
return ComplexDecimal(a, b)
def __truediv__(self, other):
"""Divide 2 numbers"""
if self.imaginary == 0 and other.imaginary == 0:
return ComplexDecimal(self.real / other.real, "0")
a = (self.real * other.real + self.imaginary * other.imaginary) / (
other.real * other.real + other.imaginary * other.imaginary)
b = (self.imaginary * other.real - self.real * other.imaginary) / (
other.real * other.real + other.imaginary * other.imaginary)
return ComplexDecimal(a, b)
def __neg__(self):
"""Negates a number"""
return ComplexDecimal(-self.real, -self.imaginary)
def __pos__(self):
"""Positive number"""
return ComplexDecimal(abs(self.real), abs(self.imaginary))
def __inverse__(self):
"""1/ number"""
return ComplexDecimal(1) / self
def __mod__(self, other): #a%b = a + b * ciel(-a/b)
"""Modolus"""
return self + other * ((-self / other).ceil())
#Powers
def __pow__(self, other):
"""Powers :D"""
if self.imaginary == 0 and other.imaginary == 0:
return ComplexDecimal(self.real**other.real)
if other.imaginary == 0:
polar = self.polar()
return self.toRect(polar[0]**other.real, polar[1] * other.real)
elif other.real == 0:
a = ComplexDecimal(self.real)
b = ComplexDecimal(self.imaginary)
c = ComplexDecimal(other.real)
d = ComplexDecimal(other.imaginary)
x = (-d * (b / a).atan()).exp() * (
(a * a + b * b).ln() * d / ComplexDecimal(2)).cos()
y = ComplexDecimal(0, 1) * (-d * (b / a).atan()).exp() * (
(a * a + b * b).ln() * d / ComplexDecimal(2)).sin()
return x + y
b = other.real
c = other.imaginary
returned = self**(ComplexDecimal(b)) * self**(ComplexDecimal(0, 1) *
ComplexDecimal(c))
returned.imaginary = -returned.imaginary
return returned
#Additional conversions
def __complex__(self):
"""Convert to complex"""
return complex(float(self.real), float(self.imaginary))
def __int__(self):
"""Convert to int"""
return int(self.real)
def __float__(self):
"""Convert to float"""
return float(self.real)
#Comparasions
def __lt__(self, other): #>
if self.imaginary != 0 or other.imaginary != 0:
raise errors.ComparasionError(
"Complex comparasion is not supported")
return self.real < other.real
def __le__(self, other): #>=
if self.imaginary != 0 or other.imaginary != 0:
raise errors.ComparasionError(
"Complex comparasion is not supported")
return self.real <= other.real
def __eq__(self, other): #==
if self.real == other.real and self.imaginary == other.imaginary:
return True
return False
def __ne__(self, other): #!=
return not self.__eq__(other)
def __gt__(self, other): #<
if self.imaginary != 0 or other.imaginary != 0:
raise errors.ComparasionError(
"Complex comparasion is not supported")
return self.real > other.real
def __ge__(self, other): #<=
if self.imaginary != 0 or other.imaginary != 0:
raise errors.ComparasionError(
"Complex comparasion is not supported")
return self.real >= other.real
#Some things reimplemented from decimal class for complex ComplexDecimals
#================================================================
def copy_abs(self):
return abs(self)
def copy_negate(self):
return -self
def copy_sign(self, other):
if other < 0:
return -abs(self)
return abs(self)
def is_finite(self):
return self.real.is_finite() and self.imaginary.is_finite()
def is_infinite(self):
return not self.is_finite()
def is_nan(self):
return self.real.is_nan() or self.imaginary.is_nan()
def is_signed(self):
return self < ComplexDecimal(0)
def is_zero(self):
return self.real.is_zero() and self.imaginary.is_zero()
def radix(self):
return Decimal(10)
def sqrt(self):
if self.imaginary.is_zero():
try:
return ComplexDecimal(self.real.sqrt())
except:
return ComplexDecimal(0, (-self.real).sqrt())
return self**ComplexDecimal("0.5")
def to_eng_string(self):
returned = self.real.to_eng_string(
) + " + " + self.imaginary.to_eng_string + "i"
return returned.replace(" + -", " - ")
def streamingExtensionsLoader(modelXbrl, mappedUri, filepath):
# check if big instance and has header with an initial incomplete tree walk (just 2 elements
def logSyntaxErrors(parsercontext):
for error in parsercontext.error_log:
modelXbrl.error("xmlSchema:syntax",
_("%(error)s, %(fileName)s, line %(line)s, column %(column)s, %(sourceAction)s source element"),
modelObject=modelDocument, fileName=os.path.basename(filepath),
error=error.message, line=error.line, column=error.column, sourceAction="streaming")
#### note: written for iterparse of lxml prior to version 3.3, otherwise rewrite to use XmlPullParser ###
#### note: iterparse wants a binary file, but file is text mode
_file, = modelXbrl.fileSource.file(filepath, binary=True)
startedAt = time.time()
modelXbrl.profileActivity()
parsercontext = etree.iterparse(_file, events=("start","end"), huge_tree=True)
foundInstance = False
foundErrors = False
streamingAspects = None
numRootFacts1 = 0
numElts = 0
elt = None
for event, elt in parsercontext:
if event == "start":
if elt.getparent() is not None:
if elt.getparent().tag == "{http://www.xbrl.org/2003/instance}xbrl":
if not foundInstance:
foundInstance = True
pi = precedingProcessingInstruction(elt, "xbrl-streamable-instance")
if pi is None:
break
else:
streamingAspects = dict(pi.attrib.copy())
if not elt.tag.startswith("{http://www.xbrl.org/"):
numRootFacts1 += 1
if numRootFacts1 % 1000 == 0:
modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
elif not foundInstance:
break
elif elt.tag == "{http://www.xbrl.org/2003/instance}xbrl" and precedingProcessingInstruction(elt, "xbrl-streamable-instance") is not None:
modelXbrl.error("streamingExtensions:headerMisplaced",
_("Header is misplaced: %(error)s, must follow xbrli:xbrl element"),
modelObject=elt)
elif event == "end":
elt.clear()
numElts += 1
if numElts % 1000 == 0 and elt.getparent() is not None:
while elt.getprevious() is not None and elt.getparent() is not None:
del elt.getparent()[0]
if elt is not None:
elt.clear()
_file.seek(0,io.SEEK_SET) # allow reparsing
if not foundInstance or streamingAspects is None:
del elt, parsercontext
_file.close()
return None
modelXbrl.profileStat(_("streaming tree check"), time.time() - startedAt)
startedAt = time.time()
try:
version = Decimal(streamingAspects.get("version"))
if int(version) != 1:
modelXbrl.error("streamingExtensions:unsupportedVersion",
_("Streaming version %(version)s, major version number must be 1"),
modelObject=elt, version=version)
foundErrors = True
except (InvalidOperation, OverflowError):
modelXbrl.error("streamingExtensions:versionError",
_("Version %(version)s, number must be 1.n"),
modelObject=elt, version=streamingAspects.get("version", "(none)"))
foundErrors = True
for bufAspect in ("contextBuffer", "unitBuffer", "footnoteBuffer"):
try:
bufLimit = Decimal(streamingAspects.get(bufAspect, "INF"))
if bufLimit < 1 or (bufLimit.is_finite() and bufLimit % 1 != 0):
raise InvalidOperation
elif bufAspect == "contextBuffer":
contextBufferLimit = bufLimit
elif bufAspect == "unitBuffer":
unitBufferLimit = bufLimit
elif bufAspect == "footnoteBuffer":
footnoteBufferLimit = bufLimit
except InvalidOperation:
modelXbrl.error("streamingExtensions:valueError",
_("Streaming %(attrib)s %(value)s, number must be a positive integer or INF"),
modelObject=elt, attrib=bufAspect, value=streamingAspects.get(bufAspect))
foundErrors = True
if parsercontext.error_log:
foundErrors = True
logSyntaxErrors(parsercontext)
if foundErrors:
_file.close()
return None
parsercontext = etree.iterparse(_file, events=("start","end"), huge_tree=True)
_parser, _parserLookupName, _parserLookupClass = parser(modelXbrl,filepath)
eltMdlObjs = {}
beforeInstanceStream = True
validator = None
contextBuffer = []
unitBuffer = []
footnoteBuffer = []
factBuffer = []
numFacts = numRootFacts2 = 1
for event, elt in parsercontext:
if event == "start":
mdlObj = _parser.makeelement(elt.tag, attrib=elt.attrib, nsmap=elt.nsmap)
mdlObj.sourceline = elt.sourceline
eltMdlObjs[elt] = mdlObj
if elt.getparent() is None:
modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, etree.ElementTree(mdlObj))
modelDocument.xmlRootElement = mdlObj
modelXbrl.modelDocument = modelDocument # needed for incremental validation
mdlObj.init(modelDocument)
modelXbrl.info("streamingExtensions:streaming",
_("Stream processing this instance."),
modelObject = modelDocument)
else:
eltMdlObjs[elt.getparent()].append(mdlObj)
mdlObj._init()
ns = mdlObj.namespaceURI
ln = mdlObj.localName
if (beforeInstanceStream and (
(ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or
(ns == XbrlConst.xbrli and ln in ("context", "unit")) or
(ns not in (XbrlConst.link, XbrlConst.xbrli)))):
beforeInstanceStream = False
if _streamingExtensionsValidate:
validator = Validate(modelXbrl)
validator.instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters())
else: # need default dimensions
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
mdlObj = None # deref
elif event == "end":
mdlObj = eltMdlObjs.pop(elt)
if elt.text: # text available after child nodes processed
mdlObj.text = elt.text
ns = mdlObj.namespaceURI
ln = mdlObj.localName
parentMdlObj = mdlObj.getparent()
if ns == XbrlConst.xbrli:
if ln == "context":
if mdlObj.get("sticky"):
del mdlObj.attrib["sticky"]
modelDocument.contextDiscover(mdlObj)
else:
if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit:
# drop before adding as dropped may have same id as added
cntx = contextBuffer.pop(0)
dropContext(modelXbrl, cntx)
del parentMdlObj[parentMdlObj.index(cntx)]
cntx = None
modelDocument.contextDiscover(mdlObj)
if contextBufferLimit.is_finite():
contextBuffer.append(mdlObj)
if _streamingExtensionsValidate:
contextsToCheck = (mdlObj,)
validator.instValidator.checkContexts(contextsToCheck)
if modelXbrl.hasXDT:
validator.instValidator.checkContextsDimensions(contextsToCheck)
del contextsToCheck # dereference
elif ln == "unit":
if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit:
# drop before additing as dropped may have same id as added
unit = unitBuffer.pop(0)
dropUnit(modelXbrl, unit)
del parentMdlObj[parentMdlObj.index(unit)]
unit = None
modelDocument.unitDiscover(mdlObj)
if unitBufferLimit.is_finite():
unitBuffer.append(mdlObj)
if _streamingExtensionsValidate:
validator.instValidator.checkUnits( (mdlObj,) )
elif ln == "xbrl": # end of document
# check remaining footnote refs
for footnoteLink in footnoteBuffer:
checkFootnoteHrefs(modelXbrl, footnoteLink)
elt.clear()
elif ns == XbrlConst.link:
if ln in ("schemaRef", "linkbaseRef"):
modelDocument.discoverHref(mdlObj)
elif ln in ("roleRef", "arcroleRef"):
modelDocument.linkbaseDiscover((mdlObj,), inInstance=True)
elif ln == "footnoteLink":
footnoteLinks = (mdlObj,)
modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True)
if footnoteBufferLimit.is_finite():
footnoteBuffer.append(mdlObj)
if _streamingExtensionsValidate:
validator.instValidator.checkLinks(footnoteLinks)
if len(footnoteBuffer) > footnoteBufferLimit:
# check that hrefObjects for locators were all satisfied
# drop before addition as dropped may have same id as added
footnoteLink = footnoteBuffer.pop(0)
checkFootnoteHrefs(modelXbrl, footnoteLink)
dropFootnoteLink(modelXbrl, footnoteLink)
del parentMdlObj[parentMdlObj.index(footnoteLink)]
footnoteLink = None
footnoteLinks = None
elt.clear()
elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl:
numRootFacts2 += 1
modelDocument.factDiscover(mdlObj, modelXbrl.facts)
XmlValidate.validate(modelXbrl, mdlObj)
if _streamingExtensionsValidate:
factsToCheck = (mdlObj,)
validator.instValidator.checkFacts(factsToCheck)
if modelXbrl.hasXDT:
validator.instValidator.checkFactsDimensions(factsToCheck)
del factsToCheck
dropFact(modelXbrl, mdlObj, modelXbrl.facts)
del parentMdlObj[parentMdlObj.index(mdlObj)]
if numRootFacts2 % 1000 == 0:
modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(numRootFacts2, numRootFacts1, 100.0 * numRootFacts2 / numRootFacts1),
minTimeToShow=20.0)
# get rid of root element from iterparse's tree
elt.clear()
while elt.getprevious() is not None: # cleans up any prior siblings
del elt.getparent()[0]
mdlObj = None # deref
logSyntaxErrors(parsercontext)
del parsercontext
if validator is not None:
validator.close()
_file.close()
modelXbrl.profileStat(_("streaming complete"), time.time() - startedAt)
return modelDocument
def streamingExtensionsLoader(modelXbrl, mappedUri, filepath, **kwargs):
# check if big instance and has header with an initial incomplete tree walk (just 2 elements
if not _streamingExtensionsCheck:
return None
# track whether modelXbrl has been validated by this streaming extension
modelXbrl._streamingExtensionValidated = False
def logSyntaxErrors(parsercontext):
for error in parsercontext.error_log:
modelXbrl.error("xmlSchema:syntax",
_("%(error)s, %(fileName)s, line %(line)s, column %(column)s, %(sourceAction)s source element"),
modelObject=modelXbrl, fileName=os.path.basename(filepath),
error=error.message, line=error.line, column=error.column, sourceAction="streaming")
#### note: written for iterparse of lxml prior to version 3.3, otherwise rewrite to use XmlPullParser ###
#### note: iterparse wants a binary file, but file is text mode
_file, = modelXbrl.fileSource.file(filepath, binary=True)
startedAt = time.time()
modelXbrl.profileActivity()
''' this seems twice as slow as iterparse
class instInfoTarget():
def __init__(self, element_factory=None, parser=None):
self.newTree = True
self.streamingAspects = None
self.foundInstance = False
self.creationSoftwareComment = ''
self.currentEltTag = "(before xbrli:xbrl)"
self.numRootFacts = 0
def start(self, tag, attrib, nsmap=None):
if self.newTree:
if tag == "{http://www.xbrl.org/2003/instance}xbrl":
self.foundInstance = True
self.newTree = False
else: # break
raise NotInstanceDocumentException()
elif not tag.startswith("{http://www.xbrl.org/"):
self.numRootFacts += 1
if self.numRootFacts % 1000 == 0:
modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
self.currentEltTag = tag
def end(self, tag):
pass
def data(self, data):
pass
def comment(self, text):
if not self.foundInstance: # accumulate comments before xbrli:xbrl
self.creationSoftwareComment += ('\n' if self.creationSoftwareComment else '') + text
elif not self.creationSoftwareComment:
self.creationSoftwareComment = text # or first comment after xbrli:xbrl
def pi(self, target, data):
if target == "xbrl-streamable-instance":
if self.currentEltTag == "{http://www.xbrl.org/2003/instance}xbrl":
self.streamingAspects = dict(etree.PI(target,data).attrib.copy()) # dereference target results
else:
modelXbrl.error("streamingExtensions:headerMisplaced",
_("Header is misplaced: %(target)s, must follow xbrli:xbrl element but was found at %(element)s"),
modelObject=modelXbrl, target=target, element=self.currentEltTag)
def close(self):
if not self.creationSoftwareComment:
self.creationSoftwareComment = None
return True
instInfo = instInfoTarget()
infoParser = etree.XMLParser(recover=True, huge_tree=True, target=instInfo)
try:
etree.parse(_file, parser=infoParser, base_url=filepath)
except NotInstanceDocumentException:
pass
'''
foundErrors = False
foundInstance = False
streamingAspects = None
creationSoftwareComment = None
instInfoNumRootFacts = 0
numElts = 0
elt = None
instInfoContext = etree.iterparse(_file, events=("start","end"), huge_tree=True)
for event, elt in instInfoContext:
if event == "start":
if elt.getparent() is not None:
if elt.getparent().tag == "{http://www.xbrl.org/2003/instance}xbrl":
if not foundInstance:
foundInstance = True
pi = precedingProcessingInstruction(elt, "xbrl-streamable-instance")
if pi is None:
break
else:
streamingAspects = dict(pi.attrib.copy())
if creationSoftwareComment is None:
creationSoftwareComment = precedingComment(elt)
if not elt.tag.startswith("{http://www.xbrl.org/"):
instInfoNumRootFacts += 1
if instInfoNumRootFacts % 1000 == 0:
modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
elif not foundInstance:
break
elif elt.tag == "{http://www.xbrl.org/2003/instance}xbrl":
creationSoftwareComment = precedingComment(elt)
if precedingProcessingInstruction(elt, "xbrl-streamable-instance") is not None:
modelXbrl.error("streamingExtensions:headerMisplaced",
_("Header is misplaced: %(error)s, must follow xbrli:xbrl element"),
modelObject=elt)
elif event == "end":
elt.clear()
numElts += 1
if numElts % 1000 == 0 and elt.getparent() is not None:
while elt.getprevious() is not None and elt.getparent() is not None:
del elt.getparent()[0]
if elt is not None:
elt.clear()
_file.seek(0,io.SEEK_SET) # allow reparsing
if not foundInstance or streamingAspects is None:
del elt
_file.close()
return None
modelXbrl.profileStat(_("streaming tree check"), time.time() - startedAt)
startedAt = time.time()
try:
version = Decimal(streamingAspects.get("version"))
if int(version) != 1:
modelXbrl.error("streamingExtensions:unsupportedVersion",
_("Streaming version %(version)s, major version number must be 1"),
modelObject=elt, version=version)
foundErrors = True
except (InvalidOperation, OverflowError):
modelXbrl.error("streamingExtensions:versionError",
_("Version %(version)s, number must be 1.n"),
modelObject=elt, version=streamingAspects.get("version", "(none)"))
foundErrors = True
for bufAspect in ("contextBuffer", "unitBuffer", "footnoteBuffer"):
try:
bufLimit = Decimal(streamingAspects.get(bufAspect, "INF"))
if bufLimit < 1 or (bufLimit.is_finite() and bufLimit % 1 != 0):
raise InvalidOperation
elif bufAspect == "contextBuffer":
contextBufferLimit = bufLimit
elif bufAspect == "unitBuffer":
unitBufferLimit = bufLimit
elif bufAspect == "footnoteBuffer":
footnoteBufferLimit = bufLimit
except InvalidOperation:
modelXbrl.error("streamingExtensions:valueError",
_("Streaming %(attrib)s %(value)s, number must be a positive integer or INF"),
modelObject=elt, attrib=bufAspect, value=streamingAspects.get(bufAspect))
foundErrors = True
if _streamingExtensionsValidate:
incompatibleValidations = []
_validateDisclosureSystem = modelXbrl.modelManager.validateDisclosureSystem
_disclosureSystem = modelXbrl.modelManager.disclosureSystem
if _validateDisclosureSystem and _disclosureSystem.EFM:
incompatibleValidations.append("EFM")
if _validateDisclosureSystem and _disclosureSystem.GFM:
incompatibleValidations.append("GFM")
if _validateDisclosureSystem and _disclosureSystem.EBA:
incompatibleValidations.append("EBA")
if _validateDisclosureSystem and _disclosureSystem.HMRC:
incompatibleValidations.append("EBA")
if modelXbrl.modelManager.validateCalcLB:
incompatibleValidations.append("calculation LB")
if incompatibleValidations:
modelXbrl.error("streamingExtensions:incompatibleValidation",
_("Streaming instance validation does not support %(incompatibleValidations)s validation"),
modelObject=modelXbrl, incompatibleValidations=', '.join(incompatibleValidations))
foundErrors = True
if instInfoContext.error_log:
foundErrors = True
logSyntaxErrors(instInfoContext)
del instInfoContext # dereference
for pluginMethod in pluginClassMethods("Streaming.BlockStreaming"):
_blockingPluginName = pluginMethod(modelXbrl)
if _blockingPluginName: # name of blocking plugin is returned
modelXbrl.error("streamingExtensions:incompatiblePlugIn",
_("Streaming instance not supported by plugin %(blockingPlugin)s"),
modelObject=modelXbrl, blockingPlugin=_blockingPluginName)
foundErrors = True
if foundErrors:
_file.close()
return None
_encoding = XmlUtil.encoding(_file.read(512))
_file.seek(0,io.SEEK_SET) # allow reparsing
if _streamingExtensionsValidate:
validator = Validate(modelXbrl)
instValidator = validator.instValidator
eltMdlObjs = {}
contextBuffer = []
unitBuffer = []
footnoteBuffer = []
factBuffer = []
numFacts = 1
_streamingValidateFactsPlugin = any(True for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"))
class modelLoaderTarget():
def __init__(self, element_factory=None, parser=None):
self.newTree = True
self.currentMdlObj = None
self.beforeInstanceStream = True
self.beforeStartStreamingPlugin = True
self.numRootFacts = 1
modelXbrl.streamingParentModelObject = None
modelXbrl.isStreamingMode = True
def start(self, tag, attrib, nsmap=None):
modelXbrl.streamingParentModelObject = self.currentMdlObj # pass parent to makeelement for ModelObjectFactory
mdlObj = _parser.makeelement(tag, attrib=attrib, nsmap=nsmap)
mdlObj.sourceline = 1
if self.newTree:
self.newTree = False
self.currentMdlObj = mdlObj
modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, mdlObj.getroottree())
modelXbrl.modelDocument = modelDocument # needed for incremental validation
mdlObj.init(modelDocument)
modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement
modelDocument.parserLookupName = _parserLookupName
modelDocument.parserLookupClass = _parserLookupClass
modelDocument.xmlRootElement = mdlObj
modelDocument.schemaLocationElements.add(mdlObj)
modelDocument.documentEncoding = _encoding
modelDocument._creationSoftwareComment = creationSoftwareComment
modelXbrl.info("streamingExtensions:streaming",
_("Stream processing this instance."),
modelObject = modelDocument)
else:
self.currentMdlObj.append(mdlObj)
self.currentMdlObj = mdlObj
mdlObj._init()
ns = mdlObj.namespaceURI
ln = mdlObj.localName
if (self.beforeInstanceStream and (
(ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or
(ns == XbrlConst.xbrli and ln in ("context", "unit")) or
(ns not in (XbrlConst.link, XbrlConst.xbrli)))):
self.beforeInstanceStream = False
if _streamingExtensionsValidate:
instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters())
else: # need default dimensions
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
elif not self.beforeInstanceStream and self.beforeStartStreamingPlugin:
for pluginMethod in pluginClassMethods("Streaming.Start"):
pluginMethod(modelXbrl)
self.beforeStartStreamingPlugin = False
return mdlObj
def end(self, tag):
modelDocument = modelXbrl.modelDocument
mdlObj = self.currentMdlObj
parentMdlObj = mdlObj.getparent()
self.currentMdlObj = parentMdlObj
ns = mdlObj.namespaceURI
ln = mdlObj.localName
if ns == XbrlConst.xbrli:
if ln == "context":
if mdlObj.get("sticky"):
del mdlObj.attrib["sticky"]
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.contextDiscover(mdlObj)
else:
if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit:
# drop before adding as dropped may have same id as added
cntx = contextBuffer.pop(0)
dropContext(modelXbrl, cntx)
del parentMdlObj[parentMdlObj.index(cntx)]
cntx = None
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.contextDiscover(mdlObj)
if contextBufferLimit.is_finite():
contextBuffer.append(mdlObj)
if _streamingExtensionsValidate:
contextsToCheck = (mdlObj,)
instValidator.checkContexts(contextsToCheck)
if modelXbrl.hasXDT:
instValidator.checkContextsDimensions(contextsToCheck)
del contextsToCheck # dereference
elif ln == "unit":
if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit:
# drop before additing as dropped may have same id as added
unit = unitBuffer.pop(0)
dropUnit(modelXbrl, unit)
del parentMdlObj[parentMdlObj.index(unit)]
unit = None
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.unitDiscover(mdlObj)
if unitBufferLimit.is_finite():
unitBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkUnits( (mdlObj,) )
elif ln == "xbrl": # end of document
# check remaining footnote refs
for footnoteLink in footnoteBuffer:
checkFootnoteHrefs(modelXbrl, footnoteLink)
for pluginMethod in pluginClassMethods("Streaming.Finish"):
pluginMethod(modelXbrl)
elif ns == XbrlConst.link:
if ln == "footnoteLink":
XmlValidate.validate(modelXbrl, mdlObj)
footnoteLinks = (mdlObj,)
modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True)
if footnoteBufferLimit.is_finite():
footnoteBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkLinks(footnoteLinks)
if len(footnoteBuffer) > footnoteBufferLimit:
# check that hrefObjects for locators were all satisfied
# drop before addition as dropped may have same id as added
footnoteLink = footnoteBuffer.pop(0)
checkFootnoteHrefs(modelXbrl, footnoteLink)
dropFootnoteLink(modelXbrl, footnoteLink)
del parentMdlObj[parentMdlObj.index(footnoteLink)]
footnoteLink = None
footnoteLinks = None
elif ln in ("schemaRef", "linkbaseRef"):
modelDocument.discoverHref(mdlObj)
elif not modelXbrl.skipDTS:
if ln in ("roleRef", "arcroleRef"):
modelDocument.linkbaseDiscover((mdlObj,), inInstance=True)
elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl:
self.numRootFacts += 1
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.factDiscover(mdlObj, modelXbrl.facts)
if _streamingExtensionsValidate or _streamingValidateFactsPlugin:
factsToCheck = (mdlObj,) # validate current fact by itself
if _streamingExtensionsValidate:
instValidator.checkFacts(factsToCheck)
if modelXbrl.hasXDT:
instValidator.checkFactsDimensions(factsToCheck)
if _streamingValidateFactsPlugin:
# plugin attempts to process batch of all root facts not yet processed (not just current one)
factsToCheck = modelXbrl.facts.copy()
factsHaveBeenProcessed = True
# can block facts deletion if required data not yet available, such as numeric unit for DpmDB
for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"):
if not pluginMethod(modelXbrl, factsToCheck):
factsHaveBeenProcessed = False
if factsHaveBeenProcessed:
for fact in factsToCheck:
dropFact(modelXbrl, fact, modelXbrl.facts)
del parentMdlObj[parentMdlObj.index(fact)]
else:
dropFact(modelXbrl, mdlObj, modelXbrl.facts) # single fact has been processed
del parentMdlObj[parentMdlObj.index(mdlObj)]
del factsToCheck # dereference fact or batch of facts
if self.numRootFacts % 1000 == 0:
modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts,
100.0 * self.numRootFacts / instInfoNumRootFacts),
minTimeToShow=20.0)
return mdlObj
def data(self, data):
self.currentMdlObj.text = data
def comment(self, text):
pass
def pi(self, target, data):
pass
def close(self):
del modelXbrl.streamingParentModelObject
return None
_parser, _parserLookupName, _parserLookupClass = parser(modelXbrl, filepath, target=modelLoaderTarget())
etree.parse(_file, parser=_parser, base_url=filepath)
logSyntaxErrors(_parser)
if _streamingExtensionsValidate and validator is not None:
_file.close()
del instValidator
validator.close()
# track that modelXbrl has been validated by this streaming extension
modelXbrl._streamingExtensionValidated = True
modelXbrl.profileStat(_("streaming complete"), time.time() - startedAt)
return modelXbrl.modelDocument
def delay(self, value):
value = Decimal(value)
if value <= 0 or not value.is_finite():
return
self._delay = value
def delay(self, value):
value = Decimal(value)
if value <= 0 or not value.is_finite():
return
self._delay = value
def decimals(min_value=None,
max_value=None,
allow_nan=None,
allow_infinity=None,
places=None):
"""Generates instances of decimals.Decimal, which may be:
- A finite rational number, between ``min_value`` and ``max_value``.
- Not a Number, if ``allow_nan`` is True. None means "allow NaN, unless
``min__value`` and ``max_value`` are not None".
- Positive or negative infinity, if ``max_value`` and ``min_value``
respectively are None, and ``allow_infinity`` is not False. None means
"allow infinity, unless excluded by the min and max values".
Note that where floats have one `NaN` value, Decimals have four: signed,
and either *quiet* or *signalling*. See `the decimal module docs
<https://docs.python.org/3/library/decimal.html#special-values>`_ for
more information on special values.
If ``places`` is not None, all finite values drawn from the strategy will
have that number of digits after the decimal place.
"""
# Convert min_value and max_value to Decimal values, and validate args
check_valid_integer(places)
if places is not None and places < 0:
raise InvalidArgument('places=%r may not be negative' % places)
if min_value is not None:
min_value = Decimal(min_value)
if min_value.is_infinite() and min_value < 0:
if not (allow_infinity or allow_infinity is None):
raise InvalidArgument('allow_infinity=%r, but min_value=%r' %
(allow_infinity, min_value))
min_value = None
elif not min_value.is_finite():
# This could be positive infinity, quiet NaN, or signalling NaN
raise InvalidArgument(u'Invalid min_value=%r' % min_value)
if max_value is not None:
max_value = Decimal(max_value)
if max_value.is_infinite() and max_value > 0:
if not (allow_infinity or allow_infinity is None):
raise InvalidArgument('allow_infinity=%r, but max_value=%r' %
(allow_infinity, max_value))
max_value = None
elif not max_value.is_finite():
raise InvalidArgument(u'Invalid max_value=%r' % max_value)
check_valid_bound(min_value, 'min_value')
check_valid_bound(max_value, 'max_value')
check_valid_interval(min_value, max_value, 'min_value', 'max_value')
if allow_infinity and (None not in (min_value, max_value)):
raise InvalidArgument('Cannot allow infinity between finite bounds')
# Set up a strategy for finite decimals
if places is not None:
# Fixed-point decimals are basically integers with a scale factor
def try_quantize(d):
try:
return d.quantize(factor)
except InvalidOperation: # pragma: no cover
return None
factor = Decimal(10)**-places
max_num = max_value / factor if max_value is not None else None
min_num = min_value / factor if min_value is not None else None
strat = integers(min_value=min_num, max_value=max_num)\
.map(lambda d: try_quantize(d * factor))\
.filter(lambda d: d is not None)
else:
# Otherwise, they're like fractions featuring a power of ten
strat = fractions(min_value=min_value, max_value=max_value).map(
lambda f: Decimal(f.numerator) / f.denominator)
# Compose with sampled_from for infinities and NaNs as appropriate
special = []
if allow_nan or (allow_nan is None and (None in (min_value, max_value))):
special.extend(map(Decimal, ('NaN', '-NaN', 'sNaN', '-sNaN')))
if allow_infinity or (allow_infinity is max_value is None):
special.append(Decimal('Infinity'))
if allow_infinity or (allow_infinity is min_value is None):
special.append(Decimal('-Infinity'))
return strat | sampled_from(special)
def format_value(self, value, format_target):
"""
Format a value nicely for human-readable output (including rounding).
@param value: the value to format
@param format_target the target the value should be formatted for
@return: a formatted String representation of the given value.
"""
# Only format counts and measures
if self.type.type != ColumnType.count and self.type.type != ColumnType.measure:
return value
if format_target not in POSSIBLE_FORMAT_TARGETS:
raise ValueError("Unknown format target")
if value is None or value == "":
return ""
if isinstance(value, str):
# If the number ends with "s" or another unit, remove it.
# Units should not occur in table cells, but in the table head.
number_str = util.remove_unit(value.strip())
number = Decimal(number_str)
elif isinstance(value, Decimal):
number = value
number_str = util.print_decimal(number)
else:
raise TypeError("Unexpected number type " + str(type(value)))
if number.is_nan():
return "NaN"
elif number == inf:
return "Inf"
elif number == -inf:
return "-Inf"
# Apply the scale factor to the value
if self.scale_factor is not None:
number *= self.scale_factor
assert number.is_finite()
if (self.number_of_significant_digits is None
and self.type.type != ColumnType.measure
and format_target == "tooltip_stochastic"):
# Column of type count (integral values) without specified sig. digits.
# However, we need to round values like stdev, so we just round somehow.
return util.print_decimal(round(number, DEFAULT_TOOLTIP_PRECISION))
number_of_significant_digits = self.get_number_of_significant_digits(
format_target)
max_dec_digits = (self.type.max_decimal_digits
if self.type.type == ColumnType.measure else 0)
if number_of_significant_digits is not None:
current_significant_digits = _get_significant_digits(number_str)
return _format_number(
number,
current_significant_digits,
number_of_significant_digits,
max_dec_digits,
format_target,
)
else:
return util.print_decimal(number)