def _durationFunction(node, sphinxContext, args):
hasArg(node, sphinxContext, args, 1)
startArg = args[0]
if isinstance(startArg, str):
startDateTime = Period(None, XmlUtil.datetimeValue(startArg, none=NONE))
elif isinstance(startArg, datetime.datetime):
startDateTime = startArg
elif isinstance(arg, datetime.date):
startDateTime = datetime.date(startArg.year, startArg.month, startArg.day)
endArg = args[1]
if isinstance(endArg, str):
endDateTime = Period(None, XmlUtil.datetimeValue(startArg, addOneDay=True, none=NONE))
elif isinstance(endArg, datetime.datetime):
endDateTime = endArg
elif isinstance(endArg, datetime.date):
endDateTime = datetime.date(endArg.year, endArg.month, endArg.day) + datetime.timedelta(1)
if startDateTime and endDateTime:
return Period(startDateTime, endDateTime)
raise SphinxException(
node,
"sphinx.functionArgumentsMismatch",
_("Function %(name)s requires two argument that are a date or datetime string or value: %(start)s and %(end)s"),
name=node.name,
start=startArg,
end=endArg,
)
def _durationFunction(node, sphinxContext, args):
hasArg(node, sphinxContext, args, 1)
startArg = args[0]
if isinstance(startArg, str):
startDateTime = Period(None, XmlUtil.datetimeValue(startArg,
none=NONE))
elif isinstance(startArg, datetime.datetime):
startDateTime = startArg
elif isinstance(arg, datetime.date):
startDateTime = datetime.date(startArg.year, startArg.month,
startArg.day)
endArg = args[1]
if isinstance(endArg, str):
endDateTime = Period(
None, XmlUtil.datetimeValue(startArg, addOneDay=True, none=NONE))
elif isinstance(endArg, datetime.datetime):
endDateTime = endArg
elif isinstance(endArg, datetime.date):
endDateTime = datetime.date(endArg.year, endArg.month,
endArg.day) + datetime.timedelta(1)
if (startDateTime and endDateTime):
return Period(startDateTime, endDateTime)
raise SphinxException(
node,
"sphinx.functionArgumentsMismatch",
_(
"Function %(name)s requires two argument that are a date or datetime string or value: %(start)s and %(end)s",
),
name=node.name,
start=startArg,
end=endArg)
def setOptions(self):
# set formula options
self.options.entityIdentScheme = self.cellEntityIdentScheme.value
self.options.entityIdentValue = self.cellEntityIdentValue.value
# need datetime.datetime base class for pickling, not ModelValue class (unpicklable)
self.options.startDate = XmlUtil.datetimeValue(self.cellStartDate.value)
self.options.endDate = XmlUtil.datetimeValue(self.cellEndDate.value, addOneDay=True)
self.options.monetaryUnit = self.cellMonetaryUnit.value
self.options.monetaryDecimals = self.cellMonetaryDecimals.value
self.options.nonMonetaryDecimals = self.cellNonMonetaryDecimals.value
def aspectValue(self, xpCtx, aspect, inherit=False):
if aspect == Aspect.DIMENSIONS:
dims = set(self.prefixedNameQname(e.get("dimension"))
for e in XmlUtil.children(self, XbrlConst.euRend, "explicitDimCoord"))
if inherit and self.parentDefinitionNode is not None:
dims |= self.parentDefinitionNode.aspectValue(None, aspect, inherit)
return dims
if inherit and not self.hasAspect(None, aspect):
if self.parentDefinitionNode is not None:
return self.parentDefinitionNode.aspectValue(None, aspect, inherit)
return None
if aspect == Aspect.CONCEPT:
priItem = XmlUtil.childAttr(self, XbrlConst.euRend, "primaryItem", "name")
if priItem is not None:
return self.prefixedNameQname(priItem)
return None
elif aspect == Aspect.PERIOD_TYPE:
if XmlUtil.hasChild(self, XbrlConst.euRend, "timeReference"):
return "instant"
elif aspect == Aspect.INSTANT:
return XmlUtil.datetimeValue(XmlUtil.childAttr(self, XbrlConst.euRend, "timeReference", "instant"),
addOneDay=True)
elif isinstance(aspect, QName):
for e in XmlUtil.children(self, XbrlConst.euRend, "explicitDimCoord"):
if self.prefixedNameQname(e.get("dimension")) == aspect:
return self.prefixedNameQname(e.get("value"))
return None
def instantDatetime(self):
"""(datetime) -- instant attribute, with adjustment to end-of-day midnight as needed"""
try:
return self._instantDatetime
except AttributeError:
self._instantDatetime = XmlUtil.datetimeValue(XmlUtil.child(self.period, XbrlConst.xbrli, "instant"), addOneDay=True)
return self._instantDatetime
def startDatetime(self):
try:
return self._startDatetime
except AttributeError:
self._startDatetime = XmlUtil.datetimeValue(
XmlUtil.child(self.period, XbrlConst.xbrli, "startDate"))
return self._startDatetime
def startDatetime(self):
"""(datetime) -- startDate attribute"""
try:
return self._startDatetime
except AttributeError:
self._startDatetime = XmlUtil.datetimeValue(XmlUtil.child(self.period, XbrlConst.xbrli, "startDate"))
return self._startDatetime
def instantDatetime(self):
try:
return self._instantDatetime
except AttributeError:
self._instantDatetime = XmlUtil.datetimeValue(XmlUtil.child(
self.period, XbrlConst.xbrli, "instant"),
addOneDay=True)
return self._instantDatetime
def characters(self, content):
if self.currentNamespaceURI:
elt = self.qnameStack[0]
if self.currentNamespaceURI == XbrlConst.xbrli:
s = content.strip()
if s:
if self.currentLocalName == "identifier":
elt._entityIdentifier = (elt._entityIdentifier[0], elt._entityIdentifier[1] + content)
elif self.currentLocalName == "startDate":
elt._startDatetime = XmlUtil.datetimeValue(s)
elt._isStartEndPeriod = True
elif self.currentLocalName == "endDate":
elt._endDatetime = XmlUtil.datetimeValue(s, addOneDay=True)
elt._isStartEndPeriod = True
elif self.currentLocalName == "instant":
elt._endDatetime = elt._instantDatetime = XmlUtil.datetimeValue(s, addOneDay=True)
elt._isInstantPeriod = True
elif self.currentLocalName == "measure":
m = self.qname(content)
parentEltLocalName = self.qnameStack[1].localName
if parentEltLocalName == "unit":
self.qnameStack[1]._measures[0].append(m)
elif parentEltLocalName == "unitNumerator" and self.qnameStack[2].localName == "unit":
self.qnameStack[2]._measures[0].append(m)
elif parentEltLocalName == "unitDenominator" and self.qnameStack[2].localName == "unit":
self.qnameStack[2]._measures[1].append(m)
elif self.currentNamespaceURI == XbrlConst.xbrldi:
s = content.strip()
if s:
if self.currentLocalName == "explicitMember" and self.dimensionPrefixedName:
dimQname = self.qname(self.currentLocalName)
memQname = self.qname(s)
dimConcept = self.modelXbrl.qnameConcepts.get(dimQname)
memConcept = self.modelXbrl.qnameConcepts.get(memQname)
elif elt is not None:
elt._elementText += content
def _instantFunction(node, sphinxContext, args):
hasArg(node, sphinxContext, args, 0)
arg = args[0]
if isinstance(arg, str):
instDateTime = Period(None, XmlUtil.datetimeValue(arg, addOneDay=True, none=NONE))
if instDateTime: # none if date is not valid
return instDateTime
elif isinstance(arg, datetime.datetime):
return Period(None, arg)
elif isinstance(arg, datetime.date): # must be turned into midnight of the day reported
return Period(None, datetime.date(arg.year, arg.month, arg.day) + datetime.timedelta(1))
raise SphinxException(
node,
"sphinx.functionArgumentsMismatch",
_("Function %(name)s argument is not a date or datetime string or value %(value)s"),
name=node.name,
value=arg,
)
def setOptions(self):
# set formula options
self.options["feedSource"] = self.cellFeed.value
if self.cellFeed.value in rssFeeds:
self.options["feedSourceUri"] = rssFeeds[self.cellFeed.value]
else:
self.options["feedSourceUri"] = self.cellFeed.value
self.options["matchTextExpr"] = self.cellMatchText.value
self.options["formulaFileUri"] = self.cellFormulaFile.value
self.options["logFileUri"] = self.cellLogFile.value
self.options["emailAddress"] = self.cellEmailAddress.value
if self.cellLatestPubDate.value:
# need datetime.datetime base class for pickling, not ModelValue class (unpicklable)
self.options["latestPubDate"] = XmlUtil.datetimeValue(self.cellLatestPubDate.value)
else:
self.options["latestPubDate"] = None
for checkbox in self.checkboxes:
self.options[checkbox.attr] = checkbox.value
def setOptions(self):
# set formula options
self.options["feedSource"] = self.cellFeed.value
if self.cellFeed.value in rssFeeds:
self.options["feedSourceUri"] = rssFeeds[self.cellFeed.value]
else:
self.options["feedSourceUri"] = self.cellFeed.value
self.options["matchTextExpr"] = self.cellMatchText.value
self.options["formulaFileUri"] = self.cellFormulaFile.value
self.options["logFileUri"] = self.cellLogFile.value
self.options["emailAddress"] = self.cellEmailAddress.value
if self.cellLatestPubDate.value:
# need datetime.datetime base class for pickling, not ModelValue class (unpicklable)
self.options["latestPubDate"] = XmlUtil.datetimeValue(self.cellLatestPubDate.value)
else:
self.options["latestPubDate"] = None
for checkbox in self.checkboxes:
self.options[checkbox.attr] = checkbox.value
def _instantFunction(node, sphinxContext, args):
hasArg(node, sphinxContext, args, 0)
arg = args[0]
if isinstance(arg, str):
instDateTime = Period(
None, XmlUtil.datetimeValue(arg, addOneDay=True, none=NONE))
if instDateTime: # none if date is not valid
return instDateTime
elif isinstance(arg, datetime.datetime):
return Period(None, arg)
elif isinstance(
arg,
datetime.date): # must be turned into midnight of the day reported
return Period(
None,
datetime.date(arg.year, arg.month, arg.day) +
datetime.timedelta(1))
raise SphinxException(
node,
"sphinx.functionArgumentsMismatch",
_("Function %(name)s argument is not a date or datetime string or value %(value)s"
),
name=node.name,
value=arg)
entryFile=dts.uri, instanceFile=dts.modelDocument.basename)
if dts:
dts.saveInstance(overrideFilepath=instanceFilename)
dts.info("info:savedSampleInstance",
_("Instance file written for %(entryFile)s in file %(instanceFile)s."),
modelObject=dts,
entryFile=dts.uri, instanceFile=instanceFilename)
elif not separateLinkroleFiles:
dts.info("info:noSampleInstance",
_("Instance file not written (no presentation line items) for %(entryFile)s in file %(instanceFile)s."),
modelObject=dts,
entryFile=dts.uri, instanceFile=instanceFilename)
sampleDataValues = {
1: {"periodStart": XmlUtil.datetimeValue("2016-01-01"),
"periodEnd": XmlUtil.datetimeValue("2016-03-31", addOneDay=True),
"date": "2016-03-03",
"dateTime": "2016-03-03T12:00:00",
"gYear": "2016",
"gMonth": "--03",
"str": "abc"},
2: {"periodStart": XmlUtil.datetimeValue("2016-04-01"),
"periodEnd": XmlUtil.datetimeValue("2016-06-30", addOneDay=True),
"date": "2016-06-04",
"dateTime": "2016-06-04T13:00:00",
"gMonth": "--06",
"gYear": "2016",
"str": "def"},
3: {"periodStart": XmlUtil.datetimeValue("2016-07-01"),
"periodEnd": XmlUtil.datetimeValue("2016-09-30", addOneDay=True),
def endDateDate(self): # return a date-typed date
"""(datetime) -- date-typed date value of endDate (which is persisted in str form)"""
return XmlUtil.datetimeValue(self.endDate, addOneDay=True)
def startDateDate(self):
"""(datetime) -- date-typed date value of startDate (which is persisted in str form)"""
return XmlUtil.datetimeValue(self.startDate)
def watchCycle(self):
while not self.stopRequested:
rssWatchOptions = self.rssModelXbrl.modelManager.rssWatchOptions
# check rss expiration
rssHeaders = self.cntlr.webCache.getheaders(self.rssModelXbrl.modelManager.rssWatchOptions.get("feedSourceUri"))
expires = parseRfcDatetime(rssHeaders.get("expires"))
reloadNow = True # texpires and expires > datetime.datetime.now()
# reload rss feed
self.rssModelXbrl.reload('checking RSS items', reloadCache=reloadNow)
if self.stopRequested: break
# setup validator
postLoadActions = []
if rssWatchOptions.get("validateDisclosureSystemRules"):
self.instValidator = ValidateFiling.ValidateFiling(self.rssModelXbrl)
postLoadActions.append(_("validating"))
elif rssWatchOptions.get("validateXbrlRules") or rssWatchOptions.get("validateFormulaAssertions"):
self.instValidator = ValidateXbrl.ValidateXbrl(self.rssModelXbrl)
postLoadActions.append(_("validating"))
if (rssWatchOptions.get("validateFormulaAssertions")):
postLoadActions.append(_("running formulas"))
else:
self.instValidator = None
matchTextExpr = rssWatchOptions.get("matchTextExpr")
if matchTextExpr:
matchPattern = re.compile(matchTextExpr)
postLoadActions.append(_("matching text"))
else:
matchPattern= None
postLoadAction = ', '.join(postLoadActions)
# anything to check new filings for
if (rssWatchOptions.get("validateDisclosureSystemRules") or
rssWatchOptions.get("validateXbrlRules") or
rssWatchOptions.get("validateCalcLinkbase") or
rssWatchOptions.get("validateFormulaAssertions") or
rssWatchOptions.get("alertMatchedFactText") or
any(pluginXbrlMethod(rssWatchOptions)
for pluginXbrlMethod in pluginClassMethods("RssWatch.HasWatchAction"))
):
# form keys in ascending order of pubdate
pubDateRssItems = []
for rssItem in self.rssModelXbrl.modelDocument.rssItems:
pubDateRssItems.append((rssItem.pubDate,rssItem.objectId()))
for pubDate, rssItemObjectId in sorted(pubDateRssItems):
rssItem = self.rssModelXbrl.modelObject(rssItemObjectId)
# update ui thread via modelManager (running in background here)
self.rssModelXbrl.modelManager.viewModelObject(self.rssModelXbrl, rssItem.objectId())
if self.stopRequested:
break
latestPubDate = XmlUtil.datetimeValue(rssWatchOptions.get("latestPubDate"))
if (latestPubDate and
rssItem.pubDate < latestPubDate):
continue
try:
# try zipped URL if possible, else expanded instance document
modelXbrl = ModelXbrl.load(self.rssModelXbrl.modelManager,
openFileSource(rssItem.zippedUrl, self.cntlr),
postLoadAction)
if self.stopRequested:
modelXbrl.close()
break
emailAlert = False
if modelXbrl.modelDocument is None:
modelXbrl.error("arelle.rssWatch",
_("RSS item %(company)s %(form)s document not loaded: %(date)s"),
modelXbrl=modelXbrl, company=rssItem.companyName,
form=rssItem.formType, date=rssItem.filingDate)
rssItem.status = "not loadable"
else:
# validate schema, linkbase, or instance
if self.stopRequested:
modelXbrl.close()
break
if self.instValidator:
self.instValidator.validate(modelXbrl)
if modelXbrl.errors and rssWatchOptions.get("alertValiditionError"):
emailAlert = True
for pluginXbrlMethod in pluginClassMethods("RssWatch.DoWatchAction"):
pluginXbrlMethod(modelXbrl, rssWatchOptions, rssItem)
# check match expression
if matchPattern:
for fact in modelXbrl.factsInInstance:
v = fact.value
if v is not None:
m = matchPattern.search(v)
if m:
fr, to = m.span()
msg = _("Fact Variable {0}\n context {1}\n matched text: {2}").format(
fact.qname, fact.contextID, v[max(0,fr-20):to+20])
modelXbrl.info("arelle.rssInfo",
msg,
modelXbrl=modelXbrl) # msg as code passes it through to the status
if rssWatchOptions.get("alertMatchedFactText"):
emailAlert = True
if (rssWatchOptions.get("formulaFileUri") and rssWatchOptions.get("validateFormulaAssertions") and
self.instValidator):
# attach formulas
ModelDocument.load(modelXbrl, rssWatchOptions["formulaFileUri"])
ValidateFormula.validate(self.instValidator)
rssItem.setResults(modelXbrl)
modelXbrl.close()
del modelXbrl # completely dereference
self.rssModelXbrl.modelManager.viewModelObject(self.rssModelXbrl, rssItem.objectId())
if rssItem.assertionUnsuccessful and rssWatchOptions.get("alertAssertionUnsuccessful"):
emailAlert = True
msg = _("Filing CIK {0}\n "
"company {1}\n "
"published {2}\n "
"form type {3}\n "
"filing date {4}\n "
"period {5}\n "
"year end {6}\n "
"results: {7}").format(
rssItem.cikNumber,
rssItem.companyName,
rssItem.pubDate,
rssItem.formType,
rssItem.filingDate,
rssItem.period,
rssItem.fiscalYearEnd,
rssItem.status)
self.rssModelXbrl.info("arelle:rssWatch", msg, modelXbrl=self.rssModelXbrl)
emailAddress = rssWatchOptions.get("emailAddress")
if emailAlert and emailAddress:
self.rssModelXbrl.modelManager.showStatus(_("sending e-mail alert"))
import smtplib
from email.mime.text import MIMEText
emailMsg = MIMEText(msg)
emailMsg["Subject"] = _("Arelle RSS Watch alert on {0}").format(rssItem.companyName)
emailMsg["From"] = emailAddress
emailMsg["To"] = emailAddress
smtp = smtplib.SMTP()
smtp.sendmail(emailAddress, [emailAddress], emailMsg.as_string())
smtp.quit()
self.rssModelXbrl.modelManager.showStatus(_("RSS item {0}, {1} completed, status {2}").format(rssItem.companyName, rssItem.formType, rssItem.status), 3500)
self.rssModelXbrl.modelManager.cntlr.rssWatchUpdateOption(rssItem.pubDate.strftime('%Y-%m-%dT%H:%M:%S'))
except Exception as err:
self.rssModelXbrl.error("arelle.rssError",
_("RSS item %(company)s, %(form)s, %(date)s, exception: %(error)s"),
modelXbrl=self.rssModelXbrl, company=rssItem.companyName,
form=rssItem.formType, date=rssItem.filingDate, error=err,
exc_info=True)
if self.stopRequested: break
if self.stopRequested:
self.cntlr.showStatus(_("RSS watch, stop requested"), 10000)
else:
import time
time.sleep(600)
self.thread = None # close thread
self.stopRequested = False
if not separateLinkroleFiles:
dts.saveInstance(overrideFilepath=instanceFilename)
dts.info("info:savedSampleInstance",
_("Instance file written for %(entryFile)s in file %(instanceFile)s."),
modelObject=dts,
entryFile=dts.uri, instanceFile=instanceFilename)
elif not separateLinkroleFiles:
dts.info("info:noSampleInstance",
_("Instance file not written (no presentation line items) for %(entryFile)s in file %(instanceFile)s."),
modelObject=dts,
entryFile=dts.uri, instanceFile=instanceFilename)
del dts.conceptSampleValue, conceptSampleUnit, conceptSampleScheme
sampleDataValues = {
1: {"periodStart": XmlUtil.datetimeValue("2016-01-01"),
"periodEnd": XmlUtil.datetimeValue("2016-03-31", addOneDay=True),
"date": "2016-03-03",
"dateTime": "2016-03-03T12:00:00",
"duration": "P1D",
"gYear": "2016",
"gMonth": "--03",
"str": "abc"},
2: {"periodStart": XmlUtil.datetimeValue("2016-04-01"),
"periodEnd": XmlUtil.datetimeValue("2016-06-30", addOneDay=True),
"date": "2016-06-04",
"dateTime": "2016-06-04T13:00:00",
"duration": "P1D",
"gMonth": "--06",
"gYear": "2016",
"str": "def"},
def instantDatetime(self):
try:
return self._instantDatetime
except AttributeError:
self._instantDatetime = XmlUtil.datetimeValue(XmlUtil.child(self.period, XbrlConst.xbrli, "instant"), addOneDay=True)
return self._instantDatetime
