def processAnnouncements(self, announceFileFullPath, archiveURL):
""" Process all company Announcements to the exchange
"""
logger.debug("Processing announcements for file: %s",
announceFileFullPath)
try:
recordPrefixPat = re.compile(
r'([a-zA-Z0-9\(\)¿ \-.^:]{3,})([ ]+[a-zA-Z0-9\-]{2,})([ ]+:)')
with open(announceFileFullPath, 'rt') as fp:
fileRecords = fp.readlines()
fp.close()
logger.debug("Read %s announcements.", len(fileRecords))
for index, announceRecord in enumerate(fileRecords):
try:
if (index > 0 and announceRecord.find(':') > -1 and
announceRecord.lower().find("declaration of nav ")
< 0 and announceRecord.lower().find(
"recommended final dividend") < 0
and announceRecord.lower().find(
"about investor presentation") < 0
and announceRecord.lower().find(
"that the record date has been fixed on") < 0
and announceRecord.lower().find(
"the maturity date of the scheme is") < 0
and announceRecord.lower().find(
"suspension of trading") < 0 and
announceRecord.lower().find("postal ballot") < 0):
recordPrefix = announceRecord[:announceRecord.
find(':') + 1]
announceContent = announceRecord[announceRecord.
find(':') + 1:]
if index + 1 < len(fileRecords) and fileRecords[
index + 1].find(':') == -1:
announceContent = announceContent + " " + fileRecords[
index + 1]
searchRes = recordPrefixPat.search(recordPrefix)
if searchRes is not None:
entityName = searchRes.group(1)
companySymbol = searchRes.group(2).strip()
thisArticle = NewsEvent(
) # make article for each announcement
thisArticle.setPublishDate(self.app_config.rundate)
thisArticle.setModuleName(self.pluginName)
thisArticle.setIndustries([companySymbol])
thisArticle.setTitle('NSE Announcement for ' +
entityName.strip())
thisArticle.setKeyWords([entityName.strip()])
thisArticle.setText(announceContent)
thisArticle.setURL(archiveURL)
# save each article to its unique filename: company symbol, limited to 10 characters
articleUniqueID = str(companySymbol.strip()[:10])
thisArticle.setArticleID(articleUniqueID)
thisArticle.setSource('NSE')
filename = BasePlugin.makeUniqueFileName(
self.pluginName,
self.identifyDataPathForRunDate(
self.baseDirName,
thisArticle.getPublishDate()),
articleUniqueID,
URL=archiveURL)
thisArticle.writeFiles(filename,
self.app_config.data_dir,
announceContent)
else:
logger.debug(
"Skipping record %s as it is not properly formatted.",
index)
except Exception as e:
logger.error("Error processing NSE announcement no %s: %s",
index, e)
except Exception as e:
logger.error("Error processing NSE announcements: %s", e)
def fetchDataFromURL(self, uRLtoFetch, WorkerID):
""" Fetch data From given URL
"""
self.pluginState = Types.STATE_FETCH_CONTENT
fullPathName = ""
dirPathName = ""
rawData = ""
sizeOfDataDownloaded = -1
uncompressSize = 0
publishDateStr = ""
publishDate = None
resultVal = None
self.master_data_dir = self.app_config.master_data_dir
logger.debug("Fetching %s, Worker ID %s", uRLtoFetch.encode("ascii"),
WorkerID)
try:
logging.captureWarnings(True)
(publishDate,
dataUniqueID) = self.extractUniqueIDFromURL(uRLtoFetch)
rawData = self.downloadDataArchive(uRLtoFetch, type(self).__name__)
publishDateStr = str(publishDate.strftime("%Y-%m-%d"))
# write data to file:
fileNameWithOutExt = BasePlugin.makeUniqueFileName(
self.pluginName,
self.identifyDataPathForRunDate(self.baseDirName,
publishDateStr),
dataUniqueID,
URL=uRLtoFetch)
dirPathName = os.path.join(self.app_config.data_dir,
publishDateStr)
fullPathName = os.path.join(dirPathName,
fileNameWithOutExt + ".zip")
sizeOfDataDownloaded = len(rawData)
except Exception as e:
logger.error("Trying to fetch data from given URL: %s", e)
if sizeOfDataDownloaded > self.minArticleLengthInChars:
try:
if os.path.isdir(dirPathName) is False:
# since dir does not exist, so try creating it:
os.mkdir(dirPathName)
except Exception as theError:
logger.error(
"Error creating data directory '%s', Exception was: %s",
dirPathName, theError)
try:
with open(fullPathName, 'wb') as fp:
n = fp.write(rawData)
logger.debug("Wrote %s bytes to file: %s", n, fullPathName)
fp.close()
# save master data:
sizeOfDataDownloaded = self.fetchMasterData(
uRLtoFetch, self.master_data_dir, WorkerID,
sizeOfDataDownloaded)
# save pledges data:
# sizeOfDataDownloaded = sizeOfDataDownloaded + self.fetchPledgesData(self.master_data_dir, publishDate)
uncompressSize = self.parseFetchedData2(
str(publishDate.strftime("%Y%m%d")), fullPathName,
dirPathName, WorkerID, uRLtoFetch)
except Exception as theError:
logger.error(
"Error saving downloaded data to zip file '%s': %s",
fullPathName, theError)
# save metrics/count of downloaded data for the given URL
resultVal = ExecutionResult(uRLtoFetch, sizeOfDataDownloaded,
uncompressSize, publishDateStr,
self.pluginName)
else:
logger.info(
"Ignoring data file '%s' since its size (%s bytes) is less than the minimum of %s bytes",
fullPathName, len(rawData), self.minArticleLengthInChars)
return (resultVal)
