def processAnnouncements(self, announceFileFullPath, archiveURL): """ Process all company Announcements to the exchange """ logger.debug("Processing announcements for file: %s", announceFileFullPath) try: recordPrefixPat = re.compile( r'([a-zA-Z0-9\(\)¿ \-.^:]{3,})([ ]+[a-zA-Z0-9\-]{2,})([ ]+:)') with open(announceFileFullPath, 'rt') as fp: fileRecords = fp.readlines() fp.close() logger.debug("Read %s announcements.", len(fileRecords)) for index, announceRecord in enumerate(fileRecords): try: if (index > 0 and announceRecord.find(':') > -1 and announceRecord.lower().find("declaration of nav ") < 0 and announceRecord.lower().find( "recommended final dividend") < 0 and announceRecord.lower().find( "about investor presentation") < 0 and announceRecord.lower().find( "that the record date has been fixed on") < 0 and announceRecord.lower().find( "the maturity date of the scheme is") < 0 and announceRecord.lower().find( "suspension of trading") < 0 and announceRecord.lower().find("postal ballot") < 0): recordPrefix = announceRecord[:announceRecord. find(':') + 1] announceContent = announceRecord[announceRecord. find(':') + 1:] if index + 1 < len(fileRecords) and fileRecords[ index + 1].find(':') == -1: announceContent = announceContent + " " + fileRecords[ index + 1] searchRes = recordPrefixPat.search(recordPrefix) if searchRes is not None: entityName = searchRes.group(1) companySymbol = searchRes.group(2).strip() thisArticle = NewsEvent( ) # make article for each announcement thisArticle.setPublishDate(self.app_config.rundate) thisArticle.setModuleName(self.pluginName) thisArticle.setIndustries([companySymbol]) thisArticle.setTitle('NSE Announcement for ' + entityName.strip()) thisArticle.setKeyWords([entityName.strip()]) thisArticle.setText(announceContent) thisArticle.setURL(archiveURL) # save each article to its unique filename: company symbol, limited to 10 characters articleUniqueID = str(companySymbol.strip()[:10]) thisArticle.setArticleID(articleUniqueID) thisArticle.setSource('NSE') filename = BasePlugin.makeUniqueFileName( self.pluginName, self.identifyDataPathForRunDate( self.baseDirName, thisArticle.getPublishDate()), articleUniqueID, URL=archiveURL) thisArticle.writeFiles(filename, self.app_config.data_dir, announceContent) else: logger.debug( "Skipping record %s as it is not properly formatted.", index) except Exception as e: logger.error("Error processing NSE announcement no %s: %s", index, e) except Exception as e: logger.error("Error processing NSE announcements: %s", e)
def fetchDataFromURL(self, uRLtoFetch, WorkerID): """ Fetch data From given URL """ self.pluginState = Types.STATE_FETCH_CONTENT fullPathName = "" dirPathName = "" rawData = "" sizeOfDataDownloaded = -1 uncompressSize = 0 publishDateStr = "" publishDate = None resultVal = None self.master_data_dir = self.app_config.master_data_dir logger.debug("Fetching %s, Worker ID %s", uRLtoFetch.encode("ascii"), WorkerID) try: logging.captureWarnings(True) (publishDate, dataUniqueID) = self.extractUniqueIDFromURL(uRLtoFetch) rawData = self.downloadDataArchive(uRLtoFetch, type(self).__name__) publishDateStr = str(publishDate.strftime("%Y-%m-%d")) # write data to file: fileNameWithOutExt = BasePlugin.makeUniqueFileName( self.pluginName, self.identifyDataPathForRunDate(self.baseDirName, publishDateStr), dataUniqueID, URL=uRLtoFetch) dirPathName = os.path.join(self.app_config.data_dir, publishDateStr) fullPathName = os.path.join(dirPathName, fileNameWithOutExt + ".zip") sizeOfDataDownloaded = len(rawData) except Exception as e: logger.error("Trying to fetch data from given URL: %s", e) if sizeOfDataDownloaded > self.minArticleLengthInChars: try: if os.path.isdir(dirPathName) is False: # since dir does not exist, so try creating it: os.mkdir(dirPathName) except Exception as theError: logger.error( "Error creating data directory '%s', Exception was: %s", dirPathName, theError) try: with open(fullPathName, 'wb') as fp: n = fp.write(rawData) logger.debug("Wrote %s bytes to file: %s", n, fullPathName) fp.close() # save master data: sizeOfDataDownloaded = self.fetchMasterData( uRLtoFetch, self.master_data_dir, WorkerID, sizeOfDataDownloaded) # save pledges data: # sizeOfDataDownloaded = sizeOfDataDownloaded + self.fetchPledgesData(self.master_data_dir, publishDate) uncompressSize = self.parseFetchedData2( str(publishDate.strftime("%Y%m%d")), fullPathName, dirPathName, WorkerID, uRLtoFetch) except Exception as theError: logger.error( "Error saving downloaded data to zip file '%s': %s", fullPathName, theError) # save metrics/count of downloaded data for the given URL resultVal = ExecutionResult(uRLtoFetch, sizeOfDataDownloaded, uncompressSize, publishDateStr, self.pluginName) else: logger.info( "Ignoring data file '%s' since its size (%s bytes) is less than the minimum of %s bytes", fullPathName, len(rawData), self.minArticleLengthInChars) return (resultVal)