def updateInstrumentFeatures(self, timeOfUpdate):
self.__totalIter = self.__totalIter + 1
# populate current book data
currentBookDataByFeature = {}
for featureKey in self.__bookDataFeatures:
currentBookDataByFeature[featureKey] = {}
for instrument in self.__instrumentsDict.values():
instrumentId = instrument.getInstrumentId()
currentInstrumentBookData = instrument.getCurrentBookData()
for featureKey in self.__bookDataFeatures:
currentBookDataByFeature[featureKey][instrumentId] = currentInstrumentBookData[featureKey]
for featureKey in self.__bookDataFeatures:
start = time.time()
self.__lookbackInstrumentFeatures.addFeatureValueForAllInstruments(timeOfUpdate, featureKey, currentBookDataByFeature[featureKey])
end = time.time()
diffms = (end - start) * 1000
self.__perfDict[featureKey] = self.__perfDict[featureKey] + diffms
logPerf('Avg time for feature: %s : %.2f' % (featureKey, self.__perfDict[featureKey] / self.__totalIter))
featureConfigs = self.tsParams.getFeatureConfigsForInstrumentType(INSTRUMENT_TYPE_STOCK) # TODO:
featureConfigKeys = [featureConfig.getFeatureKey() for featureConfig in featureConfigs]
for featureConfig in self.__compulsoryInstrumentFeatureConfigs: # TODO: Find a better way to handle compulsory dicts
featureKey = featureConfig.getFeatureKey()
if featureKey in featureConfigKeys:
i = featureConfigKeys.index(featureKey)
customFeatureConfig = featureConfigs.pop(i)
featureConfigKeys.pop(i)
featureConfigs.append(customFeatureConfig)
else:
featureConfigs.append(featureConfig)
featureConfigKeys = [featureConfig.getFeatureKey() for featureConfig in featureConfigs]
for featureConfig in featureConfigs:
start = time.time()
featureKey = featureConfig.getFeatureKey()
featureId = featureConfig.getFeatureId()
featureKey = featureConfig.getFeatureKey()
featureParams = featureConfig.getFeatureParams()
featureCls = FeatureConfig.getClassForFeatureId(featureId)
featureVal = featureCls.computeForInstrument(updateNum=self.__totalIter,
time=timeOfUpdate,
featureParams=featureParams,
featureKey=featureKey,
instrumentManager=self)
self.__lookbackInstrumentFeatures.addFeatureValueForAllInstruments(timeOfUpdate, featureKey, featureVal)
end = time.time()
diffms = (end - start) * 1000
self.__perfDict[featureKey] = self.__perfDict[featureKey] + diffms
logPerf('Avg time for feature: %s : %.2f' % (featureKey, self.__perfDict[featureKey] / self.__totalIter))
def updateFeatures(self, timeOfUpdate):
self.updateInstrumentFeatures(timeOfUpdate)
currentMarketFeatures = {}
self.__lookbackMarketFeatures.addData(timeOfUpdate,
currentMarketFeatures)
featureConfigs = self.tsParams.getMarketFeatureConfigs()
featureConfigKeys = [
featureConfig.getFeatureKey() for featureConfig in featureConfigs
]
for featureConfig in self.__compulsoryFeatureConfigs: # TODO: Find a better way to handle compulsory dicts
featureKey = featureConfig.getFeatureKey()
if featureKey in featureConfigKeys:
i = featureConfigKeys.index(featureKey)
customFeatureConfig = featureConfigs.pop(i)
featureConfigKeys.pop(i)
featureConfigs.append(customFeatureConfig)
else:
featureConfigs.append(featureConfig)
for featureConfig in featureConfigs:
start = time.time()
featureId = featureConfig.getFeatureId()
featureKey = featureConfig.getFeatureKey()
featureParams = featureConfig.getFeatureParams()
featureCls = FeatureConfig.getClassForFeatureId(featureId)
featureVal = featureCls.computeForMarket(
updateNum=self.__totalIter,
time=timeOfUpdate,
featureParams=featureParams,
featureKey=featureKey,
currentMarketFeatures=currentMarketFeatures,
instrumentManager=self)
currentMarketFeatures[featureKey] = featureVal
self.__lookbackMarketFeatures.addFeatureVal(
timeOfUpdate, featureKey, featureVal)
end = time.time()
diffms = (end - start) * 1000
self.__marketPerfDict[
featureKey] = self.__marketPerfDict[featureKey] + diffms
logPerf('Avg time for feature: %s : %.2f' %
(featureKey,
self.__marketPerfDict[featureKey] / self.__totalIter))
logInfo('Market Features: %s' % str(currentMarketFeatures))
def computeInstrumentFeatures(self, writeFeatures=True):
instrumentBookData = self.__dataParser.emitAllInstrumentUpdates()
instrumentIds = self.__dataParser.getInstrumentIds()
for bookDataFeature in self.__bookDataFeatures:
featureDf = pd.concat([instrumentBookData[instrumentId].getBookDataByFeature(bookDataFeature) for instrumentId in instrumentIds], axis=1)
featureDf.columns = instrumentIds
self.__instrumentDataManger.addFeatureValueForAllInstruments(bookDataFeature, featureDf)
# NOTE: copy in pd.concat is set to True. Check what happens when it is False
featureConfigs = self.systemParams.getFeatureConfigsForInstrumentType(INSTRUMENT_TYPE_STOCK)
featureGenerator = self.__instrumentDataManger.getSimulator(self.__chunkSize)
for chunkNumber, timeUpdates in featureGenerator:
self.__totalIter = self.__totalIter + 1
for bookDataFeature in self.__bookDataFeatures:
self.__instrumentDataManger.updateInstrumentDataChunk(bookDataFeature)
for featureConfig in featureConfigs:
featureKey = featureConfig.getFeatureKey()
featureParams = featureConfig.getFeatureParams()
featureId = featureConfig.getFeatureId()
featureCls = FeatureConfig.getClassForFeatureId(featureId)
featureDf = featureCls.computeForInstrumentData(updateNum=chunkNumber,
featureParams=featureParams,
featureKey=featureKey,
featureManager=self)
self.__instrumentDataManger.addFeatureValueChunkForAllInstruments(featureKey, featureDf)
self.__instrumentDataManger.transformInstrumentData()
if writeFeatures:
self.__instrumentDataManger.writeInstrumentData()
self.__instrumentDataManger.dumpInstrumentDataChunk()
if not self.__instrumentDataManger.checkDataIntegrity(chunkNumber):
logWarn("Some data is missing! Check logs")
if self.__chunkSize is None:
self.__instrumentDataManger.cleanup()
else:
self.__instrumentDataManger.cleanup(delInstrumentData=True)