def __init__(self, symbols, targetVariable, models=None):
self.instrumentIds = symbols
self.trainingDataSource = None
self.validationDataSource = None
self.testDataSource = None
self.validationSplit = None
self.targetVariable = None
self.validationSplit = None
self.targetVariable = None
FeatureConfig.setupCustomFeatures(self.getCustomFeatures())
self.__instrumentFeatureConfigs = {}
instrumentFeatureConfigDicts = self.getInstrumentFeatureConfigDicts()
for instrumentType in instrumentFeatureConfigDicts:
if type(instrumentFeatureConfigDicts[instrumentType]) is list:
self.__instrumentFeatureConfigs[instrumentType] = list(
map(lambda x: FeatureConfig(x),
instrumentFeatureConfigDicts[instrumentType]))
elif type(instrumentFeatureConfigDicts[instrumentType]) is dict:
self.__instrumentFeatureConfigs[instrumentType] = {
k: FeatureConfig(v)
for k, v in instrumentFeatureConfigDicts[instrumentType]
}
else:
logError("Unknown type of instruments' feature config dicts")
def __init__(self):
FeatureConfig.setupCustomFeatures(self.getCustomFeatures())
self.__instrumentFeatureConfigs = {}
instrumentFeatureConfigDicts = self.getInstrumentFeatureConfigDicts()
for instrumentType in instrumentFeatureConfigDicts:
self.__instrumentFeatureConfigs[instrumentType] = list(
map(lambda x: FeatureConfig(x),
instrumentFeatureConfigDicts[instrumentType]))
self.__marketFeatureConfigs = list(
map(lambda x: FeatureConfig(x),
self.getMarketFeatureConfigDicts()))
def updateInstrumentFeatures(self, timeOfUpdate):
self.__totalIter = self.__totalIter + 1
# populate current book data
currentBookDataByFeature = {}
for featureKey in self.__bookDataFeatures:
currentBookDataByFeature[featureKey] = {}
for instrument in self.__instrumentsDict.values():
instrumentId = instrument.getInstrumentId()
currentInstrumentBookData = instrument.getCurrentBookData()
for featureKey in self.__bookDataFeatures:
currentBookDataByFeature[featureKey][instrumentId] = currentInstrumentBookData[featureKey]
for featureKey in self.__bookDataFeatures:
start = time.time()
self.__lookbackInstrumentFeatures.addFeatureValueForAllInstruments(timeOfUpdate, featureKey, currentBookDataByFeature[featureKey])
end = time.time()
diffms = (end - start) * 1000
self.__perfDict[featureKey] = self.__perfDict[featureKey] + diffms
logPerf('Avg time for feature: %s : %.2f' % (featureKey, self.__perfDict[featureKey] / self.__totalIter))
featureConfigs = self.tsParams.getFeatureConfigsForInstrumentType(INSTRUMENT_TYPE_STOCK) # TODO:
featureConfigKeys = [featureConfig.getFeatureKey() for featureConfig in featureConfigs]
for featureConfig in self.__compulsoryInstrumentFeatureConfigs: # TODO: Find a better way to handle compulsory dicts
featureKey = featureConfig.getFeatureKey()
if featureKey in featureConfigKeys:
i = featureConfigKeys.index(featureKey)
customFeatureConfig = featureConfigs.pop(i)
featureConfigKeys.pop(i)
featureConfigs.append(customFeatureConfig)
else:
featureConfigs.append(featureConfig)
featureConfigKeys = [featureConfig.getFeatureKey() for featureConfig in featureConfigs]
for featureConfig in featureConfigs:
start = time.time()
featureKey = featureConfig.getFeatureKey()
featureId = featureConfig.getFeatureId()
featureKey = featureConfig.getFeatureKey()
featureParams = featureConfig.getFeatureParams()
featureCls = FeatureConfig.getClassForFeatureId(featureId)
featureVal = featureCls.computeForInstrument(updateNum=self.__totalIter,
time=timeOfUpdate,
featureParams=featureParams,
featureKey=featureKey,
instrumentManager=self)
self.__lookbackInstrumentFeatures.addFeatureValueForAllInstruments(timeOfUpdate, featureKey, featureVal)
end = time.time()
diffms = (end - start) * 1000
self.__perfDict[featureKey] = self.__perfDict[featureKey] + diffms
logPerf('Avg time for feature: %s : %.2f' % (featureKey, self.__perfDict[featureKey] / self.__totalIter))
def updateFeatures(self, timeOfUpdate):
self.updateInstrumentFeatures(timeOfUpdate)
currentMarketFeatures = {}
self.__lookbackMarketFeatures.addData(timeOfUpdate,
currentMarketFeatures)
featureConfigs = self.tsParams.getMarketFeatureConfigs()
featureConfigKeys = [
featureConfig.getFeatureKey() for featureConfig in featureConfigs
]
for featureConfig in self.__compulsoryFeatureConfigs: # TODO: Find a better way to handle compulsory dicts
featureKey = featureConfig.getFeatureKey()
if featureKey in featureConfigKeys:
i = featureConfigKeys.index(featureKey)
customFeatureConfig = featureConfigs.pop(i)
featureConfigKeys.pop(i)
featureConfigs.append(customFeatureConfig)
else:
featureConfigs.append(featureConfig)
for featureConfig in featureConfigs:
start = time.time()
featureId = featureConfig.getFeatureId()
featureKey = featureConfig.getFeatureKey()
featureParams = featureConfig.getFeatureParams()
featureCls = FeatureConfig.getClassForFeatureId(featureId)
featureVal = featureCls.computeForMarket(
updateNum=self.__totalIter,
time=timeOfUpdate,
featureParams=featureParams,
featureKey=featureKey,
currentMarketFeatures=currentMarketFeatures,
instrumentManager=self)
currentMarketFeatures[featureKey] = featureVal
self.__lookbackMarketFeatures.addFeatureVal(
timeOfUpdate, featureKey, featureVal)
end = time.time()
diffms = (end - start) * 1000
self.__marketPerfDict[
featureKey] = self.__marketPerfDict[featureKey] + diffms
logPerf('Avg time for feature: %s : %.2f' %
(featureKey,
self.__marketPerfDict[featureKey] / self.__totalIter))
logInfo('Market Features: %s' % str(currentMarketFeatures))
def getCompulsoryInstrumentFeatureConfigs(tsParams, instrumentType):
positionConfigDict = {'featureKey': 'position',
'featureId': 'position',
'params': {}}
feesConfigDict = {INSTRUMENT_TYPE_STOCK: {'featureKey': 'fees',
'featureId': 'fees',
'params': {'price': tsParams.getPriceFeatureKey(),
'feesDict': {1: 0.05, -1: 0.05, 0: 0}}},
INSTRUMENT_TYPE_FUTURE: {'featureKey': 'fees',
'featureId': 'fees',
'params': {'price': tsParams.getPriceFeatureKey(),
'feesDict': {1: 0.00002392, -1: 0.00012392, 0: 0}}},
INSTRUMENT_TYPE_OPTION: {'featureKey': 'fees',
'featureId': 'fees',
'params': {'price': tsParams.getPriceFeatureKey(),
'feesDict': {1: 0.0005915, -1: 0.0010915, 0: 0}}}}
profitlossConfigDict = {'featureKey': 'pnl',
'featureId': 'pnl',
'params': {'price': tsParams.getPriceFeatureKey(),
'fees': 'fees'}}
capitalConfigDict = {'featureKey': 'capital',
'featureId': 'capital',
'params': {'price': tsParams.getPriceFeatureKey(), 'fees': 'fees'}}
totalProfitConfigDict = {'featureKey': 'total_profit',
'featureId': 'total_profit',
'params': {'pnlKey': 'pnl'}}
totalLossConfigDict = {'featureKey': 'total_loss',
'featureId': 'total_loss',
'params': {'pnlKey': 'pnl'}}
countProfitConfigDict = {'featureKey': 'count_profit',
'featureId': 'count_profit',
'params': {'pnlKey': 'pnl'}}
countLossConfigDict = {'featureKey': 'count_loss',
'featureId': 'count_loss',
'params': {'pnlKey': 'pnl'}}
compulsoryConfigDicts = [positionConfigDict, feesConfigDict[instrumentType], profitlossConfigDict, capitalConfigDict,
totalProfitConfigDict, totalLossConfigDict, countProfitConfigDict, countLossConfigDict]
compulsoryInstrumentFeatureConfigs = list(map(lambda x: FeatureConfig(x), compulsoryConfigDicts))
return compulsoryInstrumentFeatureConfigs
def getCompulsoryMarketFeatureConfigs(tsParams):
profitlossConfigDict = {'featureKey': 'pnl',
'featureId': 'pnl',
'params': {'instrument_pnl_feature': 'pnl'}}
capitalConfigDict = {'featureKey': 'capital',
'featureId': 'capital',
'params': {'initial_capital': tsParams.getStartingCapital(),
'pnl': 'pnl'}}
portfoliovalueConfigDict = {'featureKey': 'portfolio_value',
'featureId': 'portfolio_value',
'params': {'initial_capital': tsParams.getStartingCapital(),
'pnl': 'pnl'}}
varianceConfigDict = {'featureKey': 'variance',
'featureId': 'variance',
'params': {'pnlKey': 'pnl'}}
maxCapitalUsageConfigDict = {'featureKey': 'capitalUsage',
'featureId': 'maxCapitalUsage',
'params': {'initial_capital': tsParams.getStartingCapital(),
'capitalKey': 'capital'}}
maxDrawdownConfigDict = {'featureKey': 'maxDrawdown',
'featureId': 'maxDrawdown',
'params': {'portfolioValueKey': 'portfolio_value'}}
totalProfitConfigDict = {'featureKey': 'total_profit',
'featureId': 'total_profit',
'params': {'pnlKey': 'pnl'}}
totalLossConfigDict = {'featureKey': 'total_loss',
'featureId': 'total_loss',
'params': {'pnlKey': 'pnl'}}
countProfitConfigDict = {'featureKey': 'count_profit',
'featureId': 'count_profit',
'params': {'pnlKey': 'pnl'}}
countLossConfigDict = {'featureKey': 'count_loss',
'featureId': 'count_loss',
'params': {'pnlKey': 'pnl'}}
compulsoryConfigDicts = [profitlossConfigDict, capitalConfigDict, portfoliovalueConfigDict,
varianceConfigDict, maxCapitalUsageConfigDict, maxDrawdownConfigDict,
totalProfitConfigDict, totalLossConfigDict, countProfitConfigDict, countLossConfigDict]
compulsoryMarketFeatureConfigs = list(map(lambda x: FeatureConfig(x), compulsoryConfigDicts))
return compulsoryMarketFeatureConfigs
def computeInstrumentFeatures(self, writeFeatures=True):
instrumentBookData = self.__dataParser.emitAllInstrumentUpdates()
instrumentIds = self.__dataParser.getInstrumentIds()
for bookDataFeature in self.__bookDataFeatures:
featureDf = pd.concat([instrumentBookData[instrumentId].getBookDataByFeature(bookDataFeature) for instrumentId in instrumentIds], axis=1)
featureDf.columns = instrumentIds
self.__instrumentDataManger.addFeatureValueForAllInstruments(bookDataFeature, featureDf)
# NOTE: copy in pd.concat is set to True. Check what happens when it is False
featureConfigs = self.systemParams.getFeatureConfigsForInstrumentType(INSTRUMENT_TYPE_STOCK)
featureGenerator = self.__instrumentDataManger.getSimulator(self.__chunkSize)
for chunkNumber, timeUpdates in featureGenerator:
self.__totalIter = self.__totalIter + 1
for bookDataFeature in self.__bookDataFeatures:
self.__instrumentDataManger.updateInstrumentDataChunk(bookDataFeature)
for featureConfig in featureConfigs:
featureKey = featureConfig.getFeatureKey()
featureParams = featureConfig.getFeatureParams()
featureId = featureConfig.getFeatureId()
featureCls = FeatureConfig.getClassForFeatureId(featureId)
featureDf = featureCls.computeForInstrumentData(updateNum=chunkNumber,
featureParams=featureParams,
featureKey=featureKey,
featureManager=self)
self.__instrumentDataManger.addFeatureValueChunkForAllInstruments(featureKey, featureDf)
self.__instrumentDataManger.transformInstrumentData()
if writeFeatures:
self.__instrumentDataManger.writeInstrumentData()
self.__instrumentDataManger.dumpInstrumentDataChunk()
if not self.__instrumentDataManger.checkDataIntegrity(chunkNumber):
logWarn("Some data is missing! Check logs")
if self.__chunkSize is None:
self.__instrumentDataManger.cleanup()
else:
self.__instrumentDataManger.cleanup(delInstrumentData=True)
