def __init__(self, symbols, targetVariable, models=None): self.instrumentIds = symbols self.trainingDataSource = None self.validationDataSource = None self.testDataSource = None self.validationSplit = None self.targetVariable = None self.validationSplit = None self.targetVariable = None FeatureConfig.setupCustomFeatures(self.getCustomFeatures()) self.__instrumentFeatureConfigs = {} instrumentFeatureConfigDicts = self.getInstrumentFeatureConfigDicts() for instrumentType in instrumentFeatureConfigDicts: if type(instrumentFeatureConfigDicts[instrumentType]) is list: self.__instrumentFeatureConfigs[instrumentType] = list( map(lambda x: FeatureConfig(x), instrumentFeatureConfigDicts[instrumentType])) elif type(instrumentFeatureConfigDicts[instrumentType]) is dict: self.__instrumentFeatureConfigs[instrumentType] = { k: FeatureConfig(v) for k, v in instrumentFeatureConfigDicts[instrumentType] } else: logError("Unknown type of instruments' feature config dicts")
def __init__(self): FeatureConfig.setupCustomFeatures(self.getCustomFeatures()) self.__instrumentFeatureConfigs = {} instrumentFeatureConfigDicts = self.getInstrumentFeatureConfigDicts() for instrumentType in instrumentFeatureConfigDicts: self.__instrumentFeatureConfigs[instrumentType] = list( map(lambda x: FeatureConfig(x), instrumentFeatureConfigDicts[instrumentType])) self.__marketFeatureConfigs = list( map(lambda x: FeatureConfig(x), self.getMarketFeatureConfigDicts()))
def updateInstrumentFeatures(self, timeOfUpdate): self.__totalIter = self.__totalIter + 1 # populate current book data currentBookDataByFeature = {} for featureKey in self.__bookDataFeatures: currentBookDataByFeature[featureKey] = {} for instrument in self.__instrumentsDict.values(): instrumentId = instrument.getInstrumentId() currentInstrumentBookData = instrument.getCurrentBookData() for featureKey in self.__bookDataFeatures: currentBookDataByFeature[featureKey][instrumentId] = currentInstrumentBookData[featureKey] for featureKey in self.__bookDataFeatures: start = time.time() self.__lookbackInstrumentFeatures.addFeatureValueForAllInstruments(timeOfUpdate, featureKey, currentBookDataByFeature[featureKey]) end = time.time() diffms = (end - start) * 1000 self.__perfDict[featureKey] = self.__perfDict[featureKey] + diffms logPerf('Avg time for feature: %s : %.2f' % (featureKey, self.__perfDict[featureKey] / self.__totalIter)) featureConfigs = self.tsParams.getFeatureConfigsForInstrumentType(INSTRUMENT_TYPE_STOCK) # TODO: featureConfigKeys = [featureConfig.getFeatureKey() for featureConfig in featureConfigs] for featureConfig in self.__compulsoryInstrumentFeatureConfigs: # TODO: Find a better way to handle compulsory dicts featureKey = featureConfig.getFeatureKey() if featureKey in featureConfigKeys: i = featureConfigKeys.index(featureKey) customFeatureConfig = featureConfigs.pop(i) featureConfigKeys.pop(i) featureConfigs.append(customFeatureConfig) else: featureConfigs.append(featureConfig) featureConfigKeys = [featureConfig.getFeatureKey() for featureConfig in featureConfigs] for featureConfig in featureConfigs: start = time.time() featureKey = featureConfig.getFeatureKey() featureId = featureConfig.getFeatureId() featureKey = featureConfig.getFeatureKey() featureParams = featureConfig.getFeatureParams() featureCls = FeatureConfig.getClassForFeatureId(featureId) featureVal = featureCls.computeForInstrument(updateNum=self.__totalIter, time=timeOfUpdate, featureParams=featureParams, featureKey=featureKey, instrumentManager=self) self.__lookbackInstrumentFeatures.addFeatureValueForAllInstruments(timeOfUpdate, featureKey, featureVal) end = time.time() diffms = (end - start) * 1000 self.__perfDict[featureKey] = self.__perfDict[featureKey] + diffms logPerf('Avg time for feature: %s : %.2f' % (featureKey, self.__perfDict[featureKey] / self.__totalIter))
def updateFeatures(self, timeOfUpdate): self.updateInstrumentFeatures(timeOfUpdate) currentMarketFeatures = {} self.__lookbackMarketFeatures.addData(timeOfUpdate, currentMarketFeatures) featureConfigs = self.tsParams.getMarketFeatureConfigs() featureConfigKeys = [ featureConfig.getFeatureKey() for featureConfig in featureConfigs ] for featureConfig in self.__compulsoryFeatureConfigs: # TODO: Find a better way to handle compulsory dicts featureKey = featureConfig.getFeatureKey() if featureKey in featureConfigKeys: i = featureConfigKeys.index(featureKey) customFeatureConfig = featureConfigs.pop(i) featureConfigKeys.pop(i) featureConfigs.append(customFeatureConfig) else: featureConfigs.append(featureConfig) for featureConfig in featureConfigs: start = time.time() featureId = featureConfig.getFeatureId() featureKey = featureConfig.getFeatureKey() featureParams = featureConfig.getFeatureParams() featureCls = FeatureConfig.getClassForFeatureId(featureId) featureVal = featureCls.computeForMarket( updateNum=self.__totalIter, time=timeOfUpdate, featureParams=featureParams, featureKey=featureKey, currentMarketFeatures=currentMarketFeatures, instrumentManager=self) currentMarketFeatures[featureKey] = featureVal self.__lookbackMarketFeatures.addFeatureVal( timeOfUpdate, featureKey, featureVal) end = time.time() diffms = (end - start) * 1000 self.__marketPerfDict[ featureKey] = self.__marketPerfDict[featureKey] + diffms logPerf('Avg time for feature: %s : %.2f' % (featureKey, self.__marketPerfDict[featureKey] / self.__totalIter)) logInfo('Market Features: %s' % str(currentMarketFeatures))
def getCompulsoryInstrumentFeatureConfigs(tsParams, instrumentType): positionConfigDict = {'featureKey': 'position', 'featureId': 'position', 'params': {}} feesConfigDict = {INSTRUMENT_TYPE_STOCK: {'featureKey': 'fees', 'featureId': 'fees', 'params': {'price': tsParams.getPriceFeatureKey(), 'feesDict': {1: 0.05, -1: 0.05, 0: 0}}}, INSTRUMENT_TYPE_FUTURE: {'featureKey': 'fees', 'featureId': 'fees', 'params': {'price': tsParams.getPriceFeatureKey(), 'feesDict': {1: 0.00002392, -1: 0.00012392, 0: 0}}}, INSTRUMENT_TYPE_OPTION: {'featureKey': 'fees', 'featureId': 'fees', 'params': {'price': tsParams.getPriceFeatureKey(), 'feesDict': {1: 0.0005915, -1: 0.0010915, 0: 0}}}} profitlossConfigDict = {'featureKey': 'pnl', 'featureId': 'pnl', 'params': {'price': tsParams.getPriceFeatureKey(), 'fees': 'fees'}} capitalConfigDict = {'featureKey': 'capital', 'featureId': 'capital', 'params': {'price': tsParams.getPriceFeatureKey(), 'fees': 'fees'}} totalProfitConfigDict = {'featureKey': 'total_profit', 'featureId': 'total_profit', 'params': {'pnlKey': 'pnl'}} totalLossConfigDict = {'featureKey': 'total_loss', 'featureId': 'total_loss', 'params': {'pnlKey': 'pnl'}} countProfitConfigDict = {'featureKey': 'count_profit', 'featureId': 'count_profit', 'params': {'pnlKey': 'pnl'}} countLossConfigDict = {'featureKey': 'count_loss', 'featureId': 'count_loss', 'params': {'pnlKey': 'pnl'}} compulsoryConfigDicts = [positionConfigDict, feesConfigDict[instrumentType], profitlossConfigDict, capitalConfigDict, totalProfitConfigDict, totalLossConfigDict, countProfitConfigDict, countLossConfigDict] compulsoryInstrumentFeatureConfigs = list(map(lambda x: FeatureConfig(x), compulsoryConfigDicts)) return compulsoryInstrumentFeatureConfigs
def getCompulsoryMarketFeatureConfigs(tsParams): profitlossConfigDict = {'featureKey': 'pnl', 'featureId': 'pnl', 'params': {'instrument_pnl_feature': 'pnl'}} capitalConfigDict = {'featureKey': 'capital', 'featureId': 'capital', 'params': {'initial_capital': tsParams.getStartingCapital(), 'pnl': 'pnl'}} portfoliovalueConfigDict = {'featureKey': 'portfolio_value', 'featureId': 'portfolio_value', 'params': {'initial_capital': tsParams.getStartingCapital(), 'pnl': 'pnl'}} varianceConfigDict = {'featureKey': 'variance', 'featureId': 'variance', 'params': {'pnlKey': 'pnl'}} maxCapitalUsageConfigDict = {'featureKey': 'capitalUsage', 'featureId': 'maxCapitalUsage', 'params': {'initial_capital': tsParams.getStartingCapital(), 'capitalKey': 'capital'}} maxDrawdownConfigDict = {'featureKey': 'maxDrawdown', 'featureId': 'maxDrawdown', 'params': {'portfolioValueKey': 'portfolio_value'}} totalProfitConfigDict = {'featureKey': 'total_profit', 'featureId': 'total_profit', 'params': {'pnlKey': 'pnl'}} totalLossConfigDict = {'featureKey': 'total_loss', 'featureId': 'total_loss', 'params': {'pnlKey': 'pnl'}} countProfitConfigDict = {'featureKey': 'count_profit', 'featureId': 'count_profit', 'params': {'pnlKey': 'pnl'}} countLossConfigDict = {'featureKey': 'count_loss', 'featureId': 'count_loss', 'params': {'pnlKey': 'pnl'}} compulsoryConfigDicts = [profitlossConfigDict, capitalConfigDict, portfoliovalueConfigDict, varianceConfigDict, maxCapitalUsageConfigDict, maxDrawdownConfigDict, totalProfitConfigDict, totalLossConfigDict, countProfitConfigDict, countLossConfigDict] compulsoryMarketFeatureConfigs = list(map(lambda x: FeatureConfig(x), compulsoryConfigDicts)) return compulsoryMarketFeatureConfigs
def computeInstrumentFeatures(self, writeFeatures=True): instrumentBookData = self.__dataParser.emitAllInstrumentUpdates() instrumentIds = self.__dataParser.getInstrumentIds() for bookDataFeature in self.__bookDataFeatures: featureDf = pd.concat([instrumentBookData[instrumentId].getBookDataByFeature(bookDataFeature) for instrumentId in instrumentIds], axis=1) featureDf.columns = instrumentIds self.__instrumentDataManger.addFeatureValueForAllInstruments(bookDataFeature, featureDf) # NOTE: copy in pd.concat is set to True. Check what happens when it is False featureConfigs = self.systemParams.getFeatureConfigsForInstrumentType(INSTRUMENT_TYPE_STOCK) featureGenerator = self.__instrumentDataManger.getSimulator(self.__chunkSize) for chunkNumber, timeUpdates in featureGenerator: self.__totalIter = self.__totalIter + 1 for bookDataFeature in self.__bookDataFeatures: self.__instrumentDataManger.updateInstrumentDataChunk(bookDataFeature) for featureConfig in featureConfigs: featureKey = featureConfig.getFeatureKey() featureParams = featureConfig.getFeatureParams() featureId = featureConfig.getFeatureId() featureCls = FeatureConfig.getClassForFeatureId(featureId) featureDf = featureCls.computeForInstrumentData(updateNum=chunkNumber, featureParams=featureParams, featureKey=featureKey, featureManager=self) self.__instrumentDataManger.addFeatureValueChunkForAllInstruments(featureKey, featureDf) self.__instrumentDataManger.transformInstrumentData() if writeFeatures: self.__instrumentDataManger.writeInstrumentData() self.__instrumentDataManger.dumpInstrumentDataChunk() if not self.__instrumentDataManger.checkDataIntegrity(chunkNumber): logWarn("Some data is missing! Check logs") if self.__chunkSize is None: self.__instrumentDataManger.cleanup() else: self.__instrumentDataManger.cleanup(delInstrumentData=True)