def record(target_dir): target_dir_len = len(target_dir) dic = {} save_dir = os.path.join(target_dir, "timestamp.txt") for root, dirs, files in os.walk(target_dir): for file in files: file_path = os.path.join(root, file) mtime_ns = TSHelper.get(file_path) dic[file_path[target_dir_len:]] = mtime_ns Data.save(dic, save_dir)
def segment(self): self.logger.info('Segmenting...') data = Data(); totalpct = float(self.configuration['totalpct']) testpct = float(self.configuration['testpct']) randomstate = int(self.configuration['randomstate']) X_train, X_test, y_train, y_test = data.segment(self.features, self.target, totalpct, testpct, randomstate) self.X_train = X_train self.X_test = X_test self.y_train = y_train self.y_test = y_test return
def Path_TDDict(self, user): path = Data.solve_path( config.json['Path_TeamDriveDict'], { 'appDataDir': config.data_dir, 'user': user } ) return path
from service.ControlService import CTPService, ModelService # s = ModelService('src/py/online.py demo aa', [], 'v0001') # print s.cmd # s.start() # print s.status() # s.stop() # s = CTPService('bin/marketService', [], 'SRV_MARKET') # print s.cmd # s.start() # print s.status() # s.stop() from common.Data import Data d = Data() cnt, data = d.getTickHistory('hc', ['hc1805'], '20180312 21:00:00', '20180312 21:01:00') print cnt print data[0]['price']
def __init__(self, strat, startRunningTime, orderWaitingTime, dataLogFixedTimeWindow, dailyExitTime=None): self.strat = strat self.user_data = UserData() self.strat.initialize(self) self.startRunningTime = startRunningTime self.timeInterval = history.frequency_to_seconds[ self.frequency] # 每次循环结束之后睡眠的时间, 单位:秒 self.orderWaitingTime = orderWaitingTime # 每次等待订单执行的最长时间 self.dataLogFixedTimeWindow = dataLogFixedTimeWindow # 每隔固定的时间打印账单信息,单位:秒 self.coinMarketType = helper.getCoinMarketTypeFromSecurity( self.security) self.dailyExitTime = dailyExitTime # 如果设置,则为每天程序的退出时间 self.TimeFormatForFileName = "%Y%m%d%H%M%S%f" self.TimeFormatForLog = "%Y-%m-%d %H:%M:%S.%f" self.huobiService = huobiService coin_type = helper.coinTypeStructure[ self.coinMarketType]["huobi"]["coin_type"] self.huobi_min_quantity = self.huobiService.getMinimumOrderQty( coin_type) self.huobi_min_cash_amount = self.huobiService.getMinimumOrderCashAmount( ) self.last_data_log_time = None # order query retry maximum times self.huobi_order_query_retry_maximum_times = 5 # setup timeLogger self.timeLogger = logging.getLogger('timeLog') self.timeLogger.setLevel(logging.DEBUG) timeLogHandler = logging.FileHandler(self.getTimeLogFileName()) timeLogHandler.setLevel(logging.DEBUG) consoleLogHandler = logging.StreamHandler() consoleLogHandler.setLevel(logging.DEBUG) # 定义handler的输出格式 formatter = logging.Formatter( '%(asctime)s - %(levelname)s - %(message)s') timeLogHandler.setFormatter(formatter) consoleLogHandler.setFormatter(formatter) # 给timeLogger添加handler self.timeLogger.addHandler(timeLogHandler) self.timeLogger.addHandler(consoleLogHandler) # setup dataLogger self.dataLogger = logging.getLogger('dataLog') self.dataLogger.setLevel(logging.DEBUG) dataLogHandler = logging.FileHandler(self.getDataLogFileName()) dataLogHandler.setLevel(logging.DEBUG) self.dataLogger.addHandler(dataLogHandler) # setup context.log self.log = WQLogger(self.timeLogger) # setup context.data data = Data() data.get_price = history.get_price data.get_current_price = history.get_current_price data.get_all_securities = history.get_all_securities self.data = data # setup context.order order = Order() order.buy = self.buy order.buy_limit = self.buy_limit order.sell = self.sell order.sell_limit = self.sell_limit self.order = order # setup context.account and context.account_initial self.account = Account() self.account_initial = Account() self.updateAccountInfo(initial_setup=True) # setup context.time self.time = Time(self.startRunningTime) # setup handle_data self.handle_data = self.strat.handle_data
def Save_TDDict(self, user): Data.save(self.TeamDriveDict(user), self.Path_TDDict(user))
def TeamDriveDict(self, user)->dict: if self._TeamDriveDicts.get(user) is None: TDDict = Data.load(self.Path_TDDict(user)) self._TeamDriveDicts[user] = TDDict if TDDict else {} return self._TeamDriveDicts[user]
def save(self): Data.save(self._json, self._path_json)
def json(self) -> dict: if self._json is None: self._json = Data.load(self._path_json) return self._json
def load(self): self.logger.info('Loading data') data = Data() trainCSV = self.configuration['trainCSV'] testCSV = self.configuration['testCSV'] featuresPKL = self.configuration['featuresPKL'] targetPKL = self.configuration['targetPKL'] testPKL = self.configuration['testPKL'] expectedCSV = None expectedPKL = None try: expectedCSV = self.configuration['expectedCSV'] expectedPKL = self.configuration['expectedPKL'] except: pass # If the dataframe (pickled) file exists, then load it # Otherwise, load the CSV, preprocess it, and then save it as a # PKL file which will reduce load times tmpFeatures = None tmpTarget = None tmptest = None if( os.path.exists(featuresPKL) ): self.logger.info('Loading train PKL: {0}'.format(featuresPKL)) tmpFeatures = data.loadDataFrame(featuresPKL) self.logger.info('Loading target PKL: {0}'.format(targetPKL)) tmpTarget = data.loadDataFrame(targetPKL) self.logger.info('Loading test PKL: {0}'.format(testPKL)) tmpTest = data.loadDataFrame(testPKL) else: self.logger.info('Loading train CSV: {0}'.format(trainCSV)) rawtrain = data.load(trainCSV) self.logger.info('Loading test CSV: {0}'.format(testCSV)) rawtest = data.load(testCSV) # Preprocess the data tmpFeatures, tmpTarget, tmpTest = self.preprocessor.execute(rawtrain, rawtest) # Save the dataframe (lower load times) self.logger.info('Saving features PKL: {0}'.format(featuresPKL)) data.saveDataFrame(tmpFeatures, featuresPKL) self.logger.info('Saving target PKL: {0}'.format(targetPKL)) data.saveDataFrame(tmpTarget, targetPKL) self.logger.info('Saving test PKL: {0}'.format(testPKL)) data.saveDataFrame(tmpTest, testPKL) if( expectedPKL and os.path.exists(expectedPKL) ): self.logger.info('loading expected PKL: {0}'.format(expectedPKL)) tmpExpected = data.loadDataFrame(expectedPKL) self.hasExpected = True elif( expectedCSV and os.path.exists(expectedCSV) ): self.logger.info('Loading expected CSV: {0}'.format(expectedCSV)) tmpExpected = data.load(expectedCSV) self.logger.info('Saving expected PKL: {0}'.format(expectedPKL)) data.saveDataFrame(tmpExpected, expectedPKL) self.hasExpected = True self.features = tmpFeatures self.target = tmpTarget self.test = tmpTest if self.hasExpected: self.expected = tmpExpected return
def __init__(self, strat, startRunningTime, orderWaitingTime, dataLogFixedTimeWindow, dailyExitTime=None): self.strat = strat self.user_data = UserData() self.strat.initialize(self) self.startRunningTime = startRunningTime self.timeInterval = history.frequency_to_seconds[self.frequency] # 每次循环结束之后睡眠的时间, 单位:秒 self.orderWaitingTime = orderWaitingTime # 每次等待订单执行的最长时间 self.dataLogFixedTimeWindow = dataLogFixedTimeWindow # 每隔固定的时间打印账单信息,单位:秒 self.coinMarketType = helper.getCoinMarketTypeFromSecurity(self.security) self.dailyExitTime = dailyExitTime # 如果设置,则为每天程序的退出时间 self.TimeFormatForFileName = "%Y%m%d%H%M%S%f" self.TimeFormatForLog = "%Y-%m-%d %H:%M:%S.%f" self.huobiService = huobiService coin_type = helper.coinTypeStructure[self.coinMarketType]["huobi"]["coin_type"] self.huobi_min_quantity = self.huobiService.getMinimumOrderQty(coin_type) self.huobi_min_cash_amount = self.huobiService.getMinimumOrderCashAmount() self.last_data_log_time = None # order query retry maximum times self.huobi_order_query_retry_maximum_times = 5 # setup timeLogger self.timeLogger = logging.getLogger('timeLog') self.timeLogger.setLevel(logging.DEBUG) timeLogHandler = logging.FileHandler(self.getTimeLogFileName()) timeLogHandler.setLevel(logging.DEBUG) consoleLogHandler = logging.StreamHandler() consoleLogHandler.setLevel(logging.DEBUG) # 定义handler的输出格式 formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') timeLogHandler.setFormatter(formatter) consoleLogHandler.setFormatter(formatter) # 给timeLogger添加handler self.timeLogger.addHandler(timeLogHandler) self.timeLogger.addHandler(consoleLogHandler) # setup dataLogger self.dataLogger = logging.getLogger('dataLog') self.dataLogger.setLevel(logging.DEBUG) dataLogHandler = logging.FileHandler(self.getDataLogFileName()) dataLogHandler.setLevel(logging.DEBUG) self.dataLogger.addHandler(dataLogHandler) # setup context.log self.log = WQLogger(self.timeLogger) # setup context.data data = Data() data.get_price = history.get_price data.get_current_price = history.get_current_price data.get_all_securities = history.get_all_securities self.data = data # setup context.order order = Order() order.buy = self.buy order.buy_limit = self.buy_limit order.sell = self.sell order.sell_limit = self.sell_limit self.order = order # setup context.account and context.account_initial self.account = Account() self.account_initial = Account() self.updateAccountInfo(initial_setup=True) # setup context.time self.time = Time(self.startRunningTime) # setup handle_data self.handle_data = self.strat.handle_data