def record(target_dir):
target_dir_len = len(target_dir)
dic = {}
save_dir = os.path.join(target_dir, "timestamp.txt")
for root, dirs, files in os.walk(target_dir):
for file in files:
file_path = os.path.join(root, file)
mtime_ns = TSHelper.get(file_path)
dic[file_path[target_dir_len:]] = mtime_ns
Data.save(dic, save_dir)
def segment(self):
self.logger.info('Segmenting...')
data = Data();
totalpct = float(self.configuration['totalpct'])
testpct = float(self.configuration['testpct'])
randomstate = int(self.configuration['randomstate'])
X_train, X_test, y_train, y_test = data.segment(self.features, self.target, totalpct, testpct, randomstate)
self.X_train = X_train
self.X_test = X_test
self.y_train = y_train
self.y_test = y_test
return
def Path_TDDict(self, user):
path = Data.solve_path(
config.json['Path_TeamDriveDict'],
{
'appDataDir': config.data_dir,
'user': user
}
)
return path
from service.ControlService import CTPService, ModelService
# s = ModelService('src/py/online.py demo aa', [], 'v0001')
# print s.cmd
# s.start()
# print s.status()
# s.stop()
# s = CTPService('bin/marketService', [], 'SRV_MARKET')
# print s.cmd
# s.start()
# print s.status()
# s.stop()
from common.Data import Data
d = Data()
cnt, data = d.getTickHistory('hc', ['hc1805'], '20180312 21:00:00',
'20180312 21:01:00')
print cnt
print data[0]['price']
def __init__(self,
strat,
startRunningTime,
orderWaitingTime,
dataLogFixedTimeWindow,
dailyExitTime=None):
self.strat = strat
self.user_data = UserData()
self.strat.initialize(self)
self.startRunningTime = startRunningTime
self.timeInterval = history.frequency_to_seconds[
self.frequency] # 每次循环结束之后睡眠的时间, 单位：秒
self.orderWaitingTime = orderWaitingTime # 每次等待订单执行的最长时间
self.dataLogFixedTimeWindow = dataLogFixedTimeWindow # 每隔固定的时间打印账单信息，单位：秒
self.coinMarketType = helper.getCoinMarketTypeFromSecurity(
self.security)
self.dailyExitTime = dailyExitTime # 如果设置，则为每天程序的退出时间
self.TimeFormatForFileName = "%Y%m%d%H%M%S%f"
self.TimeFormatForLog = "%Y-%m-%d %H:%M:%S.%f"
self.huobiService = huobiService
coin_type = helper.coinTypeStructure[
self.coinMarketType]["huobi"]["coin_type"]
self.huobi_min_quantity = self.huobiService.getMinimumOrderQty(
coin_type)
self.huobi_min_cash_amount = self.huobiService.getMinimumOrderCashAmount(
)
self.last_data_log_time = None
# order query retry maximum times
self.huobi_order_query_retry_maximum_times = 5
# setup timeLogger
self.timeLogger = logging.getLogger('timeLog')
self.timeLogger.setLevel(logging.DEBUG)
timeLogHandler = logging.FileHandler(self.getTimeLogFileName())
timeLogHandler.setLevel(logging.DEBUG)
consoleLogHandler = logging.StreamHandler()
consoleLogHandler.setLevel(logging.DEBUG)
# 定义handler的输出格式
formatter = logging.Formatter(
'%(asctime)s - %(levelname)s - %(message)s')
timeLogHandler.setFormatter(formatter)
consoleLogHandler.setFormatter(formatter)
# 给timeLogger添加handler
self.timeLogger.addHandler(timeLogHandler)
self.timeLogger.addHandler(consoleLogHandler)
# setup dataLogger
self.dataLogger = logging.getLogger('dataLog')
self.dataLogger.setLevel(logging.DEBUG)
dataLogHandler = logging.FileHandler(self.getDataLogFileName())
dataLogHandler.setLevel(logging.DEBUG)
self.dataLogger.addHandler(dataLogHandler)
# setup context.log
self.log = WQLogger(self.timeLogger)
# setup context.data
data = Data()
data.get_price = history.get_price
data.get_current_price = history.get_current_price
data.get_all_securities = history.get_all_securities
self.data = data
# setup context.order
order = Order()
order.buy = self.buy
order.buy_limit = self.buy_limit
order.sell = self.sell
order.sell_limit = self.sell_limit
self.order = order
# setup context.account and context.account_initial
self.account = Account()
self.account_initial = Account()
self.updateAccountInfo(initial_setup=True)
# setup context.time
self.time = Time(self.startRunningTime)
# setup handle_data
self.handle_data = self.strat.handle_data
def Save_TDDict(self, user):
Data.save(self.TeamDriveDict(user), self.Path_TDDict(user))
def TeamDriveDict(self, user)->dict:
if self._TeamDriveDicts.get(user) is None:
TDDict = Data.load(self.Path_TDDict(user))
self._TeamDriveDicts[user] = TDDict if TDDict else {}
return self._TeamDriveDicts[user]
def save(self):
Data.save(self._json, self._path_json)
def json(self) -> dict:
if self._json is None:
self._json = Data.load(self._path_json)
return self._json
def load(self):
self.logger.info('Loading data')
data = Data()
trainCSV = self.configuration['trainCSV']
testCSV = self.configuration['testCSV']
featuresPKL = self.configuration['featuresPKL']
targetPKL = self.configuration['targetPKL']
testPKL = self.configuration['testPKL']
expectedCSV = None
expectedPKL = None
try:
expectedCSV = self.configuration['expectedCSV']
expectedPKL = self.configuration['expectedPKL']
except:
pass
# If the dataframe (pickled) file exists, then load it
# Otherwise, load the CSV, preprocess it, and then save it as a
# PKL file which will reduce load times
tmpFeatures = None
tmpTarget = None
tmptest = None
if( os.path.exists(featuresPKL) ):
self.logger.info('Loading train PKL: {0}'.format(featuresPKL))
tmpFeatures = data.loadDataFrame(featuresPKL)
self.logger.info('Loading target PKL: {0}'.format(targetPKL))
tmpTarget = data.loadDataFrame(targetPKL)
self.logger.info('Loading test PKL: {0}'.format(testPKL))
tmpTest = data.loadDataFrame(testPKL)
else:
self.logger.info('Loading train CSV: {0}'.format(trainCSV))
rawtrain = data.load(trainCSV)
self.logger.info('Loading test CSV: {0}'.format(testCSV))
rawtest = data.load(testCSV)
# Preprocess the data
tmpFeatures, tmpTarget, tmpTest = self.preprocessor.execute(rawtrain, rawtest)
# Save the dataframe (lower load times)
self.logger.info('Saving features PKL: {0}'.format(featuresPKL))
data.saveDataFrame(tmpFeatures, featuresPKL)
self.logger.info('Saving target PKL: {0}'.format(targetPKL))
data.saveDataFrame(tmpTarget, targetPKL)
self.logger.info('Saving test PKL: {0}'.format(testPKL))
data.saveDataFrame(tmpTest, testPKL)
if( expectedPKL and os.path.exists(expectedPKL) ):
self.logger.info('loading expected PKL: {0}'.format(expectedPKL))
tmpExpected = data.loadDataFrame(expectedPKL)
self.hasExpected = True
elif( expectedCSV and os.path.exists(expectedCSV) ):
self.logger.info('Loading expected CSV: {0}'.format(expectedCSV))
tmpExpected = data.load(expectedCSV)
self.logger.info('Saving expected PKL: {0}'.format(expectedPKL))
data.saveDataFrame(tmpExpected, expectedPKL)
self.hasExpected = True
self.features = tmpFeatures
self.target = tmpTarget
self.test = tmpTest
if self.hasExpected:
self.expected = tmpExpected
return
def __init__(self, strat, startRunningTime, orderWaitingTime, dataLogFixedTimeWindow, dailyExitTime=None):
self.strat = strat
self.user_data = UserData()
self.strat.initialize(self)
self.startRunningTime = startRunningTime
self.timeInterval = history.frequency_to_seconds[self.frequency] # 每次循环结束之后睡眠的时间, 单位：秒
self.orderWaitingTime = orderWaitingTime # 每次等待订单执行的最长时间
self.dataLogFixedTimeWindow = dataLogFixedTimeWindow # 每隔固定的时间打印账单信息，单位：秒
self.coinMarketType = helper.getCoinMarketTypeFromSecurity(self.security)
self.dailyExitTime = dailyExitTime # 如果设置，则为每天程序的退出时间
self.TimeFormatForFileName = "%Y%m%d%H%M%S%f"
self.TimeFormatForLog = "%Y-%m-%d %H:%M:%S.%f"
self.huobiService = huobiService
coin_type = helper.coinTypeStructure[self.coinMarketType]["huobi"]["coin_type"]
self.huobi_min_quantity = self.huobiService.getMinimumOrderQty(coin_type)
self.huobi_min_cash_amount = self.huobiService.getMinimumOrderCashAmount()
self.last_data_log_time = None
# order query retry maximum times
self.huobi_order_query_retry_maximum_times = 5
# setup timeLogger
self.timeLogger = logging.getLogger('timeLog')
self.timeLogger.setLevel(logging.DEBUG)
timeLogHandler = logging.FileHandler(self.getTimeLogFileName())
timeLogHandler.setLevel(logging.DEBUG)
consoleLogHandler = logging.StreamHandler()
consoleLogHandler.setLevel(logging.DEBUG)
# 定义handler的输出格式
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
timeLogHandler.setFormatter(formatter)
consoleLogHandler.setFormatter(formatter)
# 给timeLogger添加handler
self.timeLogger.addHandler(timeLogHandler)
self.timeLogger.addHandler(consoleLogHandler)
# setup dataLogger
self.dataLogger = logging.getLogger('dataLog')
self.dataLogger.setLevel(logging.DEBUG)
dataLogHandler = logging.FileHandler(self.getDataLogFileName())
dataLogHandler.setLevel(logging.DEBUG)
self.dataLogger.addHandler(dataLogHandler)
# setup context.log
self.log = WQLogger(self.timeLogger)
# setup context.data
data = Data()
data.get_price = history.get_price
data.get_current_price = history.get_current_price
data.get_all_securities = history.get_all_securities
self.data = data
# setup context.order
order = Order()
order.buy = self.buy
order.buy_limit = self.buy_limit
order.sell = self.sell
order.sell_limit = self.sell_limit
self.order = order
# setup context.account and context.account_initial
self.account = Account()
self.account_initial = Account()
self.updateAccountInfo(initial_setup=True)
# setup context.time
self.time = Time(self.startRunningTime)
# setup handle_data
self.handle_data = self.strat.handle_data
