def main():
''' Parses command line to Launch/relaunch the dismod cascade from the
global level.
'''
setup_logger()
log = logging.getLogger(__name__)
log.debug("main started")
import argparse
parser = argparse.ArgumentParser("Launch/relaunch the dismod cascade "
"from the global level")
parser.add_argument('mvid', type=int)
parser.add_argument('--submit_stage', type=str, default='bootstrap')
parser.add_argument('--cv_iter', type=int, default=0)
args = parser.parse_args()
submit_stage = args.submit_stage
mvid = args.mvid
cv_iter = args.cv_iter
try:
if submit_stage == 'bootstrap':
JobTreeBootstrapper(mvid)
else:
CascadeJobTree(mvid, cv_iter)
except Exception:
log.exception("error in main run_global with args {}".format(
str(args)))
upload.update_model_status(mvid, upload.FAILED)
def run_noui():
try:
log_file_name = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)),
'logs', u'noUiMain.log'))
except Exception as ex:
print( u'Use local dict:{0}'.format(os.getcwd()))
log_file_name = os.path.abspath(os.path.join(os.getcwd(), 'logs', u'noUiMain.log'))
setup_logger(filename=log_file_name, debug=False)
noUi = NoUiMain()
noUi.Start()
def __init__(self):
self.logger = setup_logger("main")
self.a1 = A1()
self.a2 = A2()
self.a3 = A3()
self.threads = list()
sys.excepthook = self.handle_exception
def main() -> None:
logger: Logger = setup_logger(__name__)
logger.info("START")
# 接頭辞を「;」に設定
bot: WerewolfBot = WerewolfBot(command_prefix=";")
bot.game = Game()
logger.debug(bot)
# 環境変数DISCORD_BOT_TOKENからBOTのトークンを取得
token = os.environ["DISCORD_BOT_TOKEN"]
# コマンド「;ping」を追加
# コマンドの内容は ping()を参照
bot.add_command(ping)
extension_list = [
"cogs.players_cog",
"cogs.game_status_cog",
]
for extension in extension_list:
bot.load_extension(name=extension)
# bot実行
bot.run(token)
def main():
'''Set commit hash, upload model, try to write effects_plots pdfs,
aggregate model version draws up location hierarchy
'''
setup_logger()
mvid = sys.argv[1]
log = logging.getLogger(__name__)
log.info("Varnish started for mvid {}".format(mvid))
try:
try:
commit_hash = sge.get_commit_hash(dir='%s/..' % drill.this_path)
except subprocess.CalledProcessError:
# in site-packages, not git repo
commit_hash = __version__
upload.set_commit_hash(mvid, commit_hash)
upload.upload_model(mvid)
outdir = "%s/%s/full" % (drill.settings['cascade_ode_out_dir'],
str(mvid))
joutdir = "%s/%s" % (drill.settings['diag_out_dir'], mvid)
fit_stats.write_fit_stats(mvid, outdir, joutdir)
upload.upload_fit_stat(mvid)
# Write effect PDFs
plotter = "{}/effect_plots.r".format(drill.this_path)
plotter = os.path.realpath(plotter)
demo = Demographics()
try:
subprocess.check_output([
"FILEPATH", plotter,
str(mvid), joutdir, drill.settings['cascade_ode_out_dir'],
str(max(demo.year_ids))
],
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError:
log.exception("Error in effect plots")
# Launch final aggregations
log.info("Starting Save Results")
aggregate_model(mvid, demo=demo)
except Exception:
log.exception("Error in varnish")
raise
from setup_logger import setup_logger
logger = setup_logger(__name__)
class Player:
"""人狼ゲーム参加者
Attributes:
id: DiscordユーザID
"""
def __init__(self, discord_id: int) -> None:
logger.debug("Playerクラス init")
self.id: int = discord_id
# encoding: UTF-8
import os
from setup_logger import setup_logger
from vnpy.trader.util_monitor import *
from vnpy.trader.vtEngine import MainEngine
from vnpy.trader.gateway import ctpGateway
from threading import Thread
from datetime import datetime
setup_logger(debug=True)
# ----------------------------------------------------------------------
"""
Linux 下：
启动：python noUiMain.py >>logs/mylog.log &
停止：ps -ef | grep python ，找到对应的进程PID， kill -9 PID
查看：tail -f logs/noUiMain_月日_小时分钟.log -f
Windows下：
启动：命令行窗口 python noUiMain.py
停止：关闭窗口
"""
class NoUiMain(object):
def __init__(self):
# gateway 是否连接
self.connected = False
# gateway 的连接名称，在vtEngine.initGateway()里面定义，对应的配置文件是 "连接名称_connect.json"，
self.gateway_name = 'CTP'
# 启动的策略实例，须在catAlgo/CtaSetting.json 里面定义 [u'S28_RB1001', u'S28_TFT', u'S28_HCRB',u'atr_rsi']
self.strategies = [u'strategyAtrRsi']
# encoding: UTF-8
import sys
import ctypes
import platform
from vtEngine import MainEngine
from uiMainWindow import *
#from simple_monitor import *
from setup_logger import setup_logger
setup_logger(filename='logs/vnpy_{0}.log'.format(datetime.now().strftime('%m%d_%H%M')), debug=False)
# ----------------------------------------------------------------------
def main():
"""主程序入口"""
# 重载sys模块，设置默认字符串编码方式为utf8
reload(sys)
sys.setdefaultencoding('utf8')
# 设置Windows底部任务栏图标
if 'Windows' in platform.uname() :
ctypes.windll.shell32.SetCurrentProcessExplicitAppUserModelID('vn.trader')
# 初始化Qt应用对象
app = QtGui.QApplication(sys.argv)
app.setWindowIcon(QtGui.QIcon('vnpy.ico'))
app.setFont(BASIC_FONT)
long_id = proc.stdout.read().strip()
assert cores > 0
print sys._getframe().f_code.co_name, 1
def test_features_crawler_crawl_outcontainer_mem():
# Start a dummy container
proc = subprocess.Popen(
"docker run -d ubuntu sleep 60",
shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
long_id = proc.stdout.read().strip()
c = DockerContainer(long_id)
crawler = FeaturesCrawler(crawl_mode='OUTCONTAINER', container=c)
output = "%s" % list(crawler.crawl_memory())
#print output
# Kill the dummy container
proc = subprocess.Popen(
"docker rm -f %s" % long_id,
shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
long_id = proc.stdout.read().strip()
assert 'memory_used' in output
print sys._getframe().f_code.co_name, 1
if __name__ == '__main__':
setup_logger("crawlutils", "tester.log")
test_features_crawler_crawl_invm_cpu()
test_features_crawler_crawl_outcontainer_cpu()
test_features_crawler_crawl_invm_mem()
test_features_crawler_crawl_outcontainer_mem()
def __init__(self):
self.logger = setup_logger("a2")
def __init__(self):
self.logger = setup_logger("a1", stream=False)
class Strategy_MACD_01(CtaTemplate):
"""螺纹钢、15分钟级别+60分钟级别，MACD策略
v1:15f上多空仓开仓
v2:60f上的仓位管理体系
本版本现存问题：
无
已解决问题：
15f上按照百分比开仓
"""
className = 'Strategy_MACD'
author = u'横纵19950206'
# 策略在外部设置的参数
# ----------------------------------------------------------------------
def __init__(self, ctaEngine, setting=None):
"""Constructor"""
super(Strategy_MACD_01, self).__init__(ctaEngine, setting)
# 增加监控参数项目
# 增加监控变量项目
self.varList.append('pos') # 仓位，这里的仓位通常是手数
self.varList.append('entrust') # 是否正在委托，0表示没有委托，1多仓，-1空仓
self.varList.append('percentLimit') # 当前账户最大仓位
self.varList.append('percent') # 当前仓位
self.curDateTime = None # 当前Tick时间
self.curTick = None # 最新的tick
self.lastOrderTime = None # 上一次委托时间
self.cancelSeconds = 60 # 撤单时间(秒)
# 定义日内的交易窗口
self.openWindow = False # 开市窗口
self.tradeWindow = False # 交易窗口
self.closeWindow = False # 收市平仓窗口
self.inited = False # 是否完成了策略初始化
self.backtesting = False # 是否回测
self.lineM15 = None # 5分钟K线
self.lineM60 = None # 60分钟k线
# 增加仓位管理模块
self.position = CtaPosition(self)
# self.position.longPos多头持仓，self.position.shorPos多头持仓、
# self.position.pos持仓状态，self.position.maxPos最大持仓
# 增加ctabacktesing中的仓位管理
if not ctaEngine:
self.engine = BacktestingEngine()
else:
self.engine = ctaEngine
# 当前资金，当前可用资金，仓位比例，仓位比例上限
self.capital, self.available, self.percent, self.percentLimit = self.engine.getAccountInfo()
if setting:
# 根据配置文件更新参数
self.setParam(setting)
# 创建的M15 K线
lineM15Setting = {}
lineM15Setting['name'] = u'M15' # k线名称
lineM15Setting['barTimeInterval'] = 60 * 15 # K线的Bar时长
lineM15Setting['inputMacdFastPeriodLen'] = 12 # DIF快线
lineM15Setting['inputMacdSlowPeriodLen'] = 26 # DEA慢线
lineM15Setting['inputMacdSignalPeriodLen'] = 9 # MACD中绿柱
lineM15Setting['shortSymbol'] = self.shortSymbol
self.lineM15 = CtaLineBar(self, self.onBarM15, lineM15Setting)
try:
mode = setting['mode']
if mode != EMPTY_STRING:
self.lineM15.setMode(setting['mode'])
except KeyError:
self.lineM15.setMode(self.lineM15.TICK_MODE)
lineM60Setting = {}
lineM60Setting['name'] = u'M60' # k线名称
lineM60Setting['barTimeInterval'] = 60 * 60 # K线的Bar时长
lineM60Setting['inputMacdFastPeriodLen'] = 12 # DIF快线
lineM60Setting['inputMacdSlowPeriodLen'] = 26 # DEA慢线
lineM60Setting['inputMacdSignalPeriodLen'] = 9 # MACD中绿柱
lineM60Setting['shortSymbol'] = self.shortSymbol
self.lineM60 = CtaLineBar(self, self.onBarM60, lineM60Setting)
try:
mode = setting['mode']
if mode != EMPTY_STRING:
self.lineM60.setMode(setting['mode'])
except KeyError:
self.lineM60.setMode(self.lineM60.TICK_MODE)
self.onInit()
# ----------------------------------------------------------------------
def onInit(self, force=False):
"""初始化 """
if force:
self.writeCtaLog(u'策略强制初始化')
self.inited = False
self.trading = False # 控制是否启动交易
else:
self.writeCtaLog(u'策略初始化')
if self.inited:
self.writeCtaLog(u'已经初始化过，不再执行')
return
self.position.pos = EMPTY_INT # 初始化持仓
self.entrust = EMPTY_INT # 初始化委托状态
self.percent = EMPTY_INT # 初始化仓位状态
if not self.backtesting:
# 这里需要加载前置数据哦。
if not self.__initDataFromSina():
return
self.inited = True # 更新初始化标识
self.trading = True # 启动交易
self.putEvent()
self.writeCtaLog(u'策略初始化完成')
def __initDataFromSina(self):
"""从sina初始化5分钟数据"""
sina = UtilSinaClient(self)
ret = sina.getMinBars(symbol=self.symbol, minute=15, callback=self.lineM15.addBar)
if not ret:
self.writeCtaLog(u'获取M15数据失败')
return False
ret = sina.getMinBars(symbol=self.symbol, minute=60, callback=self.lineM60.addBar)
if not ret:
self.writeCtaLog(u'获取M60数据失败')
return False
return True
def onStart(self):
"""启动策略（必须由用户继承实现）"""
self.writeCtaLog(u'启动')
# ----------------------------------------------------------------------
def onStop(self):
"""停止策略（必须由用户继承实现）"""
self.uncompletedOrders.clear()
self.position.pos = EMPTY_INT
self.entrust = EMPTY_INT
self.percent = EMPTY_INT
self.writeCtaLog(u'停止')
self.putEvent()
# ----------------------------------------------------------------------
def onTrade(self, trade):
"""交易更新"""
self.writeCtaLog(u'{0},OnTrade(),当前持仓：{1},当前仓位：{2} '.format(self.curDateTime, self.position.pos, self.percent))
# ----------------------------------------------------------------------
def onOrder(self, order):
"""报单更新"""
self.writeCtaLog(
u'OnOrder()报单更新，orderID:{0},{1},totalVol:{2},tradedVol:{3},offset:{4},price:{5},direction:{6},status:{7}'
.format(order.orderID, order.vtSymbol, order.totalVolume, order.tradedVolume,
order.offset, order.price, order.direction, order.status))
# 委托单主键，vnpy使用 "gateway.orderid" 的组合
orderkey = order.gatewayName + u'.' + order.orderID
if orderkey in self.uncompletedOrders:
if order.totalVolume == order.tradedVolume:
# 开仓，平仓委托单全部成交
# 平空仓完成(cover)
if self.uncompletedOrders[orderkey]['DIRECTION'] == DIRECTION_LONG and order.offset != OFFSET_OPEN:
self.writeCtaLog(u'平空仓完成，原持仓:{0}，原仓位{1}'.format(self.position.pos, self.percent))
self.position.closePos(direction=DIRECTION_LONG, vol=order.tradedVolume)
self.writeCtaLog(u'新持仓:{0},新仓位{1}'.format(self.position.pos, self.percent))
# 平多仓完成(sell)
if self.uncompletedOrders[orderkey]['DIRECTION'] == DIRECTION_SHORT and order.offset != OFFSET_OPEN:
self.writeCtaLog(u'平空仓完成，原持仓:{0}，原仓位{1}'.format(self.position.pos, self.percent))
self.position.closePos(direction=DIRECTION_SHORT, vol=order.tradedVolume)
self.writeCtaLog(u'新持仓:{0},新仓位{1}'.format(self.position.pos, self.percent))
# 开多仓完成
if self.uncompletedOrders[orderkey]['DIRECTION'] == DIRECTION_LONG and order.offset == OFFSET_OPEN:
self.writeCtaLog(u'平空仓完成，原持仓:{0}，原仓位{1}'.format(self.position.pos, self.percent))
self.position.openPos(direction=DIRECTION_LONG, vol=order.tradedVolume, price=order.price)
self.writeCtaLog(u'新持仓:{0},新仓位{1}'.format(self.position.pos, self.percent))
# 开空仓完成
if self.uncompletedOrders[orderkey]['DIRECTION'] == DIRECTION_SHORT and order.offset == OFFSET_OPEN:
# 更新仓位
self.writeCtaLog(u'平空仓完成，原持仓:{0}，原仓位{1}'.format(self.position.pos, self.percent))
self.position.openPos(direction=DIRECTION_SHORT, vol=order.tradedVolume, price=order.price)
self.writeCtaLog(u'新持仓:{0},新仓位{1}'.format(self.position.pos, self.percent))
del self.uncompletedOrders[orderkey]
if len(self.uncompletedOrders) == 0:
self.entrust = 0
self.lastOrderTime = None
elif order.tradedVolume > 0 and not order.totalVolume == order.tradedVolume and order.offset != OFFSET_OPEN:
# 平仓委托单部分成交
pass
elif order.offset == OFFSET_OPEN and order.status == STATUS_CANCELLED:
# 开仓委托单被撤销
self.entrust = 0
pass
else:
self.writeCtaLog(u'OnOrder()委托单返回，total:{0},traded:{1}'
.format(order.totalVolume, order.tradedVolume, ))
self.putEvent() # 更新监控事件
# ----------------------------------------------------------------------
def onStopOrder(self, orderRef):
"""停止单更新"""
self.writeCtaLog(u'{0},停止单触发，orderRef:{1}'.format(self.curDateTime, orderRef))
pass
# ----------------------------------------------------------------------
def onTick(self, tick):
"""行情更新
:type tick: object
"""
self.curTick = tick
if (tick.datetime.hour >= 3 and tick.datetime.hour <= 8) or (
tick.datetime.hour >= 16 and tick.datetime.hour <= 20):
self.writeCtaLog(u'休市/集合竞价排名时数据不处理')
return
# 更新策略执行的时间（用于回测时记录发生的时间）
self.curDateTime = tick.datetime
# 2、计算交易时间和平仓时间
self.__timeWindow(self.curDateTime)
# 推送Tick到lineM15
self.lineM15.onTick(tick)
# 首先检查是否是实盘运行还是数据预处理阶段
if not (self.inited and len(self.lineM15.inputMacdSlowPeriodLen) > 0):
return
# ----------------------------------------------------------------------
def onBar(self, bar):
"""分钟K线数据更新（仅用于回测时，从策略外部调用)"""
# 更新策略执行的时间（用于回测时记录发生的时间）
# 回测数据传送的bar.datetime，为bar的开始时间，所以，到达策略时，当前时间为bar的结束时间
self.curDateTime = bar.datetime + timedelta(seconds=self.lineM15.barTimeInterval)
# 2、计算交易时间和平仓时间
self.__timeWindow(bar.datetime)
# 推送tick到15分钟K线
self.lineM15.addBar(bar)
self.lineM60.addBar(bar)
# 4、交易逻辑
# 首先检查是否是实盘运行还是数据预处理阶段
if not self.inited:
if len(self.lineM15.lineBar) > 120 + 5:
self.inited = True
else:
return
# ----------------------------------------------------------------------
def onBarM60(self, bar):
"""60分钟上的多空仓决定仓位管理"""
# 调用lineM60的显示bar内容
self.writeCtaLog(self.lineM60.displayLastBar())
# 未初始化完成
if not self.inited:
if len(self.lineM60.lineBar) > 120 + 5:
self.inited = True
else:
return
# 执行撤单逻辑
self.__cancelLogic(dt=self.curDateTime)
# ----------------------------------------------------------------------
def onBarM15(self, bar):
"""分钟K线数据更新，实盘时，由self.lineM15的回调"""
# 调用lineM15的显示bar内容
self.writeCtaLog(self.lineM15.displayLastBar())
# 未初始化完成
if not self.inited:
if len(self.lineM15.lineBar) > 120 + 5:
self.inited = True
else:
return
# 执行撤单逻辑
self.__cancelLogic(dt=self.curDateTime)
if self.lineM15.mode == self.lineM15.TICK_MODE:
idx = 2
else:
idx = 1
jincha_60f = self.lineM60.lineDif[-1 - idx] < self.lineM60[-1 - idx] \
and self.lineM60.lineDif[0 - idx] > self.lineM60.lineDea[0 - idx]
sicha_60f = self.lineM60.lineDif[-1 - idx] > self.lineM60.lineDea[-1 - idx] \
and self.lineM60.lineDif[0 - idx] < self.lineM60.lineDea[0 - idx]
jincha_15f = self.lineM15.lineDif[-1 - idx] < self.lineM15.lineDea[-1 - idx] \
and self.lineM15.lineDif[0 - idx] > self.lineM15.lineDea[0 - idx] \
and abs(self.lineM15.lineMacd[0 - idx]) >= 2
sicha_15f = self.lineM15.lineDif[-1 - idx] > self.lineM15.lineDea[-1 - idx] \
and self.lineM15.lineDif[0 - idx] < self.lineM15.lineDea[0 - idx] \
and abs(self.lineM15.lineMacd[0 - idx]) > 2
# 如果未持仓，检查是否符合开仓逻辑
if self.position.pos == 0:
"""
15f上决定开仓方向，60f与15f方向是否相同决定开仓仓位
"""
# 60分钟上金叉
if jincha_60f:
# 60F金叉的同时15f上金叉，做多，双金叉，仓位放至40%
# 如果要macd大于2的代码and abs(self.lineM15.lineMacd[0 - idx]) > 2
if jincha_15f:
self.percentLimit = 0.4
vol = self.getAvailablePos(bar)
if not vol:
return
self.writeCtaLog(u'{0},开仓多单{1}手,价格:{2}'.format(bar.datetime, vol, bar.close))
orderid = self.buy(price=bar.close, volume=vol, orderTime=self.curDateTime)
if orderid:
self.lastOrderTime = self.curDateTime
return
# 60f金叉的同时15f上死叉，60f与15f不共振，原有多仓
if sicha_15f:
self.percentLimit = 0.2
vol = self.getAvailablePos(bar)
if not vol:
return
self.writeCtaLog(u'{0},开仓空单{1}手,价格:{2}'.format(bar.datetime, vol, bar.close))
orderid = self.short(price=bar.close, volume=vol, orderTime=self.curDateTime)
if orderid:
self.lastOrderTime = self.curDateTime
return
# 60分钟上死叉
if sicha_60f:
# 60F死叉的同时15f上金叉，做多，双金叉，仓位放至20%
# 如果要macd大于2的代码and abs(self.lineM15.lineMacd[0 - idx]) > 2
if jincha_15f:
self.percentLimit = 0.2
vol = self.getAvailablePos(bar)
if not vol:
return
self.writeCtaLog(u'{0},开仓多单{1}手,价格:{2}'.format(bar.datetime, vol, bar.close))
orderid = self.buy(price=bar.close, volume=vol, orderTime=self.curDateTime)
if orderid:
self.lastOrderTime = self.curDateTime
return
# 60f死叉的同时15f上死叉，60f与15f不共振，仓位放至40%
if sicha_15f:
self.percentLimit = 0.4
vol = self.getAvailablePos(bar)
if not vol:
return
self.writeCtaLog(u'{0},开仓空单{1}手,价格:{2}'.format(bar.datetime, vol, bar.close))
orderid = self.short(price=bar.close, volume=vol, orderTime=self.curDateTime)
if orderid:
self.lastOrderTime = self.curDateTime
return
"""
如果原来有持仓
60f金叉+15f金叉的时
"""
elif self.position.pos >= 1 and self.percent >= 30 %: # 最大仓位限定在40%且多头仓位的时候，即60f金叉+15f金叉
if jincha_60f and sicha_15f: # 40%的多仓全平同时开20%的空仓
self.writeCtaLog(u'{0},平仓多单{1}手,价格:{2}'.format(bar.datetime, self.position.pos, bar.close))
orderid = self.sell(price=bar.close, volume=self.position.pos, orderTime=self.curDateTime)
if orderid:
self.lastOrderTime = self.curDateTime
if sicha_60f:
if sicha_60f and sicha_15f:
# ----------------------------------------------------------------------
def __cancelLogic(self, dt, force=False):
"撤单逻辑"""
if len(self.uncompletedOrders) < 1:
return
if not self.lastOrderTime:
self.writeCtaLog(u'异常，上一交易时间为None')
return
# 平仓检查时间比开开仓时间需要短一倍
if (self.position.pos >= 0 and self.entrust == 1) \
or (self.position.pos <= 0 and self.entrust == -1):
i = 1
else:
i = 1 # 原来是2，暂时取消
canceled = False
if ((dt - self.lastOrderTime).seconds > self.cancelSeconds / i) \
or force: # 超过设置的时间还未成交
for order in self.uncompletedOrders.keys():
self.writeCtaLog(u'{0}超时{1}秒未成交，取消委托单：{2}'.format(dt, (dt - self.lastOrderTime).seconds, order))
self.cancelOrder(str(order))
canceled = True
# 取消未完成的订单
self.uncompletedOrders.clear()
if canceled:
self.entrust = 0
else:
self.writeCtaLog(u'异常：没有撤单')
def __timeWindow(self, dt):
"""交易与平仓窗口"""
# 交易窗口 避开早盘和夜盘的前5分钟，防止隔夜跳空。
self.closeWindow = False
self.tradeWindow = False
self.openWindow = False
# 初始化当日的首次交易
# if (tick.datetime.hour == 9 or tick.datetime.hour == 21) and tick.datetime.minute == 0 and tick.datetime.second ==0:
# self.firstTrade = True
# 开市期，波动较大，用于判断止损止盈，或开仓
if (dt.hour == 9 or dt.hour == 21) and dt.minute < 2:
self.openWindow = True
# 日盘
if dt.hour == 9 and dt.minute >= 0:
self.tradeWindow = True
return
if dt.hour == 10:
if dt.minute <= 15 or dt.minute >= 30:
self.tradeWindow = True
return
if dt.hour == 11 and dt.minute <= 30:
self.tradeWindow = True
return
if dt.hour == 13 and dt.minute >= 30:
self.tradeWindow = True
return
if dt.hour == 14:
if dt.minute < 59:
self.tradeWindow = True
return
if dt.minute == 59: # 日盘平仓
self.closeWindow = True
return
# 夜盘
if dt.hour == 21 and dt.minute >= 0:
self.tradeWindow = True
return
# 上期 贵金属， 次日凌晨2:30
if self.shortSymbol in NIGHT_MARKET_SQ1:
if dt.hour == 22 or dt.hour == 23 or dt.hour == 0 or dt.hour == 1:
self.tradeWindow = True
return
if dt.hour == 2:
if dt.minute < 29: # 收市前29分钟
self.tradeWindow = True
return
if dt.minute == 29: # 夜盘平仓
self.closeWindow = True
return
return
# 上期 有色金属，黑色金属，沥青 次日01:00
if self.shortSymbol in NIGHT_MARKET_SQ2:
if dt.hour == 22 or dt.hour == 23:
self.tradeWindow = True
return
if dt.hour == 0:
if dt.minute < 59: # 收市前29分钟
self.tradeWindow = True
return
if dt.minute == 59: # 夜盘平仓
self.closeWindow = True
return
return
# 上期 天然橡胶 23:00
if self.shortSymbol in NIGHT_MARKET_SQ3:
if dt.hour == 22:
if dt.minute < 59: # 收市前1分钟
self.tradeWindow = True
return
if dt.minute == 59: # 夜盘平仓
self.closeWindow = True
return
# 郑商、大连 23:30
if self.shortSymbol in NIGHT_MARKET_ZZ or self.shortSymbol in NIGHT_MARKET_DL:
if dt.hour == 22:
self.tradeWindow = True
return
if dt.hour == 23:
if dt.minute < 29: # 收市前1分钟
self.tradeWindow = True
return
if dt.minute == 29 and dt.second > 30: # 夜盘平仓
self.closeWindow = True
return
return
# ----------------------------------------------------------------------
def strToTime(self, t, ms):
"""从字符串时间转化为time格式的时间"""
hh, mm, ss = t.split(':')
tt = datetime.time(int(hh), int(mm), int(ss), microsecond=ms)
return tt
# ----------------------------------------------------------------------
def saveData(self, id):
"""保存过程数据"""
# 保存K线
if not self.backtesting:
return
# ----------------------------------------------------------------------
# add by xy 12th August 2017
def getAvailablePos(self, bar):
capital, avail, _, _ = self.engine.getAccountInfo()
avail = min(avail, capital * self.percentLimit)
midPrice = (bar.high - bar.low) / 2 + bar.low
pricePerLot = self.engine.moneyPerLot(midPrice, self.vtSymbol)
if pricePerLot:
return int(avail / pricePerLot)
else:
return None
def testRbByTick():
# 创建回测引擎
engine = BacktestingEngine()
# 设置引擎的回测模式为Tick
engine.setBacktestingMode(engine.TICK_MODE)
# 设置回测用的数据起始日期
engine.setStartDate('20100101')
# 设置回测用的数据结束日期
engine.setEndDate('20160330')
# engine.connectMysql()
engine.setDatabase(dbName='stockcn', symbol='rb')
# 设置产品相关参数
engine.setSlippage(0.5) # 1跳（0.1）2跳0.2
engine.setRate(float(0.0001)) # 万1
engine.setSize(10) # 合约大小
settings = {}
settings['vtSymbol'] = 'RB'
settings['shortSymbol'] = 'RB'
settings['name'] = 'MACD'
settings['mode'] = 'tick'
settings['backtesting'] = True
# 在引擎中创建策略对象
engine.initStrategy(Strategy_MACD_01, setting=settings)
# 使用简单复利模式计算
engine.usageCompounding = False # True时，只针对FINAL_MODE有效
# 启用实时计算净值模式REALTIME_MODE / FINAL_MODE 回测结束时统一计算模式
engine.calculateMode = engine.REALTIME_MODE
engine.initCapital = 100000 # 设置期初资金
engine.percentLimit = 30 # 设置资金使用上限比例(%)
engine.barTimeInterval = 60 * 5 # bar的周期秒数，用于csv文件自动减时间
engine.fixCommission = 10 # 固定交易费用（每次开平仓收费）
# 开始跑回测
engine.runBacktestingWithMysql()
# 显示回测结果
engine.showBacktestingResult()
def testRbByBar():
# 创建回测引擎
engine = BacktestingEngine()
# 设置引擎的回测模式为Tick
engine.setBacktestingMode(engine.BAR_MODE)
# 设置回测用的数据起始日期
engine.setStartDate('20170101')
# 设置回测用的数据结束日期
engine.setEndDate('20170605')
# engine.setDatabase(dbName='stockcn', symbol='rb')
engine.setDatabase(dbName='stockcn', symbol='RB')
# 设置产品相关参数
engine.setSlippage(0.5) # 1跳（0.1）2跳0.2
engine.setRate(float(0.0003)) # 万3
engine.setSize(10) # 合约大小
settings = {}
# settings['vtSymbol'] = 'rb'
settings['vtSymbol'] = 'RB'
settings['shortSymbol'] = 'RB'
settings['name'] = 'MACD'
settings['mode'] = 'bar'
settings['backtesting'] = True
settings['percentLimit'] = 30
# 在引擎中创建策略对象
engine.initStrategy(Strategy_MACD_01, setting=settings)
# 使用简单复利模式计算
engine.usageCompounding = False # True时，只针对FINAL_MODE有效
# 启用实时计算净值模式REALTIME_MODE / FINAL_MODE 回测结束时统一计算模式
engine.calculateMode = engine.REALTIME_MODE
engine.initCapital = 100000 # 设置期初资金
engine.percentLimit = 40 # 设置资金使用上限比例(%)
engine.barTimeInterval = 300 # bar的周期秒数，用于csv文件自动减时间
# 开始跑回测
engine.runBackTestingWithBarFile(os.getcwd() + '/cache/RB99_20100101_20170605_15m.csv')
# 显示回测结果
engine.showBacktestingResult()
# 从csv文件进行回测
if __name__ == '__main__':
# 提供直接双击回测的功能
# 导入PyQt4的包是为了保证matplotlib使用PyQt4而不是PySide，防止初始化出错
from ctaBacktesting import *
from setup_logger import setup_logger
setup_logger(
filename=u'TestLogs/{0}_{1}.log'.format(Strategy_MACD_01.className, datetime.now().strftime('%m%d_%H%M')),
debug=False
)
# 回测螺纹
testRbByBar()
long_id = proc.stdout.read().strip()
assert cores > 0
print sys._getframe().f_code.co_name, 1
def test_features_crawler_crawl_outcontainer_mem():
# Start a dummy container
proc = subprocess.Popen(
"docker run -d ubuntu sleep 60",
shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
long_id = proc.stdout.read().strip()
c = DockerContainer(long_id)
crawler = FeaturesCrawler(crawl_mode='OUTCONTAINER', container=c)
output = "%s" % list(crawler.crawl_memory())
#print output
# Kill the dummy container
proc = subprocess.Popen(
"docker rm -f %s" % long_id,
shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
long_id = proc.stdout.read().strip()
assert 'memory_used' in output
print sys._getframe().f_code.co_name, 1
if __name__ == '__main__':
setup_logger("crawlutils", "tester.log")
test_features_crawler_crawl_invm_cpu()
test_features_crawler_crawl_outcontainer_cpu()
test_features_crawler_crawl_invm_mem()
test_features_crawler_crawl_outcontainer_mem()
def __init__(self, operation):
self.operation = operation
self.logger = setup_logger('performance')
from vnpy.trader.vtEngine import MainEngine
from vnpy.trader.uiMainWindow import *
# 加载底层接口
from gateway import ctpGateway
# 初始化的接口模块，以及其指定的名称,CTP是模块，value，是该模块下的多个连接配置文件,如 CTP_JR_connect.json
init_gateway_names = {'CTP': ['CTP', 'CTP_Prod', 'CTP_Post', 'CTP_EBF', 'CTP_JR', 'CTP_JR2']}
# 文件路径名
path = os.path.abspath(os.path.dirname(__file__))
ICON_FILENAME = 'vnpy.ico'
ICON_FILENAME = os.path.join(path, ICON_FILENAME)
from setup_logger import setup_logger
setup_logger(filename='logs/vnpy_{0}.log'.format(datetime.now().strftime('%m%d_%H%M')), debug=False)
# ----------------------------------------------------------------------
def main():
"""主程序入口"""
# 重载sys模块，设置默认字符串编码方式为utf8
reload(sys)
sys.setdefaultencoding('utf8')
# 设置Windows底部任务栏图标
if 'Windows' in platform.uname():
ctypes.windll.shell32.SetCurrentProcessExplicitAppUserModelID('vn.trader')
# 初始化Qt应用对象
app = QtGui.QApplication(sys.argv)
app.setWindowIcon(QtGui.QIcon(ICON_FILENAME))
def main():
'''Read command line arguments to run dismod for all child location ids of
given location ids.
Args:
mvid(int): model version id
location_id(int): parent location id
sex(str): one of 'male'/'female'
year_id(int): year id
debug(str, optional): If specified and value == 'debug', will run
in serial instead of in parallel
'''
mvid = int(sys.argv[1])
location_id = int(sys.argv[2])
sex = sys.argv[3]
y = int(sys.argv[4])
cv_iter = int(sys.argv[5])
setup_logger()
log = logging.getLogger(__name__)
log.info(
"Starting cascade mvid {} loc {} sex {} year {} cv_iter {}".format(
mvid, location_id, sex, y, cv_iter))
# The cascade and parent information are shared across all subprocesses.
# Make it a global to avoid the memory overhead of passing a copy to
# each process
global cascade
global cl_parent
try:
if sys.argv[6] == "debug":
debug = True
else:
debug = False
except:
debug = False
if sex == 'male':
sex_id = 0.5
elif sex == 'female':
sex_id = -0.5
log.info("Creating cascade")
cascade = Cascade(mvid, reimport=False, cv_iter=cv_iter)
log.info("Done with cascade")
year_split_lvl = cascade.model_version_meta.fix_year.values[0] - 1
lt = cascade.loctree
this_lvl = lt.get_nodelvl_by_id(location_id)
log.info("Generating cascade loc")
if location_id == 1:
cl_parent = Cascade_loc(location_id, 0, 2000, cascade, reimport=False)
else:
cl_parent = Cascade_loc(location_id,
sex_id,
y,
cascade,
reimport=False)
num_children = len(lt.get_node_by_id(location_id).children)
log.info("Done generating cascade loc")
num_cpus = mp.cpu_count()
num_workers = min(num_cpus, num_children, 10)
if not debug:
pool = mp.Pool(num_workers)
# Run child locations
arglist = []
for child_loc in lt.get_node_by_id(location_id).children:
if this_lvl >= (year_split_lvl - 1):
full_timespan = False
else:
full_timespan = True
arglist.append((child_loc.id, sex_id, y, full_timespan, debug))
if debug:
'..... RUNNING IN SINGLE PROCESS DEBUG MODE .....'
res = map(run_loc, arglist)
else:
log.info(
"Running {} child locations in parallel with {} processes".format(
len(arglist), num_workers))
res = pool.map(run_loc, arglist)
pool.close()
pool.join()
log.info("Done running")
errors = ['%s: %s' % (str(r[0]), r[1]) for r in res if r[1] != 0]
if len(errors) == 0:
log.info("No errors found")
else:
num_errors = len(errors)
error_msg = "; ".join(errors)
log.error("Found {} errors for mvid {} loc {} sex {} year {} cv_iter"
"{}: {}".format(num_errors, mvid, location_id, sex, y,
cv_iter, error_msg))
def main():
print('------------')
opt.manualSeed = random.randint(1, 100) # 设定随机数
random.seed(opt.manualSeed) # 设定随机种子
torch.manual_seed(opt.manualSeed) # 设定随机种子
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # 设定设备
opt.num_objects = 13 # 训练数据的物体种类数目
opt.num_points = 500 # 输入点云的数目
opt.outf = 'trained_models/linemod' # 训练模型保存的目录
opt.log_dir = 'logs/linemod' # log保存的目录
opt.repeat_epoch = 20 # 重复epoch数目
estimator = PoseNet(num_points=opt.num_points, num_obj=opt.num_objects) # 网络构建，构建完成，对物体的6D姿态进行预测
print(estimator)
estimator.to(device) # 选择设备
refiner = PoseRefineNet(num_points=opt.num_points, num_obj=opt.num_objects) # 对初步预测的姿态进行提炼
print(refiner)
refiner.to(device) # 选择设备
if opt.resume_posenet != '': # 对posenet模型的加载，如果有的话，
estimator.load_state_dict(torch.load('{0}/{1}'.format(opt.outf, opt.resume_posenet)))
if opt.resume_refinenet != '': # 对refinenet模型的加载，如果有的话，
refiner.load_state_dict(torch.load('{0}/{1}'.format(opt.outf, opt.resume_refinenet)))
opt.refine_start = True # 标记refine网络开始训练
opt.decay_start = True # 标记refine网络参数开始衰减
opt.lr *= opt.lr_rate # 学习率变化
opt.w *= opt.w_rate # 权重衰减率变化
opt.batch_size = int(opt.batch_size / opt.iteration) # batchsize设定
optimizer = optim.Adam(refiner.parameters(), lr=opt.lr) # 设定refine的优化器
else:
opt.refine_start = False # 标记refine网络未开始训练
opt.decay_start = False # 标记refine网络参数未开始衰减
optimizer = optim.Adam(estimator.parameters(), lr=opt.lr) # 设定posenet的优化器
# 加载训练数据集
dataset = PoseDataset('train', opt.num_points, True, opt.dataset_root, opt.noise_trans, opt.refine_start)
dataloader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=True, num_workers=opt.workers)
# 加载验证数据集
test_dataset = PoseDataset('test', opt.num_points, False, opt.dataset_root, 0.0, opt.refine_start)
test_dataloder = torch.utils.data.DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=opt.workers)
opt.sym_list = dataset.get_sym_list() # 设定对称物体列表
opt.num_points_mesh = dataset.get_num_points_mesh() # 设定点云数目
print('----------Dataset loaded!---------\nlength of the training set: {0}\nlength of the testing set: {1}\nnumber '
'of sample points on mesh: {2}\nsymmetry object list: {3}'.format(len(dataset), len(test_dataset),
opt.num_points_mesh, opt.sym_list))
criterion = Loss(opt.num_points_mesh, opt.sym_list) # loss计算
criterion_refine = Loss_refine(opt.num_points_mesh, opt.sym_list) # refine_loss计算
best_test = np.Inf # 初始位置最好的模型,loss无限大
if opt.start_epoch == 1: # 开始训练，则删除之前的日志
for log in os.listdir(opt.log_dir):
os.remove(os.path.join(opt.log_dir, log))
st_time = time.time() # 记录开始时间
# 开始循环迭代，训练模型-----------------------------------!
for epoch in range(opt.start_epoch, opt.nepoch):
# 保存开始迭代的log信息
logger = setup_logger('epoch%d' % epoch, os.path.join(opt.log_dir, 'epoch_%d_log.txt' % epoch))
# 记录每个epoch时间
logger.info('Train time {0}'.format(time.strftime('%Hh %Mm %Ss', time.gmtime(time.time() - st_time)) + 'Training started'))
train_count = 0
train_dis_avg = 0.0 # 用于计算平均距离
if opt.refine_start: # 如果refine模型，已经开始训练
estimator.eval()
refiner.train()
else:
estimator.train()
optimizer.zero_grad() # 优化器清零梯度
for rep in range(opt.repeat_epoch): # 每次epoch重复训练次数
for i, data in enumerate(dataloader, 0):
points, choose, img, target, model_points, idx = data # 读取数据
'''
points: 由深度图计算出来的点云，该点云数据以摄像头为参考坐标
choose: 所选择点云的索引[bs, 1, 500]
img: 通过box剪切下来的RGB图像
target: 根据model_points点云信息，以及旋转偏移矩阵转换过的点云信息[bs, 500, 3]
model_points: 目标初始帧对应的点云信息
idx: 训练图片的下标
'''
# 将数据放到device上
points, choose, img, target, model_points, idx = points.to(device), choose.to(device), img.to(device), target.to(device), model_points.to(device), idx.to(device)
# 进行预测获得预测的姿态，和特征向量
pred_r, pred_t, pred_c, emb = estimator(img, points, choose, idx)
'''
pred_r: 旋转矩阵[bs, 500, 4]
pred_t: 偏移矩阵[bs, 500, 3]
pred_c: 置信度[bs, 500, 1]
'''
# 计算loss
loss, dis, new_points, new_target = criterion(pred_r, pred_t, pred_c, target, model_points, idx, points, opt.w, opt.refine_start)
# 如果已经开始了refiner模型的训练
if opt.refine_start:
for iter in range(0, opt.iteration):
pred_r, pred_t = refiner(new_points, emb, idx) # 进行refiner预测
# 计算loss得到dis
dis, new_points, new_target = criterion_refine(pred_r, pred_t, new_target, model_points, idx, new_points)
dis.backward() # dis进行反向传播
else:
loss.backward() # 否则，则对loss进行反向传播
train_dis_avg += dis.item() # 用于计算平均距离
train_count += 1
# log信息存储
if train_count % opt.batch_size == 0:
logger.info('Train time {0} Epoch {1} Batch {2} Frame {3} Avg_dis:{4}'
.format(time.strftime('%Hh %Mm %Ss', time.gmtime(time.time() - st_time)),
epoch, int(train_count / opt.batch_size), train_count, train_dis_avg / opt.batch_size))
optimizer.step() # 优化器更新
optimizer.zero_grad() # 优化器梯度清零
train_dis_avg = 0.0
if train_count != 0 and train_count % 1000 == 0: # 模型保存
if opt.refine_start: # 已经开始refine模型
torch.save(refiner.state_dict(), '{0}/pose_refine_model_current.pth'.format(opt.outf))
else:
torch.save(estimator.state_dict(), '{0}.pos_model_current.pth'.format(opt.outf))
print('------------ epoch {0} train finish -----------'.format(epoch))
# 进行测试
logger = setup_logger('epoch%d test' % epoch, os.path.join(opt.log_dir, 'epoch_%d_test_log.txt' % epoch))
# 记录测试时间
logger.info('Test time {0}'.format(time.strftime('%Hh %Mm %Ss', time.gmtime(time.time() - st_time)) + ',' + 'Testing started'))
test_dis = 0.0
test_count = 0
estimator.eval() # 验证模型构建
refiner.eval() # refiner模型
for j, data in enumerate(test_dataloder, 0):
points, choose, img, target, model_points, idx = data # 读取数据
'''
points: 由深度图计算出来的点云，该点云数据以摄像头为参考坐标
choose: 所选择点云的索引[bs, 1, 500]
img: 通过box剪切下来的RGB图像
target: 根据model_points点云信息，以及旋转偏移矩阵转换过的点云信息[bs, 500, 3]
model_points: 目标初始帧对应的点云信息
idx: 训练图片的下标
'''
# 将数据放到device上
points, choose, img, target, model_points, idx = points.to(device), choose.to(device), img.to(
device), target.to(device), model_points.to(device), idx.to(device)
# 进行预测获得预测的姿态，和特征向量
pred_r, pred_t, pred_c, emb = estimator(img, points, choose, idx)
'''
pred_r: 旋转矩阵[bs, 500, 4]
pred_t: 偏移矩阵[bs, 500, 3]
pred_c: 置信度[bs, 500, 1]
'''
# 对结果进行评估
_, dis, new_points, new_target = criterion(pred_r, pred_t, pred_c, target, model_points, idx, points, opt.w, opt.refine_start)
# 如果refine模型开始训练，则同样进行评估
if opt.refine_start:
for iter in range(0, opt.iteration):
pred_r, pred_t = refiner(new_points, emb, idx)
dis, new_points, new_target = criterion_refine(pred_r, pred_t, new_target, model_points, idx, new_points)
test_dis += dis.item() # 用于计算平均距离
# 保存eval的log
logger.info('Test time {0} Test Frame No.{1} dis:{2}'.format(time.strftime('%Hh %Mm %Ss', time.gmtime(time.time() - st_time)), test_count, dis))
test_count += 1
test_dis = test_dis / test_count # 计算平均距离
logger.info('Test time {0} Epoch {1} Test finish avg_dis:{2}'.format(time.strftime('%Hh %Mm %Ss', time.gmtime(time.time() - st_time)), epoch, test_dis))
if test_dis <= best_test: # 如果此次测试结果最好，则保留当前测试结果
best_test = test_dis
if opt.refine_start: # 保存refiner
torch.save(refiner.state_dict(), '{0}/pose_refine_model_{1}_{2}.pth'.format(opt.outf, epoch, test_dis))
else:
torch.save(estimator.state_dict(), '{0}/pose_model_{1}_{2}.pth'.format(opt.outf, epoch, test_dis))
print('----------------test model save finished-------------------')
# 参数变化
# 判断模型是否达到衰减要求
if best_test < opt.decay_margin and not opt.decay_start:
opt.decay_start = True
opt.lr *= opt.lr_rate # 学习率衰减
opt.w *= opt.w_rate # 权重衰减
optimizer = optim.Adam(estimator.parameters(), lr=opt.lr)
# 模型没有达到loss阈值要求，refine_start = False，则修改相关参数，传递相关数，更新dataset和dataloader
if best_test < opt.refine_margin and not opt.refine_start:
opt.refine_start = True
opt.batch_size = int(opt.batch_size / opt.iteration)
optimizer = optim.Adam(refiner.parameters(), lr=opt.lr)
# 训练
dataset = PoseDataset('train', opt.num_points, True, opt.dataset_root, opt.noise_trans, opt.refine_start)
dataloader = DataLoader(dataset, batch_size=1, shuffle=True, num_workers=opt.workers)
# 测试
test_dataset = PoseDataset('test', opt.num_points, False, opt.dataset_root, 0.0, opt.refine_start)
test_dataloder = DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=opt.workers)
opt.sym_list = dataset.get_sym_list()
opt.num_points_mesh = dataset.get_num_points_mesh()
print('----------Dataset loaded!---------\nlength of the training set: {0}\nlength of the testing set: {1}\nnumber '
'of sample points on mesh: {2}\nsymmetry object list: {3}'.format(len(dataset), len(test_dataset),
opt.num_points_mesh, opt.sym_list))
criterion = Loss(opt.num_points_mesh, opt.sym_list)
criterion_refine = Loss_refine(opt.num_points_mesh, opt.sym_list)
# 使用简单复利模式计算
engine.usageCompounding = False # True时，只针对FINAL_MODE有效
# 启用实时计算净值模式REALTIME_MODE / FINAL_MODE 回测结束时统一计算模式
engine.calculateMode = engine.REALTIME_MODE
engine.initCapital = 100000 # 设置期初资金
engine.percentLimit = 30 # 设置资金使用上限比例(%)
engine.barTimeInterval = 300 # bar的周期秒数，用于csv文件自动减时间
# 开始跑回测
engine.runBackTestingWithBarFile(os.getcwd() +
'/cache/RB88_20160101_20161231_M5.csv')
# 显示回测结果
engine.showBacktestingResult()
# 从csv文件进行回测
if __name__ == '__main__':
# 提供直接双击回测的功能
# 导入PyQt4的包是为了保证matplotlib使用PyQt4而不是PySide，防止初始化出错
from ctaBacktesting import *
from setup_logger import setup_logger
setup_logger(filename=u'TestLogs/{0}_{1}.log'.format(
Strategy_TripleMa.className,
datetime.now().strftime('%m%d_%H%M')),
debug=False)
# 回测螺纹
testRbByBar()
import random
import string
from datetime import timedelta, datetime
from sqlalchemy import create_engine
from sqlalchemy import text
from settings import DATABASES, TABLE_COLUMNS, TABLE_NAME, QUERIES, DB_SIZE
from setup_logger import setup_logger
from timed import Timed
logger = setup_logger('performance')
def print_range(length):
for i in range(length):
print('GENERATING: {0} / {1}'.format(i + 1, length), end='\r')
yield
class Connector:
def __init__(self, dbsystem):
self.dbsystem = dbsystem.upper()
self.QUERIES = QUERIES[self.dbsystem]
self.TABLE_COLUMNS = TABLE_COLUMNS[self.dbsystem]
self.config = DATABASES[self.dbsystem]
self.conn = self.get_engine()
def get_engine(self, with_db=True):
