def main(): ''' Parses command line to Launch/relaunch the dismod cascade from the global level. ''' setup_logger() log = logging.getLogger(__name__) log.debug("main started") import argparse parser = argparse.ArgumentParser("Launch/relaunch the dismod cascade " "from the global level") parser.add_argument('mvid', type=int) parser.add_argument('--submit_stage', type=str, default='bootstrap') parser.add_argument('--cv_iter', type=int, default=0) args = parser.parse_args() submit_stage = args.submit_stage mvid = args.mvid cv_iter = args.cv_iter try: if submit_stage == 'bootstrap': JobTreeBootstrapper(mvid) else: CascadeJobTree(mvid, cv_iter) except Exception: log.exception("error in main run_global with args {}".format( str(args))) upload.update_model_status(mvid, upload.FAILED)
def run_noui(): try: log_file_name = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'logs', u'noUiMain.log')) except Exception as ex: print( u'Use local dict:{0}'.format(os.getcwd())) log_file_name = os.path.abspath(os.path.join(os.getcwd(), 'logs', u'noUiMain.log')) setup_logger(filename=log_file_name, debug=False) noUi = NoUiMain() noUi.Start()
def __init__(self): self.logger = setup_logger("main") self.a1 = A1() self.a2 = A2() self.a3 = A3() self.threads = list() sys.excepthook = self.handle_exception
def main() -> None: logger: Logger = setup_logger(__name__) logger.info("START") # 接頭辞を「;」に設定 bot: WerewolfBot = WerewolfBot(command_prefix=";") bot.game = Game() logger.debug(bot) # 環境変数DISCORD_BOT_TOKENからBOTのトークンを取得 token = os.environ["DISCORD_BOT_TOKEN"] # コマンド「;ping」を追加 # コマンドの内容は ping()を参照 bot.add_command(ping) extension_list = [ "cogs.players_cog", "cogs.game_status_cog", ] for extension in extension_list: bot.load_extension(name=extension) # bot実行 bot.run(token)
def main(): '''Set commit hash, upload model, try to write effects_plots pdfs, aggregate model version draws up location hierarchy ''' setup_logger() mvid = sys.argv[1] log = logging.getLogger(__name__) log.info("Varnish started for mvid {}".format(mvid)) try: try: commit_hash = sge.get_commit_hash(dir='%s/..' % drill.this_path) except subprocess.CalledProcessError: # in site-packages, not git repo commit_hash = __version__ upload.set_commit_hash(mvid, commit_hash) upload.upload_model(mvid) outdir = "%s/%s/full" % (drill.settings['cascade_ode_out_dir'], str(mvid)) joutdir = "%s/%s" % (drill.settings['diag_out_dir'], mvid) fit_stats.write_fit_stats(mvid, outdir, joutdir) upload.upload_fit_stat(mvid) # Write effect PDFs plotter = "{}/effect_plots.r".format(drill.this_path) plotter = os.path.realpath(plotter) demo = Demographics() try: subprocess.check_output([ "FILEPATH", plotter, str(mvid), joutdir, drill.settings['cascade_ode_out_dir'], str(max(demo.year_ids)) ], stderr=subprocess.STDOUT) except subprocess.CalledProcessError: log.exception("Error in effect plots") # Launch final aggregations log.info("Starting Save Results") aggregate_model(mvid, demo=demo) except Exception: log.exception("Error in varnish") raise
from setup_logger import setup_logger logger = setup_logger(__name__) class Player: """人狼ゲーム参加者 Attributes: id: DiscordユーザID """ def __init__(self, discord_id: int) -> None: logger.debug("Playerクラス init") self.id: int = discord_id
# encoding: UTF-8 import os from setup_logger import setup_logger from vnpy.trader.util_monitor import * from vnpy.trader.vtEngine import MainEngine from vnpy.trader.gateway import ctpGateway from threading import Thread from datetime import datetime setup_logger(debug=True) # ---------------------------------------------------------------------- """ Linux 下: 启动:python noUiMain.py >>logs/mylog.log & 停止:ps -ef | grep python ,找到对应的进程PID, kill -9 PID 查看:tail -f logs/noUiMain_月日_小时分钟.log -f Windows下: 启动:命令行窗口 python noUiMain.py 停止:关闭窗口 """ class NoUiMain(object): def __init__(self): # gateway 是否连接 self.connected = False # gateway 的连接名称,在vtEngine.initGateway()里面定义,对应的配置文件是 "连接名称_connect.json", self.gateway_name = 'CTP' # 启动的策略实例,须在catAlgo/CtaSetting.json 里面定义 [u'S28_RB1001', u'S28_TFT', u'S28_HCRB',u'atr_rsi'] self.strategies = [u'strategyAtrRsi']
# encoding: UTF-8 import sys import ctypes import platform from vtEngine import MainEngine from uiMainWindow import * #from simple_monitor import * from setup_logger import setup_logger setup_logger(filename='logs/vnpy_{0}.log'.format(datetime.now().strftime('%m%d_%H%M')), debug=False) # ---------------------------------------------------------------------- def main(): """主程序入口""" # 重载sys模块,设置默认字符串编码方式为utf8 reload(sys) sys.setdefaultencoding('utf8') # 设置Windows底部任务栏图标 if 'Windows' in platform.uname() : ctypes.windll.shell32.SetCurrentProcessExplicitAppUserModelID('vn.trader') # 初始化Qt应用对象 app = QtGui.QApplication(sys.argv) app.setWindowIcon(QtGui.QIcon('vnpy.ico')) app.setFont(BASIC_FONT)
long_id = proc.stdout.read().strip() assert cores > 0 print sys._getframe().f_code.co_name, 1 def test_features_crawler_crawl_outcontainer_mem(): # Start a dummy container proc = subprocess.Popen( "docker run -d ubuntu sleep 60", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) long_id = proc.stdout.read().strip() c = DockerContainer(long_id) crawler = FeaturesCrawler(crawl_mode='OUTCONTAINER', container=c) output = "%s" % list(crawler.crawl_memory()) #print output # Kill the dummy container proc = subprocess.Popen( "docker rm -f %s" % long_id, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) long_id = proc.stdout.read().strip() assert 'memory_used' in output print sys._getframe().f_code.co_name, 1 if __name__ == '__main__': setup_logger("crawlutils", "tester.log") test_features_crawler_crawl_invm_cpu() test_features_crawler_crawl_outcontainer_cpu() test_features_crawler_crawl_invm_mem() test_features_crawler_crawl_outcontainer_mem()
def __init__(self): self.logger = setup_logger("a2")
def __init__(self): self.logger = setup_logger("a1", stream=False)
class Strategy_MACD_01(CtaTemplate): """螺纹钢、15分钟级别+60分钟级别,MACD策略 v1:15f上多空仓开仓 v2:60f上的仓位管理体系 本版本现存问题: 无 已解决问题: 15f上按照百分比开仓 """ className = 'Strategy_MACD' author = u'横纵19950206' # 策略在外部设置的参数 # ---------------------------------------------------------------------- def __init__(self, ctaEngine, setting=None): """Constructor""" super(Strategy_MACD_01, self).__init__(ctaEngine, setting) # 增加监控参数项目 # 增加监控变量项目 self.varList.append('pos') # 仓位,这里的仓位通常是手数 self.varList.append('entrust') # 是否正在委托,0表示没有委托,1多仓,-1空仓 self.varList.append('percentLimit') # 当前账户最大仓位 self.varList.append('percent') # 当前仓位 self.curDateTime = None # 当前Tick时间 self.curTick = None # 最新的tick self.lastOrderTime = None # 上一次委托时间 self.cancelSeconds = 60 # 撤单时间(秒) # 定义日内的交易窗口 self.openWindow = False # 开市窗口 self.tradeWindow = False # 交易窗口 self.closeWindow = False # 收市平仓窗口 self.inited = False # 是否完成了策略初始化 self.backtesting = False # 是否回测 self.lineM15 = None # 5分钟K线 self.lineM60 = None # 60分钟k线 # 增加仓位管理模块 self.position = CtaPosition(self) # self.position.longPos多头持仓,self.position.shorPos多头持仓、 # self.position.pos持仓状态,self.position.maxPos最大持仓 # 增加ctabacktesing中的仓位管理 if not ctaEngine: self.engine = BacktestingEngine() else: self.engine = ctaEngine # 当前资金,当前可用资金,仓位比例,仓位比例上限 self.capital, self.available, self.percent, self.percentLimit = self.engine.getAccountInfo() if setting: # 根据配置文件更新参数 self.setParam(setting) # 创建的M15 K线 lineM15Setting = {} lineM15Setting['name'] = u'M15' # k线名称 lineM15Setting['barTimeInterval'] = 60 * 15 # K线的Bar时长 lineM15Setting['inputMacdFastPeriodLen'] = 12 # DIF快线 lineM15Setting['inputMacdSlowPeriodLen'] = 26 # DEA慢线 lineM15Setting['inputMacdSignalPeriodLen'] = 9 # MACD中绿柱 lineM15Setting['shortSymbol'] = self.shortSymbol self.lineM15 = CtaLineBar(self, self.onBarM15, lineM15Setting) try: mode = setting['mode'] if mode != EMPTY_STRING: self.lineM15.setMode(setting['mode']) except KeyError: self.lineM15.setMode(self.lineM15.TICK_MODE) lineM60Setting = {} lineM60Setting['name'] = u'M60' # k线名称 lineM60Setting['barTimeInterval'] = 60 * 60 # K线的Bar时长 lineM60Setting['inputMacdFastPeriodLen'] = 12 # DIF快线 lineM60Setting['inputMacdSlowPeriodLen'] = 26 # DEA慢线 lineM60Setting['inputMacdSignalPeriodLen'] = 9 # MACD中绿柱 lineM60Setting['shortSymbol'] = self.shortSymbol self.lineM60 = CtaLineBar(self, self.onBarM60, lineM60Setting) try: mode = setting['mode'] if mode != EMPTY_STRING: self.lineM60.setMode(setting['mode']) except KeyError: self.lineM60.setMode(self.lineM60.TICK_MODE) self.onInit() # ---------------------------------------------------------------------- def onInit(self, force=False): """初始化 """ if force: self.writeCtaLog(u'策略强制初始化') self.inited = False self.trading = False # 控制是否启动交易 else: self.writeCtaLog(u'策略初始化') if self.inited: self.writeCtaLog(u'已经初始化过,不再执行') return self.position.pos = EMPTY_INT # 初始化持仓 self.entrust = EMPTY_INT # 初始化委托状态 self.percent = EMPTY_INT # 初始化仓位状态 if not self.backtesting: # 这里需要加载前置数据哦。 if not self.__initDataFromSina(): return self.inited = True # 更新初始化标识 self.trading = True # 启动交易 self.putEvent() self.writeCtaLog(u'策略初始化完成') def __initDataFromSina(self): """从sina初始化5分钟数据""" sina = UtilSinaClient(self) ret = sina.getMinBars(symbol=self.symbol, minute=15, callback=self.lineM15.addBar) if not ret: self.writeCtaLog(u'获取M15数据失败') return False ret = sina.getMinBars(symbol=self.symbol, minute=60, callback=self.lineM60.addBar) if not ret: self.writeCtaLog(u'获取M60数据失败') return False return True def onStart(self): """启动策略(必须由用户继承实现)""" self.writeCtaLog(u'启动') # ---------------------------------------------------------------------- def onStop(self): """停止策略(必须由用户继承实现)""" self.uncompletedOrders.clear() self.position.pos = EMPTY_INT self.entrust = EMPTY_INT self.percent = EMPTY_INT self.writeCtaLog(u'停止') self.putEvent() # ---------------------------------------------------------------------- def onTrade(self, trade): """交易更新""" self.writeCtaLog(u'{0},OnTrade(),当前持仓:{1},当前仓位:{2} '.format(self.curDateTime, self.position.pos, self.percent)) # ---------------------------------------------------------------------- def onOrder(self, order): """报单更新""" self.writeCtaLog( u'OnOrder()报单更新,orderID:{0},{1},totalVol:{2},tradedVol:{3},offset:{4},price:{5},direction:{6},status:{7}' .format(order.orderID, order.vtSymbol, order.totalVolume, order.tradedVolume, order.offset, order.price, order.direction, order.status)) # 委托单主键,vnpy使用 "gateway.orderid" 的组合 orderkey = order.gatewayName + u'.' + order.orderID if orderkey in self.uncompletedOrders: if order.totalVolume == order.tradedVolume: # 开仓,平仓委托单全部成交 # 平空仓完成(cover) if self.uncompletedOrders[orderkey]['DIRECTION'] == DIRECTION_LONG and order.offset != OFFSET_OPEN: self.writeCtaLog(u'平空仓完成,原持仓:{0},原仓位{1}'.format(self.position.pos, self.percent)) self.position.closePos(direction=DIRECTION_LONG, vol=order.tradedVolume) self.writeCtaLog(u'新持仓:{0},新仓位{1}'.format(self.position.pos, self.percent)) # 平多仓完成(sell) if self.uncompletedOrders[orderkey]['DIRECTION'] == DIRECTION_SHORT and order.offset != OFFSET_OPEN: self.writeCtaLog(u'平空仓完成,原持仓:{0},原仓位{1}'.format(self.position.pos, self.percent)) self.position.closePos(direction=DIRECTION_SHORT, vol=order.tradedVolume) self.writeCtaLog(u'新持仓:{0},新仓位{1}'.format(self.position.pos, self.percent)) # 开多仓完成 if self.uncompletedOrders[orderkey]['DIRECTION'] == DIRECTION_LONG and order.offset == OFFSET_OPEN: self.writeCtaLog(u'平空仓完成,原持仓:{0},原仓位{1}'.format(self.position.pos, self.percent)) self.position.openPos(direction=DIRECTION_LONG, vol=order.tradedVolume, price=order.price) self.writeCtaLog(u'新持仓:{0},新仓位{1}'.format(self.position.pos, self.percent)) # 开空仓完成 if self.uncompletedOrders[orderkey]['DIRECTION'] == DIRECTION_SHORT and order.offset == OFFSET_OPEN: # 更新仓位 self.writeCtaLog(u'平空仓完成,原持仓:{0},原仓位{1}'.format(self.position.pos, self.percent)) self.position.openPos(direction=DIRECTION_SHORT, vol=order.tradedVolume, price=order.price) self.writeCtaLog(u'新持仓:{0},新仓位{1}'.format(self.position.pos, self.percent)) del self.uncompletedOrders[orderkey] if len(self.uncompletedOrders) == 0: self.entrust = 0 self.lastOrderTime = None elif order.tradedVolume > 0 and not order.totalVolume == order.tradedVolume and order.offset != OFFSET_OPEN: # 平仓委托单部分成交 pass elif order.offset == OFFSET_OPEN and order.status == STATUS_CANCELLED: # 开仓委托单被撤销 self.entrust = 0 pass else: self.writeCtaLog(u'OnOrder()委托单返回,total:{0},traded:{1}' .format(order.totalVolume, order.tradedVolume, )) self.putEvent() # 更新监控事件 # ---------------------------------------------------------------------- def onStopOrder(self, orderRef): """停止单更新""" self.writeCtaLog(u'{0},停止单触发,orderRef:{1}'.format(self.curDateTime, orderRef)) pass # ---------------------------------------------------------------------- def onTick(self, tick): """行情更新 :type tick: object """ self.curTick = tick if (tick.datetime.hour >= 3 and tick.datetime.hour <= 8) or ( tick.datetime.hour >= 16 and tick.datetime.hour <= 20): self.writeCtaLog(u'休市/集合竞价排名时数据不处理') return # 更新策略执行的时间(用于回测时记录发生的时间) self.curDateTime = tick.datetime # 2、计算交易时间和平仓时间 self.__timeWindow(self.curDateTime) # 推送Tick到lineM15 self.lineM15.onTick(tick) # 首先检查是否是实盘运行还是数据预处理阶段 if not (self.inited and len(self.lineM15.inputMacdSlowPeriodLen) > 0): return # ---------------------------------------------------------------------- def onBar(self, bar): """分钟K线数据更新(仅用于回测时,从策略外部调用)""" # 更新策略执行的时间(用于回测时记录发生的时间) # 回测数据传送的bar.datetime,为bar的开始时间,所以,到达策略时,当前时间为bar的结束时间 self.curDateTime = bar.datetime + timedelta(seconds=self.lineM15.barTimeInterval) # 2、计算交易时间和平仓时间 self.__timeWindow(bar.datetime) # 推送tick到15分钟K线 self.lineM15.addBar(bar) self.lineM60.addBar(bar) # 4、交易逻辑 # 首先检查是否是实盘运行还是数据预处理阶段 if not self.inited: if len(self.lineM15.lineBar) > 120 + 5: self.inited = True else: return # ---------------------------------------------------------------------- def onBarM60(self, bar): """60分钟上的多空仓决定仓位管理""" # 调用lineM60的显示bar内容 self.writeCtaLog(self.lineM60.displayLastBar()) # 未初始化完成 if not self.inited: if len(self.lineM60.lineBar) > 120 + 5: self.inited = True else: return # 执行撤单逻辑 self.__cancelLogic(dt=self.curDateTime) # ---------------------------------------------------------------------- def onBarM15(self, bar): """分钟K线数据更新,实盘时,由self.lineM15的回调""" # 调用lineM15的显示bar内容 self.writeCtaLog(self.lineM15.displayLastBar()) # 未初始化完成 if not self.inited: if len(self.lineM15.lineBar) > 120 + 5: self.inited = True else: return # 执行撤单逻辑 self.__cancelLogic(dt=self.curDateTime) if self.lineM15.mode == self.lineM15.TICK_MODE: idx = 2 else: idx = 1 jincha_60f = self.lineM60.lineDif[-1 - idx] < self.lineM60[-1 - idx] \ and self.lineM60.lineDif[0 - idx] > self.lineM60.lineDea[0 - idx] sicha_60f = self.lineM60.lineDif[-1 - idx] > self.lineM60.lineDea[-1 - idx] \ and self.lineM60.lineDif[0 - idx] < self.lineM60.lineDea[0 - idx] jincha_15f = self.lineM15.lineDif[-1 - idx] < self.lineM15.lineDea[-1 - idx] \ and self.lineM15.lineDif[0 - idx] > self.lineM15.lineDea[0 - idx] \ and abs(self.lineM15.lineMacd[0 - idx]) >= 2 sicha_15f = self.lineM15.lineDif[-1 - idx] > self.lineM15.lineDea[-1 - idx] \ and self.lineM15.lineDif[0 - idx] < self.lineM15.lineDea[0 - idx] \ and abs(self.lineM15.lineMacd[0 - idx]) > 2 # 如果未持仓,检查是否符合开仓逻辑 if self.position.pos == 0: """ 15f上决定开仓方向,60f与15f方向是否相同决定开仓仓位 """ # 60分钟上金叉 if jincha_60f: # 60F金叉的同时15f上金叉,做多,双金叉,仓位放至40% # 如果要macd大于2的代码and abs(self.lineM15.lineMacd[0 - idx]) > 2 if jincha_15f: self.percentLimit = 0.4 vol = self.getAvailablePos(bar) if not vol: return self.writeCtaLog(u'{0},开仓多单{1}手,价格:{2}'.format(bar.datetime, vol, bar.close)) orderid = self.buy(price=bar.close, volume=vol, orderTime=self.curDateTime) if orderid: self.lastOrderTime = self.curDateTime return # 60f金叉的同时15f上死叉,60f与15f不共振,原有多仓 if sicha_15f: self.percentLimit = 0.2 vol = self.getAvailablePos(bar) if not vol: return self.writeCtaLog(u'{0},开仓空单{1}手,价格:{2}'.format(bar.datetime, vol, bar.close)) orderid = self.short(price=bar.close, volume=vol, orderTime=self.curDateTime) if orderid: self.lastOrderTime = self.curDateTime return # 60分钟上死叉 if sicha_60f: # 60F死叉的同时15f上金叉,做多,双金叉,仓位放至20% # 如果要macd大于2的代码and abs(self.lineM15.lineMacd[0 - idx]) > 2 if jincha_15f: self.percentLimit = 0.2 vol = self.getAvailablePos(bar) if not vol: return self.writeCtaLog(u'{0},开仓多单{1}手,价格:{2}'.format(bar.datetime, vol, bar.close)) orderid = self.buy(price=bar.close, volume=vol, orderTime=self.curDateTime) if orderid: self.lastOrderTime = self.curDateTime return # 60f死叉的同时15f上死叉,60f与15f不共振,仓位放至40% if sicha_15f: self.percentLimit = 0.4 vol = self.getAvailablePos(bar) if not vol: return self.writeCtaLog(u'{0},开仓空单{1}手,价格:{2}'.format(bar.datetime, vol, bar.close)) orderid = self.short(price=bar.close, volume=vol, orderTime=self.curDateTime) if orderid: self.lastOrderTime = self.curDateTime return """ 如果原来有持仓 60f金叉+15f金叉的时 """ elif self.position.pos >= 1 and self.percent >= 30 %: # 最大仓位限定在40%且多头仓位的时候,即60f金叉+15f金叉 if jincha_60f and sicha_15f: # 40%的多仓全平同时开20%的空仓 self.writeCtaLog(u'{0},平仓多单{1}手,价格:{2}'.format(bar.datetime, self.position.pos, bar.close)) orderid = self.sell(price=bar.close, volume=self.position.pos, orderTime=self.curDateTime) if orderid: self.lastOrderTime = self.curDateTime if sicha_60f: if sicha_60f and sicha_15f: # ---------------------------------------------------------------------- def __cancelLogic(self, dt, force=False): "撤单逻辑""" if len(self.uncompletedOrders) < 1: return if not self.lastOrderTime: self.writeCtaLog(u'异常,上一交易时间为None') return # 平仓检查时间比开开仓时间需要短一倍 if (self.position.pos >= 0 and self.entrust == 1) \ or (self.position.pos <= 0 and self.entrust == -1): i = 1 else: i = 1 # 原来是2,暂时取消 canceled = False if ((dt - self.lastOrderTime).seconds > self.cancelSeconds / i) \ or force: # 超过设置的时间还未成交 for order in self.uncompletedOrders.keys(): self.writeCtaLog(u'{0}超时{1}秒未成交,取消委托单:{2}'.format(dt, (dt - self.lastOrderTime).seconds, order)) self.cancelOrder(str(order)) canceled = True # 取消未完成的订单 self.uncompletedOrders.clear() if canceled: self.entrust = 0 else: self.writeCtaLog(u'异常:没有撤单') def __timeWindow(self, dt): """交易与平仓窗口""" # 交易窗口 避开早盘和夜盘的前5分钟,防止隔夜跳空。 self.closeWindow = False self.tradeWindow = False self.openWindow = False # 初始化当日的首次交易 # if (tick.datetime.hour == 9 or tick.datetime.hour == 21) and tick.datetime.minute == 0 and tick.datetime.second ==0: # self.firstTrade = True # 开市期,波动较大,用于判断止损止盈,或开仓 if (dt.hour == 9 or dt.hour == 21) and dt.minute < 2: self.openWindow = True # 日盘 if dt.hour == 9 and dt.minute >= 0: self.tradeWindow = True return if dt.hour == 10: if dt.minute <= 15 or dt.minute >= 30: self.tradeWindow = True return if dt.hour == 11 and dt.minute <= 30: self.tradeWindow = True return if dt.hour == 13 and dt.minute >= 30: self.tradeWindow = True return if dt.hour == 14: if dt.minute < 59: self.tradeWindow = True return if dt.minute == 59: # 日盘平仓 self.closeWindow = True return # 夜盘 if dt.hour == 21 and dt.minute >= 0: self.tradeWindow = True return # 上期 贵金属, 次日凌晨2:30 if self.shortSymbol in NIGHT_MARKET_SQ1: if dt.hour == 22 or dt.hour == 23 or dt.hour == 0 or dt.hour == 1: self.tradeWindow = True return if dt.hour == 2: if dt.minute < 29: # 收市前29分钟 self.tradeWindow = True return if dt.minute == 29: # 夜盘平仓 self.closeWindow = True return return # 上期 有色金属,黑色金属,沥青 次日01:00 if self.shortSymbol in NIGHT_MARKET_SQ2: if dt.hour == 22 or dt.hour == 23: self.tradeWindow = True return if dt.hour == 0: if dt.minute < 59: # 收市前29分钟 self.tradeWindow = True return if dt.minute == 59: # 夜盘平仓 self.closeWindow = True return return # 上期 天然橡胶 23:00 if self.shortSymbol in NIGHT_MARKET_SQ3: if dt.hour == 22: if dt.minute < 59: # 收市前1分钟 self.tradeWindow = True return if dt.minute == 59: # 夜盘平仓 self.closeWindow = True return # 郑商、大连 23:30 if self.shortSymbol in NIGHT_MARKET_ZZ or self.shortSymbol in NIGHT_MARKET_DL: if dt.hour == 22: self.tradeWindow = True return if dt.hour == 23: if dt.minute < 29: # 收市前1分钟 self.tradeWindow = True return if dt.minute == 29 and dt.second > 30: # 夜盘平仓 self.closeWindow = True return return # ---------------------------------------------------------------------- def strToTime(self, t, ms): """从字符串时间转化为time格式的时间""" hh, mm, ss = t.split(':') tt = datetime.time(int(hh), int(mm), int(ss), microsecond=ms) return tt # ---------------------------------------------------------------------- def saveData(self, id): """保存过程数据""" # 保存K线 if not self.backtesting: return # ---------------------------------------------------------------------- # add by xy 12th August 2017 def getAvailablePos(self, bar): capital, avail, _, _ = self.engine.getAccountInfo() avail = min(avail, capital * self.percentLimit) midPrice = (bar.high - bar.low) / 2 + bar.low pricePerLot = self.engine.moneyPerLot(midPrice, self.vtSymbol) if pricePerLot: return int(avail / pricePerLot) else: return None def testRbByTick(): # 创建回测引擎 engine = BacktestingEngine() # 设置引擎的回测模式为Tick engine.setBacktestingMode(engine.TICK_MODE) # 设置回测用的数据起始日期 engine.setStartDate('20100101') # 设置回测用的数据结束日期 engine.setEndDate('20160330') # engine.connectMysql() engine.setDatabase(dbName='stockcn', symbol='rb') # 设置产品相关参数 engine.setSlippage(0.5) # 1跳(0.1)2跳0.2 engine.setRate(float(0.0001)) # 万1 engine.setSize(10) # 合约大小 settings = {} settings['vtSymbol'] = 'RB' settings['shortSymbol'] = 'RB' settings['name'] = 'MACD' settings['mode'] = 'tick' settings['backtesting'] = True # 在引擎中创建策略对象 engine.initStrategy(Strategy_MACD_01, setting=settings) # 使用简单复利模式计算 engine.usageCompounding = False # True时,只针对FINAL_MODE有效 # 启用实时计算净值模式REALTIME_MODE / FINAL_MODE 回测结束时统一计算模式 engine.calculateMode = engine.REALTIME_MODE engine.initCapital = 100000 # 设置期初资金 engine.percentLimit = 30 # 设置资金使用上限比例(%) engine.barTimeInterval = 60 * 5 # bar的周期秒数,用于csv文件自动减时间 engine.fixCommission = 10 # 固定交易费用(每次开平仓收费) # 开始跑回测 engine.runBacktestingWithMysql() # 显示回测结果 engine.showBacktestingResult() def testRbByBar(): # 创建回测引擎 engine = BacktestingEngine() # 设置引擎的回测模式为Tick engine.setBacktestingMode(engine.BAR_MODE) # 设置回测用的数据起始日期 engine.setStartDate('20170101') # 设置回测用的数据结束日期 engine.setEndDate('20170605') # engine.setDatabase(dbName='stockcn', symbol='rb') engine.setDatabase(dbName='stockcn', symbol='RB') # 设置产品相关参数 engine.setSlippage(0.5) # 1跳(0.1)2跳0.2 engine.setRate(float(0.0003)) # 万3 engine.setSize(10) # 合约大小 settings = {} # settings['vtSymbol'] = 'rb' settings['vtSymbol'] = 'RB' settings['shortSymbol'] = 'RB' settings['name'] = 'MACD' settings['mode'] = 'bar' settings['backtesting'] = True settings['percentLimit'] = 30 # 在引擎中创建策略对象 engine.initStrategy(Strategy_MACD_01, setting=settings) # 使用简单复利模式计算 engine.usageCompounding = False # True时,只针对FINAL_MODE有效 # 启用实时计算净值模式REALTIME_MODE / FINAL_MODE 回测结束时统一计算模式 engine.calculateMode = engine.REALTIME_MODE engine.initCapital = 100000 # 设置期初资金 engine.percentLimit = 40 # 设置资金使用上限比例(%) engine.barTimeInterval = 300 # bar的周期秒数,用于csv文件自动减时间 # 开始跑回测 engine.runBackTestingWithBarFile(os.getcwd() + '/cache/RB99_20100101_20170605_15m.csv') # 显示回测结果 engine.showBacktestingResult() # 从csv文件进行回测 if __name__ == '__main__': # 提供直接双击回测的功能 # 导入PyQt4的包是为了保证matplotlib使用PyQt4而不是PySide,防止初始化出错 from ctaBacktesting import * from setup_logger import setup_logger setup_logger( filename=u'TestLogs/{0}_{1}.log'.format(Strategy_MACD_01.className, datetime.now().strftime('%m%d_%H%M')), debug=False ) # 回测螺纹 testRbByBar()
long_id = proc.stdout.read().strip() assert cores > 0 print sys._getframe().f_code.co_name, 1 def test_features_crawler_crawl_outcontainer_mem(): # Start a dummy container proc = subprocess.Popen( "docker run -d ubuntu sleep 60", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) long_id = proc.stdout.read().strip() c = DockerContainer(long_id) crawler = FeaturesCrawler(crawl_mode='OUTCONTAINER', container=c) output = "%s" % list(crawler.crawl_memory()) #print output # Kill the dummy container proc = subprocess.Popen( "docker rm -f %s" % long_id, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) long_id = proc.stdout.read().strip() assert 'memory_used' in output print sys._getframe().f_code.co_name, 1 if __name__ == '__main__': setup_logger("crawlutils", "tester.log") test_features_crawler_crawl_invm_cpu() test_features_crawler_crawl_outcontainer_cpu() test_features_crawler_crawl_invm_mem() test_features_crawler_crawl_outcontainer_mem()
def __init__(self, operation): self.operation = operation self.logger = setup_logger('performance')
from vnpy.trader.vtEngine import MainEngine from vnpy.trader.uiMainWindow import * # 加载底层接口 from gateway import ctpGateway # 初始化的接口模块,以及其指定的名称,CTP是模块,value,是该模块下的多个连接配置文件,如 CTP_JR_connect.json init_gateway_names = {'CTP': ['CTP', 'CTP_Prod', 'CTP_Post', 'CTP_EBF', 'CTP_JR', 'CTP_JR2']} # 文件路径名 path = os.path.abspath(os.path.dirname(__file__)) ICON_FILENAME = 'vnpy.ico' ICON_FILENAME = os.path.join(path, ICON_FILENAME) from setup_logger import setup_logger setup_logger(filename='logs/vnpy_{0}.log'.format(datetime.now().strftime('%m%d_%H%M')), debug=False) # ---------------------------------------------------------------------- def main(): """主程序入口""" # 重载sys模块,设置默认字符串编码方式为utf8 reload(sys) sys.setdefaultencoding('utf8') # 设置Windows底部任务栏图标 if 'Windows' in platform.uname(): ctypes.windll.shell32.SetCurrentProcessExplicitAppUserModelID('vn.trader') # 初始化Qt应用对象 app = QtGui.QApplication(sys.argv) app.setWindowIcon(QtGui.QIcon(ICON_FILENAME))
def main(): '''Read command line arguments to run dismod for all child location ids of given location ids. Args: mvid(int): model version id location_id(int): parent location id sex(str): one of 'male'/'female' year_id(int): year id debug(str, optional): If specified and value == 'debug', will run in serial instead of in parallel ''' mvid = int(sys.argv[1]) location_id = int(sys.argv[2]) sex = sys.argv[3] y = int(sys.argv[4]) cv_iter = int(sys.argv[5]) setup_logger() log = logging.getLogger(__name__) log.info( "Starting cascade mvid {} loc {} sex {} year {} cv_iter {}".format( mvid, location_id, sex, y, cv_iter)) # The cascade and parent information are shared across all subprocesses. # Make it a global to avoid the memory overhead of passing a copy to # each process global cascade global cl_parent try: if sys.argv[6] == "debug": debug = True else: debug = False except: debug = False if sex == 'male': sex_id = 0.5 elif sex == 'female': sex_id = -0.5 log.info("Creating cascade") cascade = Cascade(mvid, reimport=False, cv_iter=cv_iter) log.info("Done with cascade") year_split_lvl = cascade.model_version_meta.fix_year.values[0] - 1 lt = cascade.loctree this_lvl = lt.get_nodelvl_by_id(location_id) log.info("Generating cascade loc") if location_id == 1: cl_parent = Cascade_loc(location_id, 0, 2000, cascade, reimport=False) else: cl_parent = Cascade_loc(location_id, sex_id, y, cascade, reimport=False) num_children = len(lt.get_node_by_id(location_id).children) log.info("Done generating cascade loc") num_cpus = mp.cpu_count() num_workers = min(num_cpus, num_children, 10) if not debug: pool = mp.Pool(num_workers) # Run child locations arglist = [] for child_loc in lt.get_node_by_id(location_id).children: if this_lvl >= (year_split_lvl - 1): full_timespan = False else: full_timespan = True arglist.append((child_loc.id, sex_id, y, full_timespan, debug)) if debug: '..... RUNNING IN SINGLE PROCESS DEBUG MODE .....' res = map(run_loc, arglist) else: log.info( "Running {} child locations in parallel with {} processes".format( len(arglist), num_workers)) res = pool.map(run_loc, arglist) pool.close() pool.join() log.info("Done running") errors = ['%s: %s' % (str(r[0]), r[1]) for r in res if r[1] != 0] if len(errors) == 0: log.info("No errors found") else: num_errors = len(errors) error_msg = "; ".join(errors) log.error("Found {} errors for mvid {} loc {} sex {} year {} cv_iter" "{}: {}".format(num_errors, mvid, location_id, sex, y, cv_iter, error_msg))
def main(): print('------------') opt.manualSeed = random.randint(1, 100) # 设定随机数 random.seed(opt.manualSeed) # 设定随机种子 torch.manual_seed(opt.manualSeed) # 设定随机种子 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # 设定设备 opt.num_objects = 13 # 训练数据的物体种类数目 opt.num_points = 500 # 输入点云的数目 opt.outf = 'trained_models/linemod' # 训练模型保存的目录 opt.log_dir = 'logs/linemod' # log保存的目录 opt.repeat_epoch = 20 # 重复epoch数目 estimator = PoseNet(num_points=opt.num_points, num_obj=opt.num_objects) # 网络构建,构建完成,对物体的6D姿态进行预测 print(estimator) estimator.to(device) # 选择设备 refiner = PoseRefineNet(num_points=opt.num_points, num_obj=opt.num_objects) # 对初步预测的姿态进行提炼 print(refiner) refiner.to(device) # 选择设备 if opt.resume_posenet != '': # 对posenet模型的加载,如果有的话, estimator.load_state_dict(torch.load('{0}/{1}'.format(opt.outf, opt.resume_posenet))) if opt.resume_refinenet != '': # 对refinenet模型的加载,如果有的话, refiner.load_state_dict(torch.load('{0}/{1}'.format(opt.outf, opt.resume_refinenet))) opt.refine_start = True # 标记refine网络开始训练 opt.decay_start = True # 标记refine网络参数开始衰减 opt.lr *= opt.lr_rate # 学习率变化 opt.w *= opt.w_rate # 权重衰减率变化 opt.batch_size = int(opt.batch_size / opt.iteration) # batchsize设定 optimizer = optim.Adam(refiner.parameters(), lr=opt.lr) # 设定refine的优化器 else: opt.refine_start = False # 标记refine网络未开始训练 opt.decay_start = False # 标记refine网络参数未开始衰减 optimizer = optim.Adam(estimator.parameters(), lr=opt.lr) # 设定posenet的优化器 # 加载训练数据集 dataset = PoseDataset('train', opt.num_points, True, opt.dataset_root, opt.noise_trans, opt.refine_start) dataloader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=True, num_workers=opt.workers) # 加载验证数据集 test_dataset = PoseDataset('test', opt.num_points, False, opt.dataset_root, 0.0, opt.refine_start) test_dataloder = torch.utils.data.DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=opt.workers) opt.sym_list = dataset.get_sym_list() # 设定对称物体列表 opt.num_points_mesh = dataset.get_num_points_mesh() # 设定点云数目 print('----------Dataset loaded!---------\nlength of the training set: {0}\nlength of the testing set: {1}\nnumber ' 'of sample points on mesh: {2}\nsymmetry object list: {3}'.format(len(dataset), len(test_dataset), opt.num_points_mesh, opt.sym_list)) criterion = Loss(opt.num_points_mesh, opt.sym_list) # loss计算 criterion_refine = Loss_refine(opt.num_points_mesh, opt.sym_list) # refine_loss计算 best_test = np.Inf # 初始位置最好的模型,loss无限大 if opt.start_epoch == 1: # 开始训练,则删除之前的日志 for log in os.listdir(opt.log_dir): os.remove(os.path.join(opt.log_dir, log)) st_time = time.time() # 记录开始时间 # 开始循环迭代,训练模型-----------------------------------! for epoch in range(opt.start_epoch, opt.nepoch): # 保存开始迭代的log信息 logger = setup_logger('epoch%d' % epoch, os.path.join(opt.log_dir, 'epoch_%d_log.txt' % epoch)) # 记录每个epoch时间 logger.info('Train time {0}'.format(time.strftime('%Hh %Mm %Ss', time.gmtime(time.time() - st_time)) + 'Training started')) train_count = 0 train_dis_avg = 0.0 # 用于计算平均距离 if opt.refine_start: # 如果refine模型,已经开始训练 estimator.eval() refiner.train() else: estimator.train() optimizer.zero_grad() # 优化器清零梯度 for rep in range(opt.repeat_epoch): # 每次epoch重复训练次数 for i, data in enumerate(dataloader, 0): points, choose, img, target, model_points, idx = data # 读取数据 ''' points: 由深度图计算出来的点云,该点云数据以摄像头为参考坐标 choose: 所选择点云的索引[bs, 1, 500] img: 通过box剪切下来的RGB图像 target: 根据model_points点云信息,以及旋转偏移矩阵转换过的点云信息[bs, 500, 3] model_points: 目标初始帧对应的点云信息 idx: 训练图片的下标 ''' # 将数据放到device上 points, choose, img, target, model_points, idx = points.to(device), choose.to(device), img.to(device), target.to(device), model_points.to(device), idx.to(device) # 进行预测获得预测的姿态,和特征向量 pred_r, pred_t, pred_c, emb = estimator(img, points, choose, idx) ''' pred_r: 旋转矩阵[bs, 500, 4] pred_t: 偏移矩阵[bs, 500, 3] pred_c: 置信度[bs, 500, 1] ''' # 计算loss loss, dis, new_points, new_target = criterion(pred_r, pred_t, pred_c, target, model_points, idx, points, opt.w, opt.refine_start) # 如果已经开始了refiner模型的训练 if opt.refine_start: for iter in range(0, opt.iteration): pred_r, pred_t = refiner(new_points, emb, idx) # 进行refiner预测 # 计算loss得到dis dis, new_points, new_target = criterion_refine(pred_r, pred_t, new_target, model_points, idx, new_points) dis.backward() # dis进行反向传播 else: loss.backward() # 否则,则对loss进行反向传播 train_dis_avg += dis.item() # 用于计算平均距离 train_count += 1 # log信息存储 if train_count % opt.batch_size == 0: logger.info('Train time {0} Epoch {1} Batch {2} Frame {3} Avg_dis:{4}' .format(time.strftime('%Hh %Mm %Ss', time.gmtime(time.time() - st_time)), epoch, int(train_count / opt.batch_size), train_count, train_dis_avg / opt.batch_size)) optimizer.step() # 优化器更新 optimizer.zero_grad() # 优化器梯度清零 train_dis_avg = 0.0 if train_count != 0 and train_count % 1000 == 0: # 模型保存 if opt.refine_start: # 已经开始refine模型 torch.save(refiner.state_dict(), '{0}/pose_refine_model_current.pth'.format(opt.outf)) else: torch.save(estimator.state_dict(), '{0}.pos_model_current.pth'.format(opt.outf)) print('------------ epoch {0} train finish -----------'.format(epoch)) # 进行测试 logger = setup_logger('epoch%d test' % epoch, os.path.join(opt.log_dir, 'epoch_%d_test_log.txt' % epoch)) # 记录测试时间 logger.info('Test time {0}'.format(time.strftime('%Hh %Mm %Ss', time.gmtime(time.time() - st_time)) + ',' + 'Testing started')) test_dis = 0.0 test_count = 0 estimator.eval() # 验证模型构建 refiner.eval() # refiner模型 for j, data in enumerate(test_dataloder, 0): points, choose, img, target, model_points, idx = data # 读取数据 ''' points: 由深度图计算出来的点云,该点云数据以摄像头为参考坐标 choose: 所选择点云的索引[bs, 1, 500] img: 通过box剪切下来的RGB图像 target: 根据model_points点云信息,以及旋转偏移矩阵转换过的点云信息[bs, 500, 3] model_points: 目标初始帧对应的点云信息 idx: 训练图片的下标 ''' # 将数据放到device上 points, choose, img, target, model_points, idx = points.to(device), choose.to(device), img.to( device), target.to(device), model_points.to(device), idx.to(device) # 进行预测获得预测的姿态,和特征向量 pred_r, pred_t, pred_c, emb = estimator(img, points, choose, idx) ''' pred_r: 旋转矩阵[bs, 500, 4] pred_t: 偏移矩阵[bs, 500, 3] pred_c: 置信度[bs, 500, 1] ''' # 对结果进行评估 _, dis, new_points, new_target = criterion(pred_r, pred_t, pred_c, target, model_points, idx, points, opt.w, opt.refine_start) # 如果refine模型开始训练,则同样进行评估 if opt.refine_start: for iter in range(0, opt.iteration): pred_r, pred_t = refiner(new_points, emb, idx) dis, new_points, new_target = criterion_refine(pred_r, pred_t, new_target, model_points, idx, new_points) test_dis += dis.item() # 用于计算平均距离 # 保存eval的log logger.info('Test time {0} Test Frame No.{1} dis:{2}'.format(time.strftime('%Hh %Mm %Ss', time.gmtime(time.time() - st_time)), test_count, dis)) test_count += 1 test_dis = test_dis / test_count # 计算平均距离 logger.info('Test time {0} Epoch {1} Test finish avg_dis:{2}'.format(time.strftime('%Hh %Mm %Ss', time.gmtime(time.time() - st_time)), epoch, test_dis)) if test_dis <= best_test: # 如果此次测试结果最好,则保留当前测试结果 best_test = test_dis if opt.refine_start: # 保存refiner torch.save(refiner.state_dict(), '{0}/pose_refine_model_{1}_{2}.pth'.format(opt.outf, epoch, test_dis)) else: torch.save(estimator.state_dict(), '{0}/pose_model_{1}_{2}.pth'.format(opt.outf, epoch, test_dis)) print('----------------test model save finished-------------------') # 参数变化 # 判断模型是否达到衰减要求 if best_test < opt.decay_margin and not opt.decay_start: opt.decay_start = True opt.lr *= opt.lr_rate # 学习率衰减 opt.w *= opt.w_rate # 权重衰减 optimizer = optim.Adam(estimator.parameters(), lr=opt.lr) # 模型没有达到loss阈值要求,refine_start = False,则修改相关参数,传递相关数,更新dataset和dataloader if best_test < opt.refine_margin and not opt.refine_start: opt.refine_start = True opt.batch_size = int(opt.batch_size / opt.iteration) optimizer = optim.Adam(refiner.parameters(), lr=opt.lr) # 训练 dataset = PoseDataset('train', opt.num_points, True, opt.dataset_root, opt.noise_trans, opt.refine_start) dataloader = DataLoader(dataset, batch_size=1, shuffle=True, num_workers=opt.workers) # 测试 test_dataset = PoseDataset('test', opt.num_points, False, opt.dataset_root, 0.0, opt.refine_start) test_dataloder = DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=opt.workers) opt.sym_list = dataset.get_sym_list() opt.num_points_mesh = dataset.get_num_points_mesh() print('----------Dataset loaded!---------\nlength of the training set: {0}\nlength of the testing set: {1}\nnumber ' 'of sample points on mesh: {2}\nsymmetry object list: {3}'.format(len(dataset), len(test_dataset), opt.num_points_mesh, opt.sym_list)) criterion = Loss(opt.num_points_mesh, opt.sym_list) criterion_refine = Loss_refine(opt.num_points_mesh, opt.sym_list)
# 使用简单复利模式计算 engine.usageCompounding = False # True时,只针对FINAL_MODE有效 # 启用实时计算净值模式REALTIME_MODE / FINAL_MODE 回测结束时统一计算模式 engine.calculateMode = engine.REALTIME_MODE engine.initCapital = 100000 # 设置期初资金 engine.percentLimit = 30 # 设置资金使用上限比例(%) engine.barTimeInterval = 300 # bar的周期秒数,用于csv文件自动减时间 # 开始跑回测 engine.runBackTestingWithBarFile(os.getcwd() + '/cache/RB88_20160101_20161231_M5.csv') # 显示回测结果 engine.showBacktestingResult() # 从csv文件进行回测 if __name__ == '__main__': # 提供直接双击回测的功能 # 导入PyQt4的包是为了保证matplotlib使用PyQt4而不是PySide,防止初始化出错 from ctaBacktesting import * from setup_logger import setup_logger setup_logger(filename=u'TestLogs/{0}_{1}.log'.format( Strategy_TripleMa.className, datetime.now().strftime('%m%d_%H%M')), debug=False) # 回测螺纹 testRbByBar()
import random import string from datetime import timedelta, datetime from sqlalchemy import create_engine from sqlalchemy import text from settings import DATABASES, TABLE_COLUMNS, TABLE_NAME, QUERIES, DB_SIZE from setup_logger import setup_logger from timed import Timed logger = setup_logger('performance') def print_range(length): for i in range(length): print('GENERATING: {0} / {1}'.format(i + 1, length), end='\r') yield class Connector: def __init__(self, dbsystem): self.dbsystem = dbsystem.upper() self.QUERIES = QUERIES[self.dbsystem] self.TABLE_COLUMNS = TABLE_COLUMNS[self.dbsystem] self.config = DATABASES[self.dbsystem] self.conn = self.get_engine() def get_engine(self, with_db=True):