Exemple #1
0
def iMain():
    sUsage = __doc__.strip()
    oArgParser = oParseOptions(sUsage)
    oArgParser.add_argument('lArgs', action="store",
                            nargs="*",
                            help="the Symbol Timeframe and Year to backtest (required)")
    oOptions = oArgParser.parse_args()
    lArgs = oOptions.lArgs

    assert len(lArgs) == 4, "Give the CsvFile Symbol Timeframe and Year as arguments"

    # sDir = '/t/Program Files/HotForex MetaTrader/history/tools.fxdd.com'
    # sCsvFile = os.path.join(sDir, sSymbol + sTimeFrame +'-' +sYear +'.csv')
    sCsvFile = lArgs[0]
    sSymbol = lArgs[1] # 'EURGBP'
    sTimeFrame = lArgs[2] # '1440'
    sYear = lArgs[3] # '2014'

    oOhlc = oReadMt4Csv(sCsvFile, sTimeFrame, sSymbol, sYear)
    oOhlc = oPreprocessOhlc(oOhlc)
    # import pandas
    # (Pdb) pandas.tseries.converter._dt_to_float_ordinal(oOhlc.index)[0]
    # 735235.33333333337
    dates = matplotlib.dates.date2num(oOhlc.index.to_pydatetime())
    volume = 1000*np.random.normal(size=len(oOhlc))

    vGraphData(sSymbol, dates,
               oOhlc.C.values, oOhlc.H.values, oOhlc.L.values, oOhlc.O.values,
               volume,
               oOptions.iShortSMA, oOptions.iLongSMA,
               oOptions.iRsiUpper, oOptions.iRsiLower,
               oOptions.iMacdSlow, oOptions.iMacdFast, oOptions.iMacdEma,
               bUseTalib=oOptions.bUseTalib,
               )
Exemple #2
0
def iMain():
    sUsage = __doc__.strip()
    oArgParser = oParseOptions(sUsage)
    oArgParser.add_argument('lArgs', action="store",
                            nargs="*",
                            help="the Symbol Timeframe and Year to backtest (required)")
    oOptions = oArgParser.parse_args()
    lArgs = oOptions.lArgs

    assert len(lArgs) == 4, "Give the CsvFile Symbol Timeframe and Year as arguments"

    # sDir = '/t/Program Files/HotForex MetaTrader/history/tools.fxdd.com'
    # sCsvFile = os.path.join(sDir, sSymbol + sTimeFrame +'-' +sYear +'.csv')
    sCsvFile = lArgs[0]
    sSymbol = lArgs[1] # 'EURGBP'
    sTimeFrame = lArgs[2] # '1440'
    sYear = lArgs[3] # '2014'

    oOhlc = oReadMt4Csv(sCsvFile, sTimeFrame, sSymbol, sYear)
    oOhlc = oPreprocessOhlc(oOhlc)
    # import pandas
    # (Pdb) pandas.tseries.converter._dt_to_float_ordinal(oOhlc.index)[0]
    # 735235.33333333337
    dates = matplotlib.dates.date2num(oOhlc.index.to_pydatetime())
    volume = 1000*np.random.normal(size=len(oOhlc))

    vGraphData(sSymbol, dates,
               oOhlc.C.values, oOhlc.H.values, oOhlc.L.values, oOhlc.O.values,
               volume,
               oOptions.iShortSMA, oOptions.iLongSMA,
               oOptions.iRsiUpper, oOptions.iRsiLower,
               oOptions.iMacdSlow, oOptions.iMacdFast, oOptions.iMacdEma,
               bUseTalib=oOptions.bUseTalib,
               )
Exemple #3
0
def vDoBacktestCmd(self, oArgs, oOpts=None):
    __doc__ = sBAC__doc__
    global dFEED_CACHE
    global sFEED_CACHE_KEY

    lArgs = oArgs.split()
    sDo = lArgs[0]

    # An omlette is an HDF5 file that saves all the data from a backtest
    if sDo == 'omlette':
        # is this a backtest command or should it move up
        # leave it here for now, and move it up later
        __doc__ = sBAComlette__doc__

        global _sCurrentOmletteDir
        # ingredients recipe dish
        # plot sSection
        #
        _lCmds = ['load', 'open', 'check', 'save', 'close', 'display']
        assert len(lArgs) > 1, "ERROR: " + sDo + " " + str(_lCmds)
        sCmd = lArgs[1]
        assert sCmd in _lCmds, "ERROR: " + sDo + " " + str(_lCmds)

        oOm = oEnsureOmlette(self, oOpts, sNewOmlette="")
        if sCmd == 'check':
            assert hasattr(oOm, 'oHdfStore')
            assert oOm.oHdfStore is not None
            # FixMe: something better than filename
            self.poutput(sDo + " filename: " + oOm.oHdfStore.filename)
            return

        if sCmd == 'display':
            # display gives a complete listing of the contents of the HDF file
            assert hasattr(oOm, 'oHdfStore') and oOm.oHdfStore
            self.poutput(repr(oOm.oHdfStore))
            return

        ## if sCmd == 'save':
        ##     return

        if sCmd == 'close':
            assert oOm.oHdfStore is not None, \
                   "ERROR: " +sDo +" " +sCmd +"; not open: use '" +sDo +" open FILE'"
            self.oOm.vClose()
            return

        assert len(lArgs) >= 3, \
               "ERROR: " +sDo +" " +sCmd +" " +" FILENAME"
        sFile = lArgs[2]

        if sCmd == 'open':
            o = oOm.oAddHdfStore(sFile)
            assert o is not None and oOm.oHdfStore is not None
            _sCurrentOmletteDir = os.path.dirname(sFile)
            return

        if sCmd == 'load':
            assert os.path.isfile(sFile), \
                   "ERROR: " +sDo +" " +sCmd +" file not found " +sFile
            _sCurrentOmletteDir = os.path.dirname(sFile)
            raise RuntimeError(NotImplemented)
            return

        self.vError("Unrecognized omlette command: " + str(oArgs) + '\n' +
                    __doc__)
        return

    # Create feeds (pandas DataFrames) from CSV OHLCV files
    if sDo == 'feed':
        __doc__ = sBACfeed__doc__
        #? rename delete
        _lCmds = [
            'dir', 'list', 'get', 'set', 'read_mt4_csv', 'read_yahoo_csv',
            'info', 'plot', 'to_hdf'
        ]
        assert len(
            lArgs) > 1, "ERROR: " + sDo + " command required: " + str(_lCmds)
        sCmd = lArgs[1]
        assert lArgs[1] in _lCmds, "ERROR: " + sDo + " " + str(_lCmds)

        if sCmd == 'dir':
            if len(lArgs) == 2:
                if not self.oConfig['feed']['sHistoryDir']:
                    self.poutput(
                        "No default history directory set: use \"feed dir dir\""
                    )
                elif not os.path.isdir(self.oConfig['feed']['sHistoryDir']):
                    self.poutput("History directory not found: use \"feed dir dir\": " + \
                                 self.oConfig['feed']['sHistoryDir'])
                else:
                    self.poutput("Default history directory: " + \
                                 self.oConfig['feed']['sHistoryDir'])
                return

            sDir = lArgs[2]
            assert os.path.isdir(sDir)
            self.oConfig['feed']['sHistoryDir'] = sDir
            return

        if sCmd == 'read_mt4_csv':
            from PandasMt4 import oReadMt4Csv
            assert len(lArgs) >= 3, \
                   "ERROR: " +sDo +" " +sCmd +" FILENAME [SYMBOL TIMEFRAME YEAR]"
            sFile = lArgs[2]
            if not os.path.isfile(sFile):
                sHistoryDir = os.path.join(self.oOptions.sMt4Dir, 'history')
                assert os.path.isdir(sHistoryDir)
                import glob
                l = glob.glob(sHistoryDir + "/*/" + sFile)
                if l:
                    sFile = l[0]
                    self.vInfo("Found history file: " + sFile)
            assert os.path.isfile(sFile), \
                   "ERROR: " +sDo +" " +sCmd +" file not found " +sFile
            if len(lArgs) > 5:
                sSymbol = lArgs[3]
                sTimeFrame = lArgs[4]
                sYear = lArgs[5]
            else:
                lCooked = os.path.split(sFile)[-1].split('.')[0].split('-')
                if len(lCooked) > 1:
                    sYear = lCooked[-1]
                else:
                    sYear = ""
                sSymbol = lCooked[0][:6]
                sTimeFrame = lCooked[0][6:]

            self.vDebug(sDo +" " +sCmd +" " + \
                       "sSymbol=" +sSymbol \
                       +", sYear=" +sYear \
                       +", sTimeFrame=" +sTimeFrame)

            mFeedOhlc = oReadMt4Csv(sFile, sTimeFrame, sSymbol, sYear="")
            assert mFeedOhlc is not None, "oReadMt4Csv failed on " + sFile
            mFeedOhlc.info(True, sys.stdout)

            # NaturalNameWarning: object name is not a valid Python identifier: 'Mt4_csv|EURUSD|1440|2014'; it does not match the pattern ``^[a-zA-Z_][a-zA-Z0-9_]*$``;
            sKey = 'Mt4_csv' + '_' + sSymbol + '_' + sTimeFrame + '_' + sYear
            oOm = oEnsureOmlette(self, oOpts)
            _dCurrentFeedFrame = oOm.dGetFeedFrame(sFile, sTimeFrame, sSymbol,
                                                   sYear)
            from PandasMt4 import oReadMt4Csv, oPreprocessOhlc, vResampleFiles
            mFeedOhlc = oPreprocessOhlc(_dCurrentFeedFrame['mFeedOhlc'])
            sys.stdout.write('INFO:  Data Open length: %d\n' % len(mFeedOhlc))
            _dCurrentFeedFrame['mFeedOhlc'] = mFeedOhlc

            dFEED_CACHE[sKey] = _dCurrentFeedFrame
            sFEED_CACHE_KEY = sKey
            return

        _lFeedCacheKeys = dFEED_CACHE.keys()
        if sCmd == 'list':
            self.poutput("Feed keys: %r" % (self.G(_lFeedCacheKeys, )))
            return

        if sCmd == 'get':
            self.poutput("Current Feed key: %s" % (self.G(sFEED_CACHE_KEY, )))
            return

        if sCmd == 'set':
            assert len(lArgs) >= 3, \
                   "ERROR: " +sDo +" " +sCmd +" " + '|'.join(_lFeedCacheKeys)
            sKey = lArgs[2]
            assert sKey in _lFeedCacheKeys, \
                   "ERROR: " +sDo +" " +sCmd +" " + '|'.join(_lFeedCacheKeys)
            sFEED_CACHE_KEY = sKey
            return

        # The following all require that a feed has been loaded
        if not sFEED_CACHE_KEY or sFEED_CACHE_KEY not in dFEED_CACHE:
            self.vError("Run \"back read_*\" first to read a DataFrame")
            return
        _dCurrentFeedFrame = dFEED_CACHE[sFEED_CACHE_KEY]

        if _dCurrentFeedFrame is None:
            self.vError("Run \"back read_*\" first to read a DataFrame")
            return
        sSymbol = _dCurrentFeedFrame['sSymbol']
        sKey = _dCurrentFeedFrame['sKey']
        sTimeFrame = _dCurrentFeedFrame['sTimeFrame']
        mFeedOhlc = _dCurrentFeedFrame['mFeedOhlc']

        if sCmd in ['to_hdf']:
            """ DataFrame.to_hdf(path_or_buf, key, **kwargs)
            activate the HDFStore
            Parameters :
              path_or_buf : the path (string) or buffer to put the store
              key : string indentifier for the group in the store
              """

            assert len(lArgs) >= 3, \
                   "ERROR: " +sDo +" " +sCmd +" fixed|table filename"
            sType = lArgs[2]
            assert sType in ['fixed', 'table']
            sFile = lArgs[3]
            # FixME: if absolute assert os.path.exists(os.path.dirname(sFile))
            #? lArgs[4:] -> **kw ?
            vRetval = mFeedOhlc.to_hdf(sFile, sKey, format=sType)
            return

        _dPlotParams = self.oConfig['feed.plot.params']

        if sCmd == 'info':
            """Concise summary of a DataFrame.
            Parameters :
            verbose : boolean, default True
                    If False, don’t print column count summary
            buf : writable buffer, defaults to sys.stdout
            """
            import yaml

            mFeedOhlc.info(True, sys.stdout)

            s = '|  %s  |' % ("Plot Params", )
            self.poutput('-' * len(s))
            self.poutput(s)
            self.poutput('-' * len(s))
            yaml.dump(_dPlotParams,
                      allow_unicode=True,
                      default_flow_style=False)
            self.poutput('-' * len(s))
            return

        # 'download_hst_zip'
        if sCmd == 'plot':
            import matplotlib
            if 'matplotlib.rcParams' in self.oConfig:
                for sKey, gVal in self.oConfig['matplotlib.rcParams'].items():
                    matplotlib.rcParams[sKey] = gVal
            import numpy as np
            from OTPpnAmgc import vGraphData

            from OTBackTest import oPreprocessOhlc
            mFeedOhlc = oPreprocessOhlc(mFeedOhlc)
            # (Pdb) pandas.tseries.converter._dt_to_float_ordinal(mFeedOhlc.index)[0]
            # 735235.33333333337
            nDates = matplotlib.dates.date2num(mFeedOhlc.index.to_pydatetime())
            nVolume = 1000 * np.random.normal(size=len(mFeedOhlc))

            self.vInfo(
                "This may take minutes to display depending on your computer's speed"
            )
            vGraphData(sSymbol, nDates, mFeedOhlc.C.values, mFeedOhlc.H.values,
                       mFeedOhlc.L.values, mFeedOhlc.O.values, nVolume,
                       **_dPlotParams)
            return

        self.vError("Unrecognized feed command: " + str(oArgs) + '\n' +
                    __doc__)
        return

    # Set the recipe that the chef will use, and make the ingredients from the feeds
    if sDo == 'recipe':
        __doc__ = sBACrecipe__doc__
        from .Omlettes import lKnownRecipes
        # self.vDebug("lKnownRecipes: " + repr(lKnownRecipes))
        # are ingredients under chef?
        _lCmds = ['set', 'list', 'get', 'make', 'ingredients', 'config']

        sCmd = str(lArgs[1])
        if sCmd == 'list':
            self.poutput("Known Recipes: %r" % (self.G(lKnownRecipes, )))
            return

        if sCmd == 'get' or (sCmd == 'set' and len(lArgs) == 2):
            self.poutput("Current Recipe: %s" % (self.sRecipe, ))
            return

        assert len(lArgs) > 1, "ERROR: not enough args: " + sDo + str(_lCmds)
        assert sCmd in _lCmds, "ERROR: %s %s not in: %r " % (sDo, sCmd, _lCmds)

        if sCmd == 'config':
            oRecipe = oEnsureRecipe(self, oOpts)
            self.vConfigOp(lArgs, oRecipe.oConfig)
            return

        if sCmd == 'set':
            assert len(lArgs) > 2, \
                   "ERROR: %s %s requires one of: %s" % (
                sDo, sCmd, '|'.join(lKnownRecipes))
            sNewRecipe = str(lArgs[2])
            assert sNewRecipe in lKnownRecipes, \
                   "ERROR: %s %s %s not in: %s" % (
                sDo, sCmd, sNewRecipe, '|'.join(lKnownRecipes))
            if self.sRecipe != sNewRecipe:
                self.sRecipe = sNewRecipe
                oRecipe = oEnsureRecipe(self, oOpts, sNewRecipe=sNewRecipe)
            #? do we update the config file? - I think not
            #? self.oConfig['backtest']['sRecipe'] = sNewRecipe
            return

        # The following all require that a feed has been loaded
        assert sFEED_CACHE_KEY
        assert sFEED_CACHE_KEY in dFEED_CACHE
        _dCurrentFeedFrame = dFEED_CACHE[sFEED_CACHE_KEY]
        assert _dCurrentFeedFrame
        if sCmd == 'make' or sCmd == 'ingredients':
            assert _dCurrentFeedFrame
            oRecipe = oEnsureRecipe(self, oOpts)
            # ugly
            dFeedParams = _dCurrentFeedFrame
            mFeedOhlc = _dCurrentFeedFrame['mFeedOhlc']
            dFeeds = dict(mFeedOhlc=mFeedOhlc, dFeedParams=dFeedParams)

            oRecipe.dMakeIngredients(dFeeds)
            assert oRecipe.dIngredients
            return

        self.vError("Unrecognized recipe command: " + str(oArgs) + '\n' +
                    __doc__)
        return

    # Set the chef that we will use, and cook from the ingredients and the feeds
    if sDo == 'chef':
        __doc__ = sBACchef__doc__
        from .Omlettes import lKnownChefs
        # self.vDebug("lKnownChefs: " + repr(lKnownChefs))

        _lCmds = ['get', 'set', 'list', 'cook']
        assert len(lArgs) > 1, "ERROR: not enough args: " + sDo + str(_lCmds)
        sCmd = lArgs[1]

        if sCmd == 'list':
            self.poutput("Known Chefs: %r" % (lKnownChefs, ))
            return

        if sCmd == 'get' or (sCmd == 'set' and len(lArgs) == 2):
            self.poutput("Current Chef: %s" % (self.sChef, ))
            return

        assert sCmd in _lCmds, "ERROR: not in: " + sDo + str(_lCmds)

        if sCmd == 'set':
            assert len(lArgs) > 2, \
                   "ERROR: %s %s needs one of: %s" % (
                sDo, sCmd, '|'.join(lKnownChefs))
            sNewChef = str(lArgs[2])
            assert sNewChef in lKnownChefs, \
                   "ERROR: %s %s %s not in: %s" % (
                sDo, sCmd, sNewChef, '|'.join(lKnownChefs))
            if self.sChef != sNewChef:
                self.sChef = sNewChef
                oChef = oEnsureChef(self, oOpts, sNewChef=sNewChef)
            #? do we update the config file? - I think not
            #? self.oConfig['backtest']['sChef'] = sNewChef
            return

        # The following all require that a feed has been loaded
        assert sFEED_CACHE_KEY
        assert sFEED_CACHE_KEY in dFEED_CACHE
        _dCurrentFeedFrame = dFEED_CACHE[sFEED_CACHE_KEY]
        assert _dCurrentFeedFrame

        # There's always a default provided of these
        oOm = oEnsureOmlette(self, oOpts)
        oRecipe = oEnsureRecipe(self, oOpts)
        oChefModule = oEnsureChef(self, oOpts)

        if sCmd == 'cook':
            from .OTBackTest import oPyBacktestCook
            assert oRecipe.dIngredients
            # ugly
            dFeeds = _dCurrentFeedFrame

            oBt = oPyBacktestCook(dFeeds, oRecipe, oChefModule, oOm)
            assert oBt is not None
            if type(oBt) == str:
                raise RuntimeError(oBt)
            oOm.oBt = oBt
            # self.vDebug("Cooked " + oBt.sSummary())
            return

        self.vError("Unrecognized chef command: " + str(oArgs) + '\n' +
                    __doc__)
        return

    oOm = oEnsureOmlette(self, oOpts)
    oRecipe = oEnsureRecipe(self, oOpts)
    oChefModule = oEnsureChef(self, oOpts)

    # List the servings the chef has cooked, and dish out the servings
    if sDo == 'servings':
        __doc__ = sBACservings__doc__
        if not hasattr(oOm, 'oBt') or not oOm.oBt:
            self.vError(
                "You must use \"chef cook\" before you can have servings")
            return
        oBt = oOm.oBt

        # ['signals', 'trades', 'positions', 'equity', 'reviews', 'trade_price']
        _lCmds = oChefModule.lProducedServings[:]
        if tabview and tabview not in _lCmds: _lCmds += ['tabview']

        if len(lArgs) == 1 or lArgs[1] == 'list':
            self.poutput("Produced Servings: %r" %
                         (oChefModule.lProducedServings, ))
            return

        assert len(lArgs) > 1, "ERROR: argument required" + sDo + str(_lCmds)
        sCmd = lArgs[1]
        # self.vDebug("lProducedServings: " + repr(_lCmds))
        assert sCmd in _lCmds, "ERROR: %s %s not in %r" % (sDo, sCmd, _lCmds)

        oFd = sys.stdout
        ## oFun = getattr(self.oBt, sCmd)
        ## self.poutput(oFun())
        if sCmd == 'signals':
            # this was the same as: oBt._mSignals = bt.mSignals() or oBt.signals
            oBt._mSignals = oRecipe.mSignals(oBt)
            oFd.write('INFO:  bt.signals found: %d\n' % len(oBt.signals))
            oOm.vAppendHdf('recipe/servings/mSignals', oBt.signals)
            return

        if sCmd == 'trades':
            # this was the same as: oBt._mTrades =  oBt.mTrades() or oBt.trades
            oBt._mTrades = oRecipe.mTrades(oBt)
            oFd.write('INFO:  bt.trades found: %d\n' % len(oBt.trades))
            oOm.vAppendHdf('recipe/servings/mTrades', oBt.trades)
            return

        if sCmd == 'positions':
            # this was the same as: oBt._rPositions = oBt.rPositions() or oBt.positions
            oBt._rPositions = oRecipe.rPositions(oBt, init_pos=0)
            oFd.write('INFO:  bt.positions found: %d\n' % len(oBt.positions))
            oOm.vAppendHdf('recipe/servings/rPositions', oBt.positions)
            return

        if sCmd == 'equity':
            # this was the same as: oBt._rEquity = oBt.rEquity() or oBt.equity
            oBt._rEquity = oRecipe.rEquity(oBt)
            oFd.write('INFO:  bt.equity found: %d\n' % len(oBt.equity))
            oOm.vAppendHdf('recipe/servings/rEquity', oBt.equity)
            return

        if sCmd == 'trade_price':
            # oFd.write('INFO:  bt.rTradePrice() found: %d\n' % len(oBt.rTradePrice()))
            oFd.write('INFO:  bt.trade_price found: %d\n' %
                      len(oBt.trade_price))
            oOm.vAppendHdf('recipe/servings/rTradePrice', oBt.trade_price)
            return

        if sCmd == 'metrics' or sCmd == 'reviews':
            oOm.vSetTitleHdf('recipe/servings', oOm.oChefModule.sChef)
            #? Leave this as derived or store it? reviews?
            oOm.vSetMetadataHdf('recipe/servings', oBt.dSummary())
            oFd.write(oBt.sSummary())
            return

        if tabview and sCmd == 'tabview':
            assert len(lArgs) > 2, "ERROR: " +sDo +" " +sCmd \
                   +": serving required, one of: reviews " +str(oChefModule.lProducedServings)

            sServing = lArgs[2]
            if sServing in ['metrics', 'reviews']:
                l = [['Metric', 'Value']]
                l += oBt.lSummary()
                l.sort()
                # , hdr_rows=lHdrRows
                tabview.view(l, column_width='max')
                return

            assert sServing in oChefModule.lProducedServings
            mDf = getattr(oBt, sServing)
            # FixMe: need index timestamp for mva
            tabview.view(mDf)
            return

        self.vError("Unrecognized servings command: " + str(oArgs) + '\n' +
                    __doc__)
        return

    # Plot the servings the chef has cooked, using matplotlib
    if sDo == 'plot':
        __doc__ = sBACplot__doc__
        _lCmds = ['show', 'trades', 'equity']
        if not hasattr(oOm, 'oBt') or not oOm.oBt:
            self.vError("You must use \"chef cook\" before you can plot")
            return
        oBt = oOm.oBt

        assert len(lArgs) > 1, "ERROR: " + sDo + str(_lCmds)
        sCmd = lArgs[1]
        assert sCmd in _lCmds, "ERROR: " + sDo + str(_lCmds)

        import matplotlib
        import matplotlib.pylab as pylab
        if sCmd == 'show':
            pylab.show()
            return

        # FixMe:
        matplotlib.rcParams['figure.figsize'] = (10, 5)

        if sCmd == 'equity':
            # FixMe: derive the period from the sTimeFrame
            sPeriod = 'W'
            close_label = 'C'
            mOhlc = oRecipe.dIngredients['mOhlc']
            oChefModule.vPlotEquity(
                oBt.equity,
                mOhlc,
                sTitle="%s\nEquity" % repr(oBt),
                sPeriod=sPeriod,
                close_label=close_label,
            )
            pylab.show()
            return

        if sCmd == 'trades':
            oBt.vPlotTrades()
            pylab.legend(loc='lower left')
            pylab.show()

            return
        self.vError("Unrecognized plot command: " + str(oArgs) + "\n" +
                    sBAC__doc__)
        return

    self.vError("Unrecognized backtest command: " + str(oArgs) + "\n" +
                sBAC__doc__)
def vDoBacktestCmd(self, oArgs, oOpts=None):
    __doc__ = sBAC__doc__
    global dFEED_CACHE
    global sFEED_CACHE_KEY

    lArgs = oArgs.split()
    sDo = lArgs[0]

    # An omlette is an HDF5 file that saves all the data from a backtest
    if sDo == 'omlette':
        # is this a backtest command or should it move up
        # leave it here for now, and move it up later
        __doc__ = sBAComlette__doc__

        global _sCurrentOmletteDir
        # ingredients recipe dish
        # plot sSection
        #
        _lCmds = ['load', 'open', 'check', 'save', 'close', 'display']
        assert len(lArgs) > 1, "ERROR: " +sDo +" " +str(_lCmds)
        sCmd = lArgs[1]
        assert sCmd in _lCmds, "ERROR: " +sDo +" " +str(_lCmds)

        oOm = oEnsureOmlette(self, oOpts, sNewOmlette="")
        if sCmd == 'check':
            assert hasattr(oOm, 'oHdfStore')
            assert oOm.oHdfStore is not None
            # FixMe: something better than filename
            self.poutput(sDo +" filename: " +oOm.oHdfStore.filename)
            return

        if sCmd == 'display':
            # display gives a complete listing of the contents of the HDF file
            assert hasattr(oOm, 'oHdfStore') and oOm.oHdfStore
            self.poutput(repr(oOm.oHdfStore))
            return

        ## if sCmd == 'save':
        ##     return

        if sCmd == 'close':
            assert oOm.oHdfStore is not None, \
                   "ERROR: " +sDo +" " +sCmd +"; not open: use '" +sDo +" open FILE'"
            self.oOm.vClose()
            return

        assert len(lArgs) >= 3, \
               "ERROR: " +sDo +" " +sCmd +" " +" FILENAME"
        sFile = lArgs[2]

        if sCmd == 'open':
            o = oOm.oAddHdfStore(sFile)
            assert o is not None and oOm.oHdfStore is not None
            _sCurrentOmletteDir = os.path.dirname(sFile)
            return

        if sCmd == 'load':
            assert os.path.isfile(sFile), \
                   "ERROR: " +sDo +" " +sCmd +" file not found " +sFile
            _sCurrentOmletteDir = os.path.dirname(sFile)
            raise RuntimeError(NotImplemented)
            return

        self.vError("Unrecognized omlette command: " + str(oArgs) +'\n' +__doc__)
        return

    # Create feeds (pandas DataFrames) from CSV OHLCV files
    if sDo == 'feed':
        __doc__ = sBACfeed__doc__
        #? rename delete
        _lCmds = ['dir', 'list', 'get', 'set',
                  'read_mt4_csv', 'read_yahoo_csv', 'info', 'plot', 'to_hdf']
        assert len(lArgs) > 1, "ERROR: " +sDo +" " +str(_lCmds)
        sCmd = lArgs[1]
        assert lArgs[1] in _lCmds, "ERROR: " +sDo +" " +str(_lCmds)

        if sCmd == 'dir':
            if len(lArgs) == 2:
                if not self.oConfig['feed']['sHistoryDir']:
                    self.poutput("No default history directory set: use \"feed dir dir\"")
                elif not os.path.isdir(self.oConfig['feed']['sHistoryDir']):
                    self.poutput("History directory not found: use \"feed dir dir\": " + \
                                 self.oConfig['feed']['sHistoryDir'])
                else:
                    self.poutput("Default history directory: " + \
                                 self.oConfig['feed']['sHistoryDir'])
                return

            sDir = lArgs[2]
            assert os.path.isdir(sDir)
            self.oConfig['feed']['sHistoryDir'] = sDir
            return

        if sCmd == 'read_mt4_csv':
            from PandasMt4 import oReadMt4Csv
            assert len(lArgs) >= 3, \
                   "ERROR: " +sDo +" " +sCmd +" FILENAME [SYMBOL TIMEFRAME YEAR]"
            sFile = lArgs[2]
            if not os.path.isfile(sFile):
                sHistoryDir = os.path.join(self.oOptions.sMt4Dir, 'history')
                assert os.path.isdir(sHistoryDir)
                import glob
                l = glob.glob(sHistoryDir +"/*/" +sFile)
                if l:
                    sFile = l[0]
                    self.vInfo("Found history file: " + sFile)
            assert os.path.isfile(sFile), \
                   "ERROR: " +sDo +" " +sCmd +" file not found " +sFile
            if len(lArgs) > 5:
                sSymbol = lArgs[3]
                sTimeFrame = lArgs[4]
                sYear = lArgs[5]
            else:
                lCooked = os.path.split(sFile)[-1].split('.')[0].split('-')
                if len(lCooked) > 1:
                    sYear = lCooked[-1]
                else:
                    sYear = ""
                sSymbol = lCooked[0][:6]
                sTimeFrame = lCooked[0][6:]

            self.vDebug(sDo +" " +sCmd +" " + \
                       "sSymbol=" +sSymbol \
                       +", sYear=" +sYear \
                       +", sTimeFrame=" +sTimeFrame)

            mFeedOhlc = oReadMt4Csv(sFile, sTimeFrame, sSymbol, sYear="")
            assert mFeedOhlc is not None, "oReadMt4Csv failed on " + sFile
            mFeedOhlc.info(True, sys.stdout)

            # NaturalNameWarning: object name is not a valid Python identifier: 'Mt4_csv|EURUSD|1440|2014'; it does not match the pattern ``^[a-zA-Z_][a-zA-Z0-9_]*$``;
            sKey = 'Mt4_csv' +'_' +sSymbol +'_' +sTimeFrame +'_' +sYear
            oOm = oEnsureOmlette(self, oOpts)
            _dCurrentFeedFrame = oOm.dGetFeedFrame(sFile,
                                                   sTimeFrame,
                                                   sSymbol,
                                                   sYear)
            from PandasMt4 import oReadMt4Csv, oPreprocessOhlc, vResampleFiles
            mFeedOhlc = oPreprocessOhlc(_dCurrentFeedFrame['mFeedOhlc'])
            sys.stdout.write('INFO:  Data Open length: %d\n' % len(mFeedOhlc))
            _dCurrentFeedFrame['mFeedOhlc'] = mFeedOhlc

            dFEED_CACHE[sKey] = _dCurrentFeedFrame
            sFEED_CACHE_KEY = sKey
            return

        _lFeedCacheKeys = dFEED_CACHE.keys()
        if sCmd == 'list':
            self.poutput("Feed keys: %r" % (self.G(_lFeedCacheKeys,)))
            return

        if sCmd == 'get':
            self.poutput("Current Feed key: %s" % (self.G(sFEED_CACHE_KEY,)))
            return

        if sCmd == 'set':
            assert len(lArgs) >= 3, \
                   "ERROR: " +sDo +" " +sCmd +" " + '|'.join(_lFeedCacheKeys)
            sKey = lArgs[2]
            assert sKey in _lFeedCacheKeys, \
                   "ERROR: " +sDo +" " +sCmd +" " + '|'.join(_lFeedCacheKeys)
            sFEED_CACHE_KEY = sKey
            return

        # The following all require that a feed has been loaded
        _dCurrentFeedFrame = dFEED_CACHE[sFEED_CACHE_KEY]

        if _dCurrentFeedFrame is None:
            self.vError("Run \"back read_*\" first to read a DataFrame")
            return
        sSymbol = _dCurrentFeedFrame['sSymbol']
        sKey = _dCurrentFeedFrame['sKey']
        sTimeFrame = _dCurrentFeedFrame['sTimeFrame']
        mFeedOhlc = _dCurrentFeedFrame['mFeedOhlc']

        if sCmd in ['to_hdf']:
            """ DataFrame.to_hdf(path_or_buf, key, **kwargs)
            activate the HDFStore
            Parameters :
              path_or_buf : the path (string) or buffer to put the store
              key : string indentifier for the group in the store
              """

            assert len(lArgs) >= 3, \
                   "ERROR: " +sDo +" " +sCmd +" fixed|table filename"
            sType = lArgs[2]
            assert sType in ['fixed', 'table']
            sFile = lArgs[3]
            # FixME: if absolute assert os.path.exists(os.path.dirname(sFile))
            #? lArgs[4:] -> **kw ?
            vRetval = mFeedOhlc.to_hdf(sFile, sKey, format=sType)
            return

        _dPlotParams = self.oConfig['feed.plot.params']

        if sCmd == 'info':
            """Concise summary of a DataFrame.
            Parameters :
            verbose : boolean, default True
                    If False, don’t print column count summary
            buf : writable buffer, defaults to sys.stdout
            """
            import yaml

            mFeedOhlc.info(True, sys.stdout)

            s = '|  %s  |' % ("Plot Params",)
            self.poutput('-' * len(s))
            self.poutput(s)
            self.poutput('-' * len(s))
            yaml.dump(_dPlotParams,
                      allow_unicode=True,
                      default_flow_style=False)
            self.poutput('-' * len(s))
            return

        # 'download_hst_zip'
        if sCmd == 'plot':
            import matplotlib
            import numpy as np
            from OTPpnAmgc import vGraphData
            if 'matplotlib.rcParams' in self.oConfig:
                for sKey, gVal in self.oConfig['matplotlib.rcParams'].items():
                    matplotlib.rcParams[sKey] = gVal

            from OTBackTest import oPreprocessOhlc
            mFeedOhlc = oPreprocessOhlc(mFeedOhlc)
            # (Pdb) pandas.tseries.converter._dt_to_float_ordinal(mFeedOhlc.index)[0]
            # 735235.33333333337
            nDates = matplotlib.dates.date2num(mFeedOhlc.index.to_pydatetime())
            nVolume = 1000*np.random.normal(size=len(mFeedOhlc))

            self.vInfo("This may take minutes to display depending on your computer's speed")
            vGraphData(sSymbol, nDates,
                       mFeedOhlc.C.values, mFeedOhlc.H.values, mFeedOhlc.L.values, mFeedOhlc.O.values,
                       nVolume,
                       **_dPlotParams)
            return

        self.vError("Unrecognized feed command: " + str(oArgs) +'\n' +__doc__)
        return

    # Set the recipe that the chef will use, and make the ingredients from the feeds
    if sDo == 'recipe':
        __doc__ = sBACrecipe__doc__
        from .Omlettes import lKnownRecipes
        # self.vDebug("lKnownRecipes: " + repr(lKnownRecipes))
        # are ingredients under chef?
        _lCmds = ['set', 'list', 'get', 'make', 'ingredients', 'config']

        sCmd = str(lArgs[1])
        if sCmd == 'list':
            self.poutput("Known Recipes: %r" % (self.G(lKnownRecipes,)))
            return

        if sCmd == 'get' or (sCmd == 'set' and len(lArgs) == 2):
            self.poutput("Current Recipe: %s" % (self.sRecipe,))
            return

        assert len(lArgs) > 1, "ERROR: not enough args: " +sDo +str(_lCmds)
        assert sCmd in _lCmds, "ERROR: %s %s not in: %r " % (
            sDo, sCmd, _lCmds)


        if sCmd == 'config':
            oRecipe = oEnsureRecipe(self, oOpts)
            self.vConfigOp(lArgs, oRecipe.oConfig)
            return

        if sCmd == 'set':
            assert len(lArgs) > 2, \
                   "ERROR: %s %s requires one of: %s" % (
                sDo, sCmd, '|'.join(lKnownRecipes))
            sNewRecipe = str(lArgs[2])
            assert sNewRecipe in lKnownRecipes, \
                   "ERROR: %s %s %s not in: %s" % (
                sDo, sCmd, sNewRecipe, '|'.join(lKnownRecipes))
            if self.sRecipe != sNewRecipe:
                self.sRecipe = sNewRecipe
                oRecipe = oEnsureRecipe(self, oOpts, sNewRecipe=sNewRecipe)
            #? do we update the config file? - I think not
            #? self.oConfig['backtest']['sRecipe'] = sNewRecipe
            return

        # The following all require that a feed has been loaded
        assert sFEED_CACHE_KEY
        assert sFEED_CACHE_KEY in dFEED_CACHE
        _dCurrentFeedFrame = dFEED_CACHE[sFEED_CACHE_KEY]
        assert _dCurrentFeedFrame
        if sCmd == 'make' or sCmd == 'ingredients':
            assert _dCurrentFeedFrame
            oRecipe = oEnsureRecipe(self, oOpts)
            # ugly
            dFeedParams = _dCurrentFeedFrame
            mFeedOhlc = _dCurrentFeedFrame['mFeedOhlc']
            dFeeds = dict(mFeedOhlc=mFeedOhlc, dFeedParams=dFeedParams)

            oRecipe.dMakeIngredients(dFeeds)
            assert oRecipe.dIngredients
            return

        self.vError("Unrecognized recipe command: " + str(oArgs) +'\n' +__doc__)
        return

    # Set the chef that we will use, and cook from the ingredients and the feeds
    if sDo == 'chef':
        __doc__ = sBACchef__doc__
        from .Omlettes import lKnownChefs
        # self.vDebug("lKnownChefs: " + repr(lKnownChefs))

        _lCmds = ['get', 'set', 'list', 'cook']
        assert len(lArgs) > 1, "ERROR: not enough args: " +sDo +str(_lCmds)
        sCmd = lArgs[1]

        if sCmd == 'list':
            self.poutput("Known Chefs: %r" % (lKnownChefs,))
            return

        if sCmd == 'get' or (sCmd == 'set' and len(lArgs) == 2):
            self.poutput("Current Chef: %s" % (self.sChef,))
            return

        assert sCmd in _lCmds, "ERROR: not in: " +sDo +str(_lCmds)

        if sCmd == 'set':
            assert len(lArgs) > 2, \
                   "ERROR: %s %s needs one of: %s" % (
                sDo, sCmd, '|'.join(lKnownChefs))
            sNewChef = str(lArgs[2])
            assert sNewChef in lKnownChefs, \
                   "ERROR: %s %s %s not in: %s" % (
                sDo, sCmd, sNewChef, '|'.join(lKnownChefs))
            if self.sChef != sNewChef:
                self.sChef = sNewChef
                oChef = oEnsureChef(self, oOpts, sNewChef=sNewChef)
            #? do we update the config file? - I think not
            #? self.oConfig['backtest']['sChef'] = sNewChef
            return

        # The following all require that a feed has been loaded
        assert sFEED_CACHE_KEY
        assert sFEED_CACHE_KEY in dFEED_CACHE
        _dCurrentFeedFrame = dFEED_CACHE[sFEED_CACHE_KEY]
        assert _dCurrentFeedFrame

        # There's always a default provided of these
        oOm = oEnsureOmlette(self, oOpts)
        oRecipe = oEnsureRecipe(self, oOpts)
        oChefModule = oEnsureChef(self, oOpts)

        if sCmd == 'cook':
            from .OTBackTest import oPyBacktestCook
            assert oRecipe.dIngredients
            # ugly
            dFeeds = _dCurrentFeedFrame

            oBt = oPyBacktestCook(dFeeds, oRecipe, oChefModule, oOm)
            assert oBt is not None
            if type(oBt) == str:
                raise RuntimeError(oBt)
            oOm.oBt = oBt
            # self.vDebug("Cooked " + oBt.sSummary())
            return

        self.vError("Unrecognized chef command: " + str(oArgs) +'\n' +__doc__)
        return

    oOm = oEnsureOmlette(self, oOpts)
    oRecipe = oEnsureRecipe(self, oOpts)
    oChefModule = oEnsureChef(self, oOpts)

    # List the servings the chef has cooked, and dish out the servings
    if sDo == 'servings':
        __doc__ = sBACservings__doc__
        if not hasattr(oOm, 'oBt') or not oOm.oBt:
            self.vError("You must use \"chef cook\" before you can have servings")
            return
        oBt = oOm.oBt

        # ['signals', 'trades', 'positions', 'equity', 'reviews', 'trade_price']
        _lCmds = oChefModule.lProducedServings[:]
        if tabview not in _lCmds: _lCmds += ['tabview']

        if len(lArgs) == 1 or lArgs[1] == 'list':
            self.poutput("Produced Servings: %r" % (oChefModule.lProducedServings,))
            return

        assert len(lArgs) > 1, "ERROR: argument required" +sDo +str(_lCmds)
        sCmd = lArgs[1]
        # self.vDebug("lProducedServings: " + repr(_lCmds))
        assert sCmd in _lCmds, "ERROR: %s %s not in %r" % (
            sDo, sCmd, _lCmds)

        oFd = sys.stdout
        ## oFun = getattr(self.oBt, sCmd)
        ## self.poutput(oFun())
        if sCmd == 'signals':
            # this was the same as: oBt._mSignals = bt.mSignals() or oBt.signals
            oBt._mSignals = oRecipe.mSignals(oBt)
            oFd.write('INFO:  bt.signals found: %d\n' % len(oBt.signals))
            oOm.vAppendHdf('recipe/servings/mSignals', oBt.signals)
            return

        if sCmd == 'trades':
            # this was the same as: oBt._mTrades =  oBt.mTrades() or oBt.trades
            oBt._mTrades = oRecipe.mTrades(oBt)
            oFd.write('INFO:  bt.trades found: %d\n' % len(oBt.trades))
            oOm.vAppendHdf('recipe/servings/mTrades', oBt.trades)
            return

        if sCmd == 'positions':
            # this was the same as: oBt._rPositions = oBt.rPositions() or oBt.positions
            oBt._rPositions = oRecipe.rPositions(oBt, init_pos=0)
            oFd.write('INFO:  bt.positions found: %d\n' % len(oBt.positions))
            oOm.vAppendHdf('recipe/servings/rPositions', oBt.positions)
            return

        if sCmd == 'equity':
            # this was the same as: oBt._rEquity = oBt.rEquity() or oBt.equity
            oBt._rEquity = oRecipe.rEquity(oBt)
            oFd.write('INFO:  bt.equity found: %d\n' % len(oBt.equity))
            oOm.vAppendHdf('recipe/servings/rEquity', oBt.equity)
            return

        if sCmd == 'trade_price':
            # oFd.write('INFO:  bt.rTradePrice() found: %d\n' % len(oBt.rTradePrice()))
            oFd.write('INFO:  bt.trade_price found: %d\n' % len(oBt.trade_price))
            oOm.vAppendHdf('recipe/servings/rTradePrice', oBt.trade_price)
            return

        if sCmd == 'metrics' or sCmd == 'reviews':
            oOm.vSetTitleHdf('recipe/servings', oOm.oChefModule.sChef)
            #? Leave this as derived or store it? reviews?
            oOm.vSetMetadataHdf('recipe/servings', oBt.dSummary())
            oFd.write(oBt.sSummary())
            return

        if tabview and sCmd == 'tabview':
            assert len(lArgs) > 2, "ERROR: " +sDo +" " +sCmd \
                   +": serving required, one of: reviews " +str(oChefModule.lProducedServings)

            sServing = lArgs[2]
            if sServing in ['metrics','reviews']:
                l = [['Metric', 'Value']]
                l += oBt.lSummary()
                l.sort()
                # , hdr_rows=lHdrRows
                tabview.view(l, column_width='max')
                return

            assert sServing in oChefModule.lProducedServings
            mDf = getattr(oBt, sServing)
            # FixMe: need index timestamp for mva
            tabview.view(mDf)
            return

        self.vError("Unrecognized servings command: " + str(oArgs) +'\n' +__doc__)
        return

    # Plot the servings the chef has cooked, using matplotlib
    if sDo == 'plot':
        __doc__ = sBACplot__doc__
        _lCmds = ['show', 'trades', 'equity']
        if not hasattr(oOm, 'oBt') or not oOm.oBt:
            self.vError("You must use \"chef cook\" before you can plot")
            return
        oBt = oOm.oBt

        assert len(lArgs) > 1, "ERROR: " +sDo +str(_lCmds)
        sCmd = lArgs[1]
        assert sCmd in _lCmds, "ERROR: " +sDo +str(_lCmds)

        import matplotlib
        import matplotlib.pylab as pylab
        if sCmd == 'show':
            pylab.show()
            return

        # FixMe:
        matplotlib.rcParams['figure.figsize'] = (10, 5)

        if sCmd == 'equity':
            # FixMe: derive the period from the sTimeFrame
            sPeriod='W'
            close_label='C'
            mOhlc = oRecipe.dIngredients['mOhlc']
            oChefModule.vPlotEquity(oBt.equity, mOhlc, sTitle="%s\nEquity" % repr(oBt),
                                    sPeriod=sPeriod,
                                    close_label=close_label,
                                    )
            pylab.show()
            return

        if sCmd == 'trades':
            oBt.vPlotTrades()
            pylab.legend(loc='lower left')
            pylab.show()

            return
        self.vError("Unrecognized plot command: " + str(oArgs) +"\n" + sBAC__doc__)
        return

    self.vError("Unrecognized backtest command: " + str(oArgs) +"\n" + sBAC__doc__)
Exemple #5
0
def oOmain(lArgv):
    # FixMe: logging
    oFd = sys.stdout

    oArgParser = oParseOptions(__doc__.strip())
    oOptions = oArgParser.parse_args(lArgv)
    lArgs = oOptions.lArgs

    assert len(lArgs) == 4, "Give the CsvFile Symbol Timeframe and Year as arguments"

    #sDir = '/t/Program Files/HotForex MetaTrader/history/tools.fxdd.com'
    # sCsvFile = os.path.join(sDir, sSymbol + sTimeFrame +'-' +sYear +'.csv')
    sCsvFile = lArgs[0]
    sSymbol = lArgs[1] # 'EURGBP'
    sTimeFrame = lArgs[2] # '1440'
    sYear = lArgs[3] # '2014'
    # if not os.path.isfile(sCsvFile): vResampleFiles(sSymbol, sDir)
    assert os.path.isfile(sCsvFile)

    oOm = Omlette.Omlette(sHdfStore=oOptions.sOmelette, oFd=oFd)

    oRecipe = oOm.oAddRecipe(oOptions.sRecipe)
    oChefModule = oOm.oAddChef(oOptions.sChef)

    dFeedParams = dict(sTimeFrame=sTimeFrame, sSymbol=sSymbol, sYear=sYear)
    dFeed = oOm.dGetFeedFrame(sCsvFile, **dFeedParams)
    mFeedOhlc = dFeed['mFeedOhlc']

    mFeedOhlc = oPreprocessOhlc(mFeedOhlc)
    oFd.write('INFO:  Data Open length: %d\n' % len(mFeedOhlc))
    # ugly - should be a list of different feed timeframes etc.
    dFeeds = dict(mFeedOhlc=mFeedOhlc, dFeedParams=dFeedParams)
    # dRecipeParams now comes from the recipe ini file: bUseTalib=oOptions.bUseTalib
    dIngredientsParams = dict(dRecipeParams=dict())
    oRecipe.dMakeIngredients(dFeeds)
    assert oRecipe.dIngredients

    oBt = oPyBacktestCook(dFeeds, oRecipe, oChefModule, oOm)
    assert oBt is not None
    if isinstance(oBt, basestring):
        raise RuntimeError(oBt)

    # this was the same as: oBt._mSignals = bt.mSignals() or oBt.signals
    oBt._mSignals = oRecipe.mSignals(oBt)
    oFd.write('INFO:  bt.signals found: %d\n' % len(oBt.signals))
    oOm.vAppendHdf('recipe/servings/mSignals', oBt.signals)

    # this was the same as: oBt._mTrades =  oBt.mTrades() or oBt.trades
    oBt._mTrades = oRecipe.mTrades(oBt)
    oFd.write('INFO:  bt.trades found: %d\n' % len(oBt.trades))
    oOm.vAppendHdf('recipe/servings/mTrades', oBt.trades)

    # this was the same as: oBt._rPositions = oBt.rPositions() or oBt.positions
    oBt._rPositions = oRecipe.rPositions(oBt, init_pos=0)
    oFd.write('INFO:  bt.positions found: %d\n' % len(oBt.positions))
    oOm.vAppendHdf('recipe/servings/rPositions', oBt.positions)

    # this was the same as: oBt._rEquity = oBt.rEquity() or oBt.equity
    oBt._rEquity = oRecipe.rEquity(oBt)
    oFd.write('INFO:  bt.equity found: %d\n' % len(oBt.equity))
    oOm.vAppendHdf('recipe/servings/rEquity', oBt.equity)

    # oFd.write('INFO:  bt.rTradePrice() found: %d\n' % len(oBt.rTradePrice()))
    oFd.write('INFO:  bt.trade_price found: %d\n' % len(oBt.trade_price))
    oOm.vAppendHdf('recipe/servings/rTradePrice', oBt.trade_price)

    oOm.vSetTitleHdf('recipe/servings', oOm.oChefModule.sChef)
    #? Leave this as derived or store it? reviews?
    oOm.vSetMetadataHdf('recipe/servings', oBt.dSummary())
    oFd.write(oBt.sSummary())

    oOm.vSetMetadataHdf('recipe', dict(oRecipe.oConfig))

    if oOptions.bPlotEquity:
        mOhlc = oRecipe.dIngredients['mOhlc']
        vPlotEquityCurves(oBt, mOhlc, oOm.oChefModule)
    return oOm