Esempio n. 1
0
File: models.py Progetto: pgwthf/TSB
 def _init_runs(self):
     '''
     Initialisation of overall running parameters. Must only be used once
     for an MS instance.
     '''
     self.positions = Positions()
     self.trades = Trades()
     self.equity = EquityHistory()
     self.build_structure()
     # the same date range is used for all stocks in the pool:
     self.pool.index.global_date_range = (self.startdate, self.enddate)
Esempio n. 2
0
File: views.py Progetto: pgwthf/TSB
def draw_equity_chart(request, system_id):
    '''
    Return full size chart of the equity curve of this result.
    '''
    system = System.objects.get(id=system_id)
    metasystem = system.metasystem
    metasystem.make_system(system.params)

    equity = EquityHistory(system_id=system_id)
    if not equity.exists():
        return None
    total = equity.percentage()

    index = metasystem.pool.index
    index.date_range = (total.dates[0], total.dates[-1])

    markettype = metasystem.markettype
    chart = index.chart(title='equity', style='market_type', size=(1024, 480), 
            markettype=markettype)
    chart.add_secondary(total, colour='blue')
    return chart.plot()
Esempio n. 3
0
File: models.py Progetto: pgwthf/TSB
class MetaSystem(models.Model):
    '''
    MetaSystem defines the structure of a system. It has a set of variable 
    parameters, setting the parameters turns a MetaSystem into a System. 
    Parameters can be assigned random values at runtime.

    MetaSystems become read-only (and cannot be deleted) as soon as they have 
    Systems. The only fields that can be written after are:
        * name
        * description
        * parameters that were indicated as "variable" - so they appear in
            system.params
    '''
    name = models.CharField(max_length=20, unique=True)
    comments = models.CharField(max_length=100, blank=True)
    startdate = models.DateField()
    enddate = models.DateField()
    pool = models.ForeignKey(Pool)
    markettype = fields.ParamField(par_type='market')
    active = models.BooleanField(default=False)
    allocation = fields.ParamField(par_type='alloc')
    equitymodel = fields.ParamField(par_type='equity')
    maxresults = models.PositiveIntegerField(default=0)
    conditions = models.CharField(max_length=250, blank=True, null=True)
    startcash = models.PositiveIntegerField()
    group = models.ForeignKey(Group, null=True, default=None)


    def build_structure(self):
        '''
        Builds the structure of the MetaSystem in memory.
        '''
        self.methods = self.method_set.all().order_by('markettype', 'id')
        for method in self.methods:
            method.entries = method.entry_set.all()
            method.exits = method.exit_set.all()


    def _traverse_structure(self, function, **kwargs):
        '''
        Traverses the metasystem structure and applies the same method 
        <function> with <kwargs> to every parameter field.
        Return values must be dicts.
        '''
        retval = {}
        retval.update(getattr(self.allocation, function)(**kwargs))
        retval.update(getattr(self.markettype, function)(**kwargs))
        retval.update(getattr(self.equitymodel, function)(**kwargs))
        for method in self.methods:
            retval.update(getattr(method.rank, function)(
                    method_id=method.id, **kwargs))
            for entry in method.entries:
                retval.update(getattr(entry.params, function)(
                    method_id=method.id, **kwargs))
            for exit_ in method.exits:
                retval.update(getattr(exit_.params, function)(
                    method_id=method.id, **kwargs))
        return retval


    def _set_variable_parameters(self, param_dict):
        '''
        Sets all parameters with variable ranges to fixed values.
        <param_dict> is a dict with unique-key: value pairs.
        '''
        self._traverse_structure('set_variable_parameters', 
                parameters=param_dict)


    def _get_variable_parameters(self):
        '''
        Returns a dict with all variable parameters of this metasystem and 
        their (fixed) values. The keys of the dict are the names of the 
        parameters, but modified to be unique.
        '''
        return self._traverse_structure('get_variable_parameter_values')


    def _del_variable_parameters(self):
        '''
        Deletes the fixed value of the variable parameters from every parameter
        in this metasystem.
        '''
        self._traverse_structure('del_variable_parameters')


    def make_system(self, parameters):
        '''
        Sets all parameters with variable ranges to fixed values. This turns
        a MetaSystem into a system.
        '''
        self.build_structure()
        self._set_variable_parameters(str2dict(parameters))
        self._traverse_structure('initialise', pool=self.pool)


    def _init_runs(self):
        '''
        Initialisation of overall running parameters. Must only be used once
        for an MS instance.
        '''
        self.positions = Positions()
        self.trades = Trades()
        self.equity = EquityHistory()
        self.build_structure()
        # the same date range is used for all stocks in the pool:
        self.pool.index.global_date_range = (self.startdate, self.enddate)


    def _reset(self):
        '''
        Clears all positions, trades and equity. Run this before each backtest.
        '''
        self.positions.clear()
        self.trades.clear()
        self.equity.clear(self.startcash)


    def _traverse_parameters(self, value_lists, values=None):
        '''
        Runs the system for all parameter combinations
        '''
        if values is None:
            values = {}
        if len(value_lists) == 0:
#            self._reset()
            self._set_variable_parameters(values)
            self._traverse_structure('initialise', pool=self.pool)
            self._run_once()
        else:
            key, value_list = value_lists[0]
            for value in value_list:
                values[key] = value
                self._traverse_parameters(value_lists[1:], values)


#    def _init_system_run(self):
#        '''
#        Assigns parameters (unique_keys) to the MetaSystem,
#        so it can be used to backtest or real-time run a system.
#        '''
##        self._reset()
#        traverse_lists = self._traverse_structure('assign_values')
#        if traverse_lists:
#            print "TRAVERSE_LIST", traverse_lists
#            self._traverse_parameters(traverse_lists.items())
#        else:
#            self._traverse_structure('initialise', pool=self.pool)
#            self._run_once()
##        # the same date range is used for all stocks in the pool:
##        self.pool.index.global_date_range = (self.startdate, self.enddate)


    def _run_once(self):
        '''
        Run a single backtest for which all parameters have already been set.
        '''
        self._reset()
        signals = Signals()
        for today in self.pool.index.price.close.get_dates(self.startdate, 
                self.enddate):
            print 'TODAY: ', today
            todays_trades = signals.trade(today, self.positions, self.equity)
            self.trades.extend(todays_trades)
            ordered_method_list = self.markettype.get_methods(date=today,
                    method_list=self.methods)
            # generate exit signals for tomorrow:
            signals.set_exits(today, self.positions, self.pool, 
                    ordered_method_list)
            # generate entry signals for tomorrow:
            for method in ordered_method_list:
                signals.set_entries(today, method, self.pool, self.positions)
            # set position size of entry signals:
            signals.set_volume(today, self.allocation, self.equity, 
                    self.equitymodel, self.positions)

#            for signal in signals.entries:
#                print signal.stock, signal.volume
#            raise SystemExit()

        self.trades.extend(self.positions.close_all(today))
        self.trades.calc_performance(self.startdate, self.enddate, 
                                                self.positions.max_positions)
        data = self.trades.data
        sysdata = {'metasystem': self, 'active': False, 
                'params': repr(self._get_variable_parameters()),
                'enddate': self.enddate}
        data.update(sysdata)
        data.update(self.equity.calc_results())

#debugging
#        system = System.objects.create(**data) #keep only this
#the following replaces the previous line for debugging
        try:
            system = System.objects.create(**data)
        except:
            print data
            raise ValueError('ERROR saving system')
#/debugging

        self.equity.write_thumbnail_to_db(system)
        if eval_conditions(self.conditions, data):
            self.trades.write_to_db(system)
            self.equity.save(system)
        self._del_variable_parameters()


    def check_duplicate_entries_exits(self):
        '''
        Returns True if there are multiple entries or exits that use the same
        rule.
        '''
        duplicates = False
        for method in self.method_set.all():
            if method.check_duplicate_entries_exits():
                duplicates = True
        return duplicates


    def check_entries_exits(self):
        '''
        Returns errors if any method does not have at least one exit and one
        entry.
        '''
        errors = []
        for i, method in enumerate(self.methods):
            if not len(method.entries):
                errors.append('no entries found for method {}'.format(i))
            if not len(method.exits):
                errors.append('no exits found for method {}'.format(i))
        return errors


    def set_inactive(self):
        '''
        Set the current (Meta)System instance to inactive.
        '''
        self.active = False
        self.save()


    def set_active(self):
        '''
        Activates the MetaSystem for generating systems, if it has:
            * at least one variable parameter,
            * at least one entry and one exit,
            * a valid end date that is before today, and
            * a pool with a sufficient date range.
#TODO: reconsider if all this errors business is required
        '''
        errors = []
        self._init_runs()
        errors.extend(self.check_entries_exits())
        if not self.enddate or self.enddate >= datetime.date.today():
            errors.append('invalid end date')
        errors.extend(self.pool.check_date_ranges(self.startdate, 
                self.enddate))
        if not errors:
            self.active = True
            self.save()
            self._run_all() # start separate process and return
        return errors


    def has_systems(self):
        '''
        Return True if this metasystems has systems
        '''
        return self.system_set.count() > 0


    def is_discretionary(self):
        '''
        Return True if this metasystem is discretionary
TODO: discr systems should have LONG + SHORT method
        '''
        methods = self.method_set.all()
        if methods.count() != 1:
            return False
        entries = methods[0].entry_set.all()
        exits = methods[0].exit_set.all()
        print entries.count(), exits.count()
        if (entries.count() != 1) or (exits.count() != 1):
            return False
        print entries[0].params.get_dict()
        if entries[0].params.rule != 'd' or exits[0].params.rule != 'disc':
            return False
        return True


    def is_randomisable(self):
        '''
        Return True if this metasystem has random parameters and no parameter
        lists.
        '''
        if not hasattr(self, 'methods'):
            self.build_structure()
        randomisable = self._traverse_structure('is_randomisable')
        traversable = self._traverse_structure('get_lists')
        return randomisable and not traversable


    def is_traversable(self):
        '''
        Return True if this metasystem has at least one parameter list and 
        no random parameters.
        '''
        if not hasattr(self, 'methods'):
            self.build_structure()
        randomisable = self._traverse_structure('is_randomisable')
        n = self.get_number_of_calculations()
        return (n is not None) and (n > 1) and (not randomisable)


    def get_number_of_calculations(self):
        '''
        Return the total number of calculations if this is a traversable 
        metasystem. Return 0 if this metasystem is not traversable, 1 if only
        a single calculation is required.
        '''
        if not hasattr(self, 'methods'):
            self.build_structure()
        if self._traverse_structure('is_randomisable'):
            return 0
        traversable = self._traverse_structure('get_lists')
        n = 1
        for unused, value in traversable.items():
            n *= len(value)
        return n


#    @queue_command
#TODO: start this in separate process
    def _run_all(self):
        '''
TODO: drop speed measurement and move to % of total progress
        Run this MetaSystem with random parameters.
        The average number of calculations per hour is written to the
        database so it can be displayed by a view.
        '''
        self._init_runs()
        if self.is_randomisable():

            rs = None
            rs_key = 'TSBspeed{}'.format(self.id)
            factor_curr = 2./11.         # Exponential moving average over 10
            factor_prev = 1 - factor_curr #     speed measurements
            prev_time = time.time()
            moving_avg = 0
            while MetaSystem.objects.get(id=self.id).active:
                self._traverse_structure('randomise')
                print self._traverse_structure('get_variable_parameter_values')
                self._traverse_structure('initialise', pool=self.pool)
                self._run_once()
            # save running speed:
#TODO: works only for single process, need to count how many were added
                curr_time = time.time()
                speed = 3600 // (curr_time - prev_time)
                prev_time = curr_time
                if not moving_avg:
                    moving_avg = speed
                moving_avg = moving_avg * factor_prev + speed * factor_curr
                rs.set(rs_key, moving_avg)
                db.reset_queries() # this prevents flooding the memory
                if self.system_set.all().count() >= \
                        MetaSystem.objects.get(id=self.id).maxresults:
                    break

        elif self.is_traversable():
            traverse_lists = self._traverse_structure('get_lists')
            self._traverse_parameters(traverse_lists.items())

        elif self.get_number_of_calculations() == 1:
            # run once - all parameters were already stored as properties
            self._traverse_structure('randomise')
            self._traverse_structure('initialise', pool=self.pool)
            self._run_once()

        else:
            print('Random AND lists detected - cannot run')
        self.set_inactive()


    def get_system_summary(self, parameter_list):
        '''
        Return a dict with the best performance statistics of systems in this
        metasystem. If there are no systems, return None
        '''
        systems = self.system_set.all()
        if not systems:
            return None
        values = {}
        for sys in systems:
            for par in parameter_list:
                val = getattr(sys, par)
                if par not in values or val > values[par]:
                    values[par] = val
        return values


    def copy(self):
        '''
        Generate a deep copy of the current MetaSystem instance. Associated 
        Systems are *NOT* copied.
        '''
        return MetaSystem._copy(self)


    @classmethod
    def _copy(cls, instance):
        '''
        Copy <instance> to a new instance with a unique name.

        It is a deep copy for all associated methods, entries and exits are 
        copied too.
        '''
        fields = model_to_dict(instance, exclude=['id'])
        fields['pool'] = instance.pool
        fields['group'] = instance.group
        newname = fields['name'][:19] + random_string(1)
        fields['name'] = newname
        new_system = cls.objects.create(**fields)
        Method.copy_all(instance, new_system)
        return new_system


    def __unicode__(self):
        return self.name
Esempio n. 4
0
File: views.py Progetto: pgwthf/TSB
def edit_system(request, system_id=None):
    '''
    Show System with <system_id>
    '''
    message = ''
    system = System.objects.get(id=system_id)

    # generate performance list
    performance = []
    results = []
    for param, fmt in OUTPUT_FORMATS.items():
        value = getattr(system, param, None)
        if fmt['type'] == 'performance' and value is not None: # 0 is allowed!
            performance.append((fmt['name'], fmt['format'].format(value)))
            results.append((fmt['name'], fmt['format'].format(value)))

    #generate equity table
    equity = EquityHistory(system_id=system.id)
    if equity.exists():
        total_equity = equity.get('year')
        equity_table = EquityTable(total_equity, order_by=('date',))
        RequestConfig(request, paginate=None).configure(equity_table)
    else:
        equity_table = None
    #/equity table

    #generate portfolio table
    trades = Trade.objects.filter(system=system, rule_exit=None)
    if len(trades):
        latest_date = trades[0].stock.get_latest_date()
        stoploss_date = next_weekday(latest_date)
    else:
        stoploss_date = None
    portfolio_table = PortfolioTable(trades, order_by=('date_entry',))
    RequestConfig(request, paginate=None).configure(portfolio_table)

    #generate trades table
    trades = Trade.objects.filter(system=system, rule_exit__isnull=False)
    trades_table = TradesTable(trades, exclude=('id', 'system'),
                                                    order_by=('date_entry',))
    RequestConfig(request, paginate=None).configure(trades_table)
    #/trades table

    # copy system to method if required
    if request.method == 'POST':
        if request.POST.get('todo') == 'Delete trade':
            trade_list = request.POST.getlist('tradeid')
            Trade.objects.filter(id__in=trade_list).delete()
            tradeform = TradeForm()
        else:
            tradeform = TradeForm(data=request.POST)
            if tradeform.is_valid():
                name = tradeform.cleaned_data.get('symbol').upper()
                volume = tradeform.cleaned_data.get('volume')
                price = tradeform.cleaned_data.get('price')
                date = tradeform.cleaned_data.get('date')
                method = system.metasystem.method_set.all()[0]
                if request.POST.get('todo') == 'Enter position': 
                    if not volume:
                        message = 'volume must be specified for entry'
                    else: 
                        try:
                            stock = Stock.objects.get(name=name)
                            print 'NAME', stock.name
                            trade = Trade(system=system, stock=stock,
                                    method=method, rule_entry='discret.',
                                    price_entry=price, date_entry=date,
                                    volume=volume)
                            trade.save()
                            message = 'Position entered'
                        except:
                            message = 'Failed to enter position'
                elif request.POST.get('todo') == 'Exit position':
                    position_list = request.POST.getlist('positionid')
                    if len(position_list) != 1:
                        message = 'One position must be selected to exit'
                    else:
                        position_id = position_list[0]
                        stock = Trade.objects.get(id=position_id).stock
                        try:
                            trade = Trade.objects.get(id=position_id,
                                    system=system, stock=stock, method=method, 
                                    rule_entry='discret.', rule_exit=None)
                            trade.rule_exit = 'discret.'
                            trade.price_exit = price
                            trade.date_exit = date
                            trade.save()
                            message = 'Position exited'
                        except:
                            message = 'Failed to exit position'

    else: # this is the first page load
        tradeform = TradeForm()


    return render(request, 'edit_system.html', 
            {'system': system,
            'trades_table': trades_table,
            'stoploss_date': stoploss_date,
            'equity_table': equity_table,
            'portfolio_table': portfolio_table,
            'tradeform': tradeform,
            'performance': performance,
            'result': results,
            'message': message })
Esempio n. 5
0
File: views.py Progetto: pgwthf/TSB
def show_system(request, system_id=None):
    '''
    Show System with <system_id>
    '''
    notify = None

    system = System.objects.get(id=system_id)
    metasystem = system.metasystem
    metasystem.make_system(system.params)

    # generate performance list
    performance = []
    results = []
    for param, fmt in OUTPUT_FORMATS.items():
        value = getattr(system, param, None)
        if fmt['type'] == 'performance' and value is not None: # 0 is allowed!
            performance.append((fmt['name'], fmt['format'].format(value)))
        if fmt['type'] == 'result' and value is not None:
            results.append((fmt['name'], fmt['format'].format(value)))

    #generate equity table
    equity = EquityHistory(system_id=system.id)
    if equity.exists():
        total_equity = equity.get('year')
        equity_table = EquityTable(total_equity, order_by=('date',))
        RequestConfig(request, paginate=None).configure(equity_table)
    else:
        equity_table = None
    #/equity table

    # copy system to method if required
    if request.method == 'POST':
        copyform = CopyForm(data=request.POST)
        if request.POST.get('action') == 'Copy System' and copyform.is_valid():
            new_ms_id = copyform.cleaned_data['metasystem']
            reverse = copyform.cleaned_data['reverse']
            for method in metasystem.method_set.all():
                parameters = system.get_params(method)
                method.copy(new_ms_id, parameters, reverse=reverse,
                        comment='copy from system {}'.format(system.id))
    else:
        copyform = CopyForm()

    # bookmark system if required
    if request.method == 'POST':
        bookmarkform = BookmarkForm(data=request.POST)
        if request.POST.get('action') == 'Bookmark Selected':
            process_bookmarkform(bookmarkform, request.POST, [system_id])
    elif system.bookmark:
        bookmarkform = BookmarkForm(initial={'bookmark': system.bookmark.id})
    else:
        bookmarkform = BookmarkForm()

    # process trades/positions
    tradeform = TradeForm()
    if request.method == 'POST':
        if request.POST.get('action') in ('Delete trade', 'Delete position'):
            trade_list = request.POST.getlist('tradeid')
            trades = ','.join(Trade.objects.get(id=t).stock.name for t in 
                    trade_list)
            if not len(trade_list):
                notify = Notify('Delete failed, select position to delete')
            else:
                notify = Notify('Delete Trade(s) {}?'.format(trades))
                notify.set_replies(request, ('Yes', 'No'))
                request.session['trade_delete_list'] = trade_list

        elif request.POST.get('reply'):
            trade_list = request.session.get('trade_delete_list')
            if not trade_list:
                raise AttributeError('session has no valid trade_delete_list')
            del request.session['trade_delete_list']
            if request.POST.get('reply') == 'Yes':
                trades = ','.join(Trade.objects.get(id=t).stock.name for t in 
                        trade_list)
                Trade.objects.filter(id__in=trade_list).delete()
                notify = Notify('Trade(s) {} deleted'.format(trades))
            elif request.POST.get('reply') == 'No':
                notify = Notify('Delete cancelled')

        else:
            tradeform = TradeForm(data=request.POST)
            if tradeform.is_valid():
                name = tradeform.cleaned_data.get('symbol').upper()
                volume = tradeform.cleaned_data.get('volume')
                price = tradeform.cleaned_data.get('price')
                date = tradeform.cleaned_data.get('date')
                method = system.metasystem.method_set.all()[0]
                if request.POST.get('action') == 'Enter position': 
                    if not volume:
                        notify = Notify('volume must be specified for entry')
                    else: 
                        try:
                            stock = Stock.objects.get(name=name)
                            print 'NAME', stock.name
                            trade = Trade(system=system, stock=stock,
                                    method=method, rule_entry='discret.',
                                    price_entry=price, date_entry=date,
                                    volume=volume)
                            trade.save()
                            notify = Notify('Position entered')
                        except:
                            notify = Notify('Failed to enter position')
                elif request.POST.get('action') == 'Exit position':
                    position_list = request.POST.getlist('positionid')
                    if len(position_list) != 1:
                        notify = Notify('One position must be selected to exit')
                    else:
                        position_id = position_list[0]
                        stock = Trade.objects.get(id=position_id).stock
                        try:
                            trade = Trade.objects.get(id=position_id,
                                    system=system, stock=stock, method=method, 
                                    rule_entry='discret.', rule_exit=None)
                            trade.rule_exit = 'discret.'
                            trade.price_exit = price
                            trade.date_exit = date
                            trade.save()
                            notify = Notify('Position exited')
                        except:
                            notify = Notify('Failed to exit position')

    else: # this is the first page load
        tradeform = TradeForm()

    #generate portfolio table
    trades = Trade.objects.filter(system=system, rule_exit=None)
    if len(trades):
        latest_date = trades[0].stock.get_latest_date()
        stoploss_date = next_weekday(latest_date)
    else:
        stoploss_date = None
    portfolio_table = PortfolioTable(trades, order_by=('date_entry',))
    RequestConfig(request, paginate=None).configure(portfolio_table)

    #generate trades table
    trades = Trade.objects.filter(system=system).exclude(rule_exit=None)
    trades_table = TradesTable(trades, order_by=('date_entry',))
    RequestConfig(request, paginate=None).configure(trades_table)
    #/trades table


    return render(request, 'show_system.html', {
            'metasystem': metasystem,
            'system': system,
            'trades_table': trades_table,
            'equity_table': equity_table,
            'performance': performance,
            'result': results,
            'copyform': copyform,
            'bookmarkform': bookmarkform,
            'portfolio_table': portfolio_table,
            'stoploss_date': stoploss_date,
            'tradeform': tradeform,
            'notify': notify,
            })