def __init__(self, config_dir='', offline=False): ''' Parameters configuration_folder: str default is current location for configuration files offline: boolean Can force to switch off internal server ''' #NOTE timezone as parameter super(Setup, self).__init__() self.configuration_folder = config_dir if config_dir else '/'.join((os.environ['QTRADE'], 'config')) # Config data structures self.config_backtest = dict() self.config_strategie = dict() self.config_environment = self._inspect_environment() # Client for easy mysql database access self.datafeed = DataFeed() self.offline = offline if not offline: # It makes the entire simulator able to receive configuration and # send informations to remote users and other processes like web frontend self.server = network.ZMQ_Dealer(id=self.__class__.__name__)
def __init__(self, configuration): #NOTE Allowing different data access ? #self.metrics = None #self.server = ZMQ_Dealer(id=self.__class__.__name__) self.configuration = configuration if 'quandl' in configuration['env']: self.datafeed = DataFeed(configuration['env']['quandl']) else: self.datafeed = DataFeed()
def __init__(self, parameters): PortfolioManager.__init__(self, parameters) # Database access for symbols retrieving self.feeds = DataFeed() # R stuff: R functions file and rpy interface self.r = robjects.r portfolio_opt_file = '/'.join( (os.environ['QTRADE'], 'neuronquant/ai/opt_utils.R')) self.r('source("{}")'.format(portfolio_opt_file))
def __init__(self, country_code=None): ''' Parameters country_code: str This information is used to setup International object and get the right local conventions for lang, dates and currencies. None will stand for 'fr' ''' #self.locatioon = world.International(country_code) self.datafeed = DataFeed()
def __init__(self, data_descriptor, **kwargs): assert isinstance(data_descriptor['index'], pd.tseries.index.DatetimeIndex) self.data_descriptor = data_descriptor # Unpack config dictionary with default values. self.sids = kwargs.get('sids', data_descriptor['tickers']) self.start = kwargs.get('start', data_descriptor['index'][0]) self.end = kwargs.get('end', data_descriptor['index'][-1]) # Hash_value for downstream sorting. self.arg_string = hash_args(data_descriptor, **kwargs) self._raw_data = None self.feed = DataFeed()
def __init__(self, parameters): ''' Parameters parameters : dict(...) Named parameters used either for general portfolio settings (server and constraints), and for user optimizer function ''' super(PortfolioManager, self).__init__() self.log = Logger('Manager') self.datafeed = DataFeed() self.portfolio = None self.date = None self.name = parameters.get('name', 'Chuck Norris') self._optimizer_parameters = parameters self.connected = False #TODO Should send stuff anyway, and accept new connections while running self.connected = parameters.get('connected', False) # Run the server if the engine didn't while it is asked if 'server' in parameters: self.server = parameters.pop('server') if self.server.port is None and self.connected: self.log.info('Binding manager on default port...') self.server.run(host='127.0.0.1', port=5570)
class Remote(object): ''' Entry point to remote access to data Mainly offer a simpler interface and a dataaccess harmonisation Plus symbol check and complete, timezone, currency conversion Plus reindexage ? ''' def __init__(self, country_code=None): ''' Parameters country_code: str This information is used to setup International object and get the right local conventions for lang, dates and currencies None will stand for 'fr' ''' #self.locatioon = world.International(country_code) self.datafeed = DataFeed() pass def fetch_equities_snapshot(self, *args, **kwargs): ''' Use Yahoo and google finance service to fetch current infromations about given equitiy names ______________________________________________ Parameters args: tuple company names to consider kwargs['level']: int Quantity of information level ______________________________________________ Return snapshot: pandas.DataFrame with names as columns and informations as index ''' equities = args if not equities: equities = kwargs.get('symbols', []) level = kwargs.get('level', 0) symbols = [self.datafeed.guess_name(equity) for equity in equities] if not level: # default level, the lightest snapshot = snapshot_yahoo_pandas(symbols) elif level == 1: snapshot = snapshot_google_light(symbols) elif level == 2: snapshot = snapshot_google_heavy(symbols) else: raise ValueError('Invalid level of information requested') # Give columns back equities names requested snapshot.columns = equities return snapshot def _localize_data(self, data): ''' Inspect data and applie localisation on date and monnaie values ''' assert isinstance(data, pd.Dataframe) or isinstance(data, dict)
class OptimalFrontier(PortfolioManager): ''' Compute with R the efficient frontier and pick up the optimize point on it ''' def __init__(self, parameters): PortfolioManager.__init__(self, parameters) # Database access for symbols retrieving self.feeds = DataFeed() # R stuff: R functions file and rpy interface self.r = robjects.r portfolio_opt_file = '/'.join((os.environ['QTRADE'], 'neuronquant/ai/opt_utils.R')) self.r('source("{}")'.format(portfolio_opt_file)) def optimize(self, date, to_buy, to_sell, parameters): symbols = [] allocations = dict() # Considere only portfolio positions + future positions - positions about to be sold positions = set([t for t in self.portfolio.positions.keys() if self.portfolio.positions[t].amount]).union(to_buy).difference(to_sell) if not positions and to_sell: for t in to_sell: allocations[t] = - parameters.get('perc_sell', 1.0) return allocations, 0, 1 try: assert(positions) except: self.log.error('** No positions determined') if len(positions) == 1: return {positions.pop(): parameters.get('max_weigths', 0.2)}, 0, 1 for p in positions: symbols.append(self.feeds.guess_name(p).lower()) loopback = parameters.get('loopback', 50) source = parameters.get('source', 'yahoo') start = pd.datetime.strftime(date - pd.datetools.BDay(loopback), format = '%Y-%m-%d') date = pd.datetime.strftime(date, format = '%Y-%m-%d') r_symbols = self.r('c("{}")'.format('", "'.join(symbols))) r_names = self.r('c("{}.Return")'.format('.Return", "'.join(symbols))) data = self.r('importSeries')(r_symbols, start, date, source=source) frontier = self.r('getEfficientFrontier')(data, r_names, points = 500, Debug = False, graph = False) if not frontier: self.log.warning('No optimal frontier found') return dict(), None, None try: mp = self.r('marketPortfolio')(frontier, 0.02, Debug = False, graph = False) except: self.log.error('** Error running R optimizer') return dict(), None, None self.log.debug('Allocation: {}'.format(mp)) #FIXME Some key errors survive so far for s, t in zip(symbols, positions): allocations[t] = round(mp.rx('.'.join((s, 'Return')))[0][0], 2) er = round(mp.rx('er')[0][0], 2) eStd = round(mp.rx('eStd')[0][0], 2) self.log.debug('Allocation: {}\nWith expected return: {}\tand expected risk: {}'.format(allocations, er, eStd)) return allocations, er, eStd
def __init__(self, parameters): PortfolioManager.__init__(self, parameters) # Database access for symbols retrieving self.feeds = DataFeed() # R stuff: R functions file and rpy interface self.r = robjects.r portfolio_opt_file = '/'.join((os.environ['QTRADE'], 'neuronquant/ai/opt_utils.R')) self.r('source("{}")'.format(portfolio_opt_file))
def __init__(self, data=None, flavor='mysql', lvl='debug'): #NOTE Allowing different data access ? self.data = data if not data: self.feeds = DataFeed() self.backtest_cfg = None self.algo_cfg = None self.manager_cfg = None self.monthly_perfs = None self.server = ZMQ_Dealer(id=self.__class__.__name__)
def __init__(self, data, **kwargs): assert isinstance(data['index'], pd.tseries.index.DatetimeIndex) self.data = data # Unpack config dictionary with default values. self.sids = kwargs.get('sids', data['tickers']) self.start = kwargs.get('start', data['index'][0]) self.end = kwargs.get('end', data['index'][-1]) self.fake_index = pd.date_range(self.start, self.end, freq=pd.datetools.BDay()) # Hash_value for downstream sorting. self.arg_string = hash_args(data, **kwargs) self._raw_data = None self.remote = Fetcher() self.feed = DataFeed()
def __init__(self, *args, **kwargs): super(Analyze, self).__init__() # MySQL Database client self.datafeed = kwargs.pop('datafeed') if 'datafeed' in kwargs else DataFeed() # R analysis file only need portfolio returns self.returns = kwargs.pop('returns') if 'returns' in kwargs else None # Final risk measurments as returned by the backtester self.results = kwargs.pop('results') if 'results' in kwargs else None # Simulation rolling performance self.metrics = kwargs.pop('metrics') if 'metrics' in kwargs else None # You better should know what was simulation's parameters self.configuration = kwargs.pop('configuration') if 'configuration' in kwargs else None
def __init__(self, configuration): ''' Parameters configuration : dict Named parameters used either for general portfolio settings (server and constraints), and for user optimizer function ''' super(PortfolioManager, self).__init__() # Easy mysql access self.datafeed = DataFeed() # Zipline portfolio object, updated during simulation with self.date self.portfolio = None self.date = None # Portfolio owner, mainly used for database saving and client communication self.name = configuration.get('name', 'ChuckNorris') self.log = logbook.Logger('Manager::' + self.name) # Other parameters are used in user optimize() method self._optimizer_parameters = configuration # Make the manager talk to connected clients self.connected = configuration.get('connected', False) # Send android notifications when orders are processed # It's only possible with a running server self.android = configuration.get('android', False) & self.connected # Delete from database data with the same portfolio name if configuration.get('clean', True): self.log.info('Cleaning previous trades.') clean_previous_trades(self.name) # Run the server if the engine didn't, while it is asked for if 'server' in configuration and self.connected: # Getting server object instanciated anyway before (by Setup object) self.server = configuration.pop('server') # Web based dashboard where real time results are monitored #FIXME With dynamic generation, dashboad never exists at this point #self.dashboard = Dashboard(self.name) # In case user optimization would need to retrieve more data self.remote = Remote()
def smart_tickers_select(tickers_description, exchange=''): ''' Take tickers string description and return an array of explicit and usuable symbols ''' # Informations are coma separated within the string tickers_description = tickers_description.split(',') # Useful way of stocks selection in order to test algorithm strength if tickers_description[0] == 'random': # Basic check: the second argument is the the number, integer, of stocks to pick up randomly assert len(tickers_description) == 2 assert int(tickers_description[1]) # Pick up stocks on specified (or not) market exchange tickers_description = DataFeed().random_stocks( int(tickers_description[1]), exchange=exchange.split(',')) return tickers_description
class OptimalFrontier(PortfolioManager): ''' Compute with R the efficient frontier and pick up the optimize point on it ''' def __init__(self, parameters): PortfolioManager.__init__(self, parameters) # Database access for symbols retrieving self.feeds = DataFeed() # R stuff: R functions file and rpy interface self.r = robjects.r portfolio_opt_file = '/'.join( (os.environ['QTRADE'], 'neuronquant/ai/opt_utils.R')) self.r('source("{}")'.format(portfolio_opt_file)) def optimize(self, date, to_buy, to_sell, parameters): symbols = [] allocations = dict() # Considere only portfolio positions + future positions - positions about to be sold positions = set([ t for t in self.portfolio.positions.keys() if self.portfolio.positions[t].amount ]).union(to_buy).difference(to_sell) if not positions and to_sell: for t in to_sell: allocations[t] = -parameters.get('perc_sell', 1.0) return allocations, 0, 1 try: assert (positions) except: self.log.error('** No positions determined') if len(positions) == 1: return {positions.pop(): parameters.get('max_weigths', 0.2)}, 0, 1 for p in positions: symbols.append(self.feeds.guess_name(p).lower()) loopback = parameters.get('loopback', 50) source = parameters.get('source', 'yahoo') start = pd.datetime.strftime(date - pd.datetools.BDay(loopback), format='%Y-%m-%d') date = pd.datetime.strftime(date, format='%Y-%m-%d') r_symbols = self.r('c("{}")'.format('", "'.join(symbols))) r_names = self.r('c("{}.Return")'.format('.Return", "'.join(symbols))) data = self.r('importSeries')(r_symbols, start, date, source=source) frontier = self.r('getEfficientFrontier')(data, r_names, points=500, Debug=False, graph=False) if not frontier: self.log.warning('No optimal frontier found') return dict(), None, None try: mp = self.r('marketPortfolio')(frontier, 0.02, Debug=False, graph=False) except: self.log.error('** Error running R optimizer') return dict(), None, None self.log.debug('Allocation: {}'.format(mp)) #FIXME Some key errors survive so far for s, t in zip(symbols, positions): allocations[t] = round(mp.rx('.'.join((s, 'Return')))[0][0], 2) er = round(mp.rx('er')[0][0], 2) eStd = round(mp.rx('eStd')[0][0], 2) self.log.debug( 'Allocation: {}\nWith expected return: {}\tand expected risk: {}'. format(allocations, er, eStd)) return allocations, er, eStd
class DataLiveSource(DataSource): """ Yields all events in event_list that match the given sid_filter. If no event_list is specified, generates an internal stream of events to filter. Returns all events if filter is None. Configuration options: sids : list of values representing simulated internal sids start : start date delta : timedelta between internal events filter : filter to remove the sids """ def __init__(self, data, **kwargs): assert isinstance(data['index'], pd.tseries.index.DatetimeIndex) self.data = data # Unpack config dictionary with default values. self.sids = kwargs.get('sids', data['tickers']) self.start = kwargs.get('start', data['index'][0]) self.end = kwargs.get('end', data['index'][-1]) self.fake_index = pd.date_range(self.start, self.end, freq=pd.datetools.BDay()) # Hash_value for downstream sorting. self.arg_string = hash_args(data, **kwargs) self._raw_data = None self.remote = Fetcher() self.feed = DataFeed() @property def mapping(self): return { 'dt': (lambda x: x, 'dt'), 'sid': (lambda x: x, 'sid'), 'price': (float, 'price'), 'currency': (str, 'currency'), 'perc_change': (float, 'perc_change'), 'volume': (int, 'volume'), } @property def instance_hash(self): return self.arg_string def raw_data_gen(self): current_dt = datetime.datetime.now() index = self.data['index'] selector = (index.day > current_dt.day) \ | ((index.day == current_dt.day) & (index.hour > current_dt.hour)) \ | ((index.day == current_dt.day) & (index.hour == current_dt.hour) & (index.minute >= current_dt.minute)) #NOTE Not an equal size issue ? for fake_dt, dt in zip(self.fake_index, index[selector]): while (current_dt.minute != dt.minute) or (current_dt.hour != dt.hour): time.sleep(15) current_dt = datetime.datetime.now() print('Waiting {} / {}'.format(current_dt, dt)) #for fake_dt, dt in zip(self.fake_index, self.data['index']): for sid in self.data['tickers']: if sid in self.sids: symbol = self.feed.guess_name(sid).lower() #FIXME Erros because no markets specifie, use light=True and add market #snapshot = self.remote.get_stock_snapshot(symbol, light=False) snapshot = self.remote.get_stock_snapshot(symbol, light=True) import ipdb ipdb.set_trace() log.debug('Data available:\n{}'.format( json.dumps(snapshot, sort_keys=True, indent=4, separators=(',', ': ')))) if not snapshot: log.error( '** No data snapshot available, maybe stopped by google ?' ) sys.exit(2) event = { 'dt': fake_dt, 'trade_time': dt, 'sid': sid, 'price': float(snapshot[symbol]['last']), 'currency': snapshot[symbol]['currency'], 'perc_change': float(snapshot[symbol]['perc_change']), 'volume': int(snapshot[symbol]['volume']), } yield event @property def raw_data(self): if not self._raw_data: self._raw_data = self.raw_data_gen() return self._raw_data
class DBPriceSource(DataSource): """ Yields all events in event_list that match the given sid_filter. If no event_list is specified, generates an internal stream of events to filter. Returns all events if filter is None. Configuration options: sids : list of values representing simulated internal sids start : start date delta : timedelta between internal events filter : filter to remove the sids """ def __init__(self, data_descriptor, **kwargs): assert isinstance(data_descriptor['index'], pd.tseries.index.DatetimeIndex) self.data_descriptor = data_descriptor # Unpack config dictionary with default values. self.sids = kwargs.get('sids', data_descriptor['tickers']) self.start = kwargs.get('start', data_descriptor['index'][0]) self.end = kwargs.get('end', data_descriptor['index'][-1]) # Hash_value for downstream sorting. self.arg_string = hash_args(data_descriptor, **kwargs) self._raw_data = None self.feed = DataFeed() @property def mapping(self): return { 'dt': (lambda x: x, 'dt'), 'sid': (lambda x: x, 'sid'), 'price': (float, 'price'), 'volume': (int, 'volume'), } @property def instance_hash(self): return self.arg_string def _get(self): return self.feed.quotes(self.sids, start_date=self.start, end_date=self.end) def raw_data_gen(self): self.data = self._get() for dt, series in self.data.iterrows(): for sid, price in series.iterkv(): if sid in self.sids: event = { 'dt': dt, 'sid': sid, 'price': price, 'volume': 1000, } yield event @property def raw_data(self): if not self._raw_data: self._raw_data = self.raw_data_gen() return self._raw_data
class DataLiveSource(DataSource): """ Yields all events in event_list that match the given sid_filter. If no event_list is specified, generates an internal stream of events to filter. Returns all events if filter is None. Configuration options: sids : list of values representing simulated internal sids start : start date delta : timedelta between internal events filter : filter to remove the sids """ def __init__(self, data, **kwargs): assert isinstance(data['index'], pd.tseries.index.DatetimeIndex) self.data = data # Unpack config dictionary with default values. self.sids = kwargs.get('sids', data['tickers']) self.start = kwargs.get('start', data['index'][0]) self.end = kwargs.get('end', data['index'][-1]) self.fake_index = pd.date_range(self.start, self.end, freq=pd.datetools.BDay()) # Hash_value for downstream sorting. self.arg_string = hash_args(data, **kwargs) self._raw_data = None self.remote = Fetcher() self.feed = DataFeed() @property def mapping(self): return { 'dt': (lambda x: x, 'dt'), 'sid': (lambda x: x, 'sid'), 'price': (float, 'price'), 'currency': (str, 'currency'), 'perc_change': (float, 'perc_change'), 'volume': (int, 'volume'), } @property def instance_hash(self): return self.arg_string def raw_data_gen(self): current_dt = datetime.datetime.now() index = self.data['index'] selector = (index.day > current_dt.day) \ | ((index.day == current_dt.day) & (index.hour > current_dt.hour)) \ | ((index.day == current_dt.day) & (index.hour == current_dt.hour) & (index.minute >= current_dt.minute)) #NOTE Not an equal size issue ? for fake_dt, dt in zip(self.fake_index, index[selector]): while (current_dt.minute != dt.minute) or (current_dt.hour != dt.hour) : time.sleep(15) current_dt = datetime.datetime.now() print('Waiting {} / {}'.format(current_dt, dt)) #for fake_dt, dt in zip(self.fake_index, self.data['index']): for sid in self.data['tickers']: if sid in self.sids: symbol = self.feed.guess_name(sid).lower() #FIXME Erros because no markets specifie, use light=True and add market #snapshot = self.remote.get_stock_snapshot(symbol, light=False) snapshot = self.remote.get_stock_snapshot(symbol, light=True) import ipdb; ipdb.set_trace() log.debug('Data available:\n{}'.format(json.dumps(snapshot, sort_keys=True, indent=4, separators=(',', ': ')))) if not snapshot: log.error('** No data snapshot available, maybe stopped by google ?') sys.exit(2) event = { 'dt': fake_dt, 'trade_time': dt, 'sid': sid, 'price': float(snapshot[symbol]['last']), 'currency': snapshot[symbol]['currency'], 'perc_change': float(snapshot[symbol]['perc_change']), 'volume': int(snapshot[symbol]['volume']), } yield event @property def raw_data(self): if not self._raw_data: self._raw_data = self.raw_data_gen() return self._raw_data
class Simulation(object): ''' Take a trading strategie and evalute its results ''' def __init__(self, data=None, flavor='mysql', lvl='debug'): #NOTE Allowing different data access ? self.data = data if not data: self.feeds = DataFeed() self.backtest_cfg = None self.algo_cfg = None self.manager_cfg = None self.monthly_perfs = None self.server = ZMQ_Dealer(id=self.__class__.__name__) def configure(self, bt_cfg=None, a_cfg=None, m_cfg=None): ''' Reads and provides a clean configuration for the simulation ''' if not bt_cfg: parser = argparse.ArgumentParser(description='Backtester module, the terrific financial simukation') parser.add_argument('-v', '--version', action='version', version='%(prog)s v0.8.1 Licence rien du tout', help='Print program version') parser.add_argument('-d', '--delta', type=str, action='store', default='D', required=False, help='Delta in days betweend two quotes fetch') parser.add_argument('-a', '--algorithm', action='store', required=True, help='Trading algorithm to be used') parser.add_argument('-m', '--manager', action='store', required=True, help='Portfolio strategie to be used') parser.add_argument('-b', '--database', action='store', default='test', required=False, help='Table to considere in database') parser.add_argument('-t', '--tickers', action='store', required=True, help='target names to process') parser.add_argument('-s', '--start', action='store', default='1/1/2006', required=False, help='Start date of the backtester') parser.add_argument('-e', '--end', action='store', default='1/12/2010', required=False, help='Stop date of the backtester') parser.add_argument('-r', '--remote', action='store_true', help='Indicates if the program was ran manually or not') parser.add_argument('-l', '--live', action='store_true', help='makes the engine work in real-time !') parser.add_argument('-p', '--port', action='store', default=5570, required=False, help='Activates the diffusion of the universe on the network, on the port provided') args = parser.parse_args() self.backtest_cfg = {'algorithm' : args.algorithm, 'delta' : args.delta, 'manager' : args.manager, 'database' : args.database, 'tickers' : args.tickers.split(','), 'start' : args.start, 'end' : args.end, 'live' : args.live, 'port' : args.port, 'remote' : args.remote} else: self.backtest_cfg = bt_cfg if isinstance(self.backtest_cfg['start'], str) and isinstance(self.backtest_cfg['end'], str): ###############test = pd.datetime.strptime('2012-02-01:14:00', '%Y-%m-%d:%H:%M') self.backtest_cfg['start'] = pytz.utc.localize(pd.datetime.strptime(self.backtest_cfg['start'], '%Y-%m-%d')) self.backtest_cfg['end'] = pytz.utc.localize(pd.datetime.strptime(self.backtest_cfg['end'], '%Y-%m-%d')) elif isinstance(self.backtest_cfg['start'], pd.datetime) and isinstance(self.backtest_cfg['end'], pd.datetime): if not self.backtest_cfg['start'].tzinfo: self.backtest_cfg['start'] = pytz.utc.localize(self.backtest_cfg['start']) if not self.backtest_cfg['end'].tzinfo: self.backtest_cfg['end'] = pytz.utc.localize(self.backtest_cfg['end']) else: raise NotImplementedError() #if self.backtest_cfg['interactive']: self.read_config(remote=self.backtest_cfg['remote'], manager=m_cfg, algorithm=a_cfg) #else: #self.server.run_forever(port=self.backtest_cfg['port'], on_recv=self.read_config) return self.backtest_cfg def read_config(self, *args, **kwargs): self.manager_cfg = kwargs.get('manager', None) self.algo_cfg = kwargs.get('algorithm', None) if kwargs.get('remote', False): self.server.run(host='127.0.0.1', port=self.backtest_cfg['port']) # In remote mode, client sends missing configuration through zmq socket if (not self.manager_cfg) or (not self.algo_cfg): log.info('Fetching backtest configuration from client') msg = self.server.receive(json=True) log.debug('Got it !') # Set simulation parameters with it assert isinstance(msg, dict) if self.manager_cfg is None: self.manager_cfg = msg['manager'] if self.algo_cfg is None: self.algo_cfg = msg['algorithm'] else: # Reading configuration from json files config_dir = '/'.join((os.environ['QTRADE'], 'config/')) try: # Files store many algos and manager parameters, # use backtest configuration to pick up the right one if self.manager_cfg is None: self.manager_cfg = json.load(open('{}/managers.cfg'.format(config_dir), 'r'))[self.backtest_cfg['manager']] if self.algo_cfg is None: self.algo_cfg = json.load(open('{}/algos.cfg'.format(config_dir), 'r'))[self.backtest_cfg['algorithm']] except: log.error('** loading json configuration.') sys.exit(1) # The manager can use the same socket during simulation to emit portfolio informations self.manager_cfg['server'] = self.server log.info('Configuration is Done.') def run_backtest(self): # No configuration, no backtest man if self.backtest_cfg is None or self.algo_cfg is None or self.manager_cfg is None: log.error('** Backtester not configured properly') return 1 #'''_____________________________________________________________________ Parameters ________ if self.backtest_cfg['tickers'][0] == 'random': assert(len(self.backtest_cfg['tickers']) == 2) assert(int(self.backtest_cfg['tickers'][1])) self.backtest_cfg['tickers'] = self.feeds.random_stocks(int(self.backtest_cfg['tickers'][1])) if self.backtest_cfg['live']: #dates = pd.date_range(self.backtest_cfg['start'], self.backtest_cfg['end'], freq=self.backtest_cfg['delta']) #NOTE A temporary hack to avoid zipline dirty modification periods = self.backtest_cfg['end'] - self.backtest_cfg['start'] dates = pd.date_range(pd.datetime.now(), periods=periods.days + 1, freq=self.backtest_cfg['delta']) ''' if dates.freq > pd.datetools.Day(): #fr_selector = ((dates.hour > 8) & (dates.hour < 17)) | ((dates.hour == 17) & (dates.minute < 31)) us_selector = ((dates.hour > 15) & (dates.hour < 22)) | ((dates.hour == 15) & (dates.minute > 31)) dates = dates[us_selector] if not dates: log.warning('! Market closed.') sys.exit(0) ''' data = {'tickers': self.backtest_cfg['tickers'], 'index': dates.tz_localize(pytz.utc)} else: data = self.feeds.quotes(self.backtest_cfg['tickers'], start_date = self.backtest_cfg['start'], end_date = self.backtest_cfg['end']) assert isinstance(data, pd.DataFrame) assert data.index.tzinfo #___________________________________________________________________________ Running ________ log.info('\n-- Running backetester...\nUsing algorithm: {}\n'.format(self.backtest_cfg['algorithm'])) log.info('\n-- Using portfolio manager: {}\n'.format(self.backtest_cfg['manager'])) backtester = BacktesterEngine(self.backtest_cfg['algorithm'], self.backtest_cfg['manager'], self.algo_cfg, self.manager_cfg) self.results, self.monthly_perfs = backtester.run(data) #self.backtest_cfg['start'], #self.backtest_cfg['end']) return self.results def rolling_performances(self, timestamp='one_month', save=False, db_id=None): ''' Filters self.perfs and, if asked, save it to database ''' #TODO Study the impact of month choice #TODO Check timestamp in an enumeration #NOTE Script args define default database table name (test), make it consistent if db_id is None: db_id = self.backtest_cfg['algorithm'] + pd.datetime.strftime(pd.datetime.now(), format='%Y%m%d') if self.monthly_perfs: perfs = dict() length = range(len(self.monthly_perfs[timestamp])) index = self._get_index(self.monthly_perfs[timestamp]) perfs['Name'] = np.array([db_id] * len(self.monthly_perfs[timestamp])) #perfs['Period'] = np.array([self.monthly_perfs[timestamp][i]['period_label'] for i in length]) perfs['Period'] = np.array([pd.datetime.date(date) for date in index]) perfs['Sharpe.Ratio'] = np.array([self.monthly_perfs[timestamp][i]['sharpe'] for i in length]) perfs['Returns'] = np.array([self.monthly_perfs[timestamp][i]['algorithm_period_return'] for i in length]) perfs['Max.Drawdown'] = np.array([self.monthly_perfs[timestamp][i]['max_drawdown'] for i in length]) perfs['Volatility'] = np.array([self.monthly_perfs[timestamp][i]['algo_volatility'] for i in length]) perfs['Beta'] = np.array([self.monthly_perfs[timestamp][i]['beta'] for i in length]) perfs['Alpha'] = np.array([self.monthly_perfs[timestamp][i]['alpha'] for i in length]) perfs['Excess.Returns'] = np.array([self.monthly_perfs[timestamp][i]['excess_return'] for i in length]) perfs['Benchmark.Returns'] = np.array([self.monthly_perfs[timestamp][i]['benchmark_period_return'] for i in length]) perfs['Benchmark.Volatility'] = np.array([self.monthly_perfs[timestamp][i]['benchmark_volatility'] for i in length]) perfs['Treasury.Returns'] = np.array([self.monthly_perfs[timestamp][i]['treasury_period_return'] for i in length]) else: #TODO Get it from DB if it exists raise NotImplementedError() try: data = pd.DataFrame(perfs, index=index) except: import ipdb; ipdb.set_trace() if save: self.feeds.stock_db.save_metrics(data) return data def overall_metrics(self, timestamp='one_month', save=False, db_id=None): ''' Use zipline results to compute some performance indicators and store it in database ''' perfs = dict() metrics = self.rolling_performances(timestamp=timestamp, save=False, db_id=db_id) riskfree = np.mean(metrics['Treasury.Returns']) #NOTE Script args define default database table name (test), make it consistent if db_id is None: db_id = self.backtest_cfg['algorithm'] + pd.datetime.strftime(pd.datetime.now(), format='%Y%m%d') perfs['Name'] = db_id perfs['Sharpe.Ratio'] = tsu.get_sharpe_ratio(metrics['Returns'].values, risk_free = riskfree) perfs['Returns'] = (((metrics['Returns'] + 1).cumprod()) - 1)[-1] perfs['Max.Drawdown'] = max(metrics['Max.Drawdown']) perfs['Volatility'] = np.mean(metrics['Volatility']) perfs['Beta'] = np.mean(metrics['Beta']) perfs['Alpha'] = np.mean(metrics['Alpha']) perfs['Benchmark.Returns'] = (((metrics['Benchmark.Returns'] + 1).cumprod()) - 1)[-1] if save: self.feeds.stock_db.save_performances(perfs) return perfs #TODO Save returns def get_returns(self, benchmark=None, timestamp='one_month', save=False, db_id=None): returns = dict() if benchmark: #TODO Benchmark fields in database for guessing name like for stocks benchmark_symbol = '^GSPC' benchmark_data = get_benchmark_returns(benchmark_symbol, self.backtest_cfg['start'], self.backtest_cfg['end']) else: raise NotImplementedError() #benchmark_data = [d for d in benchmark_data if (d.date >= self.backtest_cfg['start']) and (d.date <= self.backtest_cfg['end'])] dates = pd.DatetimeIndex([d.date for d in benchmark_data]) returns['Benchmark.Returns'] = pd.Series([d.returns for d in benchmark_data], index=dates) returns['Benchmark.CReturns'] = ((returns['Benchmark.Returns'] + 1).cumprod()) - 1 returns['Returns'] = pd.Series(self.results.returns, index=dates) returns['CReturns'] = pd.Series(((self.results.returns + 1).cumprod()) - 1, index=dates) df = pd.DataFrame(returns, index=dates) if save: raise NotImplementedError() self.feeds.stock_db.saveDFToDB(df, table=db_id) if benchmark is None: df = df.drop(['Benchmark.Returns', 'Benchmark.CReturns'], axis=1) return df def _get_index(self, perfs): #NOTE No frequency infos or just period number ? start = pytz.utc.localize(pd.datetime.strptime(perfs[0]['period_label'] + '-01', '%Y-%m-%d')) end = pytz.utc.localize(pd.datetime.strptime(perfs[-1]['period_label'] + '-01', '%Y-%m-%d')) return pd.date_range(start - pd.datetools.BDay(10), end, freq=pd.datetools.MonthBegin()) def _extract_perf(self, perfs, field): index = self._get_index(perfs) values = [perfs[i][field] for i in range(len(perfs))] return pd.Series(values, index=index) def __del__(self): del self.server
class Remote(object): ''' Entry point to remote access to data Mainly offer a simpler interface and a dataaccess harmonisation Plus symbol check and complete, timezone, currency conversion Plus reindexage ? ''' def __init__(self, country_code=None): ''' Parameters country_code: str This information is used to setup International object and get the right local conventions for lang, dates and currencies. None will stand for 'fr' ''' #self.locatioon = world.International(country_code) self.datafeed = DataFeed() #NOTE with args and kwargs ? def fetch_equities_daily(self, equities, ohlc=False, r_type=False, returns=False, **kwargs): if len(equities) == 0: return pd.DataFrame() if isinstance(equities, str): equities = equities.split(',') symbols = [self.datafeed.guess_name(equity) for equity in equities] if ohlc: data = load_bars_from_yahoo(stocks=symbols, **kwargs) data.items = equities else: data = load_from_yahoo(stocks=symbols, **kwargs) data.columns = equities #NOTE Would it work with a pandas panel ? if returns: data = ((data - data.shift(1)) / data).fillna(method='bfill') if r_type: data = convert_to_r_matrix(data) return data def fetch_equities_snapshot(self, *args, **kwargs): ''' Use Yahoo and google finance service to fetch current infromations about given equitiy names ______________________________________________ Parameters args: tuple company names to consider kwargs['level']: int Quantity of information level ______________________________________________ Return snapshot: pandas.DataFrame with names as columns and informations as index ''' equities = args if not equities: equities = kwargs.get('symbols', []) level = kwargs.get('level', 0) #TODO Symbols are usualy reused, cach them symbols = [self.datafeed.guess_name(equity) for equity in equities] if not level: # default level, the lightest snapshot = snapshot_yahoo_pandas(symbols) elif level == 1: snapshot = snapshot_google_light(symbols) elif level == 2: snapshot = snapshot_google_heavy(symbols) else: raise ValueError('Invalid level of information requested') # Give columns back equities names requested snapshot.columns = equities return snapshot def _localize_data(self, data): ''' Inspect data and applie localisation on date and monnaie values ''' assert isinstance(data, pd.Dataframe) or isinstance(data, dict)
class Setup(object): ''' Configuration object for the Trading engine''' def __init__(self, config_dir='', offline=False): ''' Parameters configuration_folder: str default is current location for configuration files offline: boolean Can force to switch off internal server ''' #NOTE timezone as parameter super(Setup, self).__init__() self.configuration_folder = config_dir if config_dir else '/'.join((os.environ['QTRADE'], 'config')) # Config data structures self.config_backtest = dict() self.config_strategie = dict() self.config_environment = self._inspect_environment() # Client for easy mysql database access self.datafeed = DataFeed() self.offline = offline if not offline: # It makes the entire simulator able to receive configuration and # send informations to remote users and other processes like web frontend self.server = network.ZMQ_Dealer(id=self.__class__.__name__) def _inspect_environment(self, local_file='~/.quantrade/default.json'): ''' Read common user and project configuration files for usual environment parameters ''' context = dict() if os.path.exists(os.path.expanduser(local_file)): log.info('Found local configuration file, loading {}'.format(local_file)) context = self._read_structured_file(os.path.expanduser(local_file)) return context def _read_structured_file(self, formatfile, config_folder=False, select_field=None, format='json'): ''' Map well structured, i.e. common file format like key-value storage, csv, ..., file content into a dictionnary ''' if format == 'json': try: # Read given file in specified format from default or given config directory if config_folder: content = json.load(open('/'.join([self.configuration_folder, formatfile]), 'r')) else: content = json.load(open(formatfile, 'r')) except: log.error('** loading json configuration.') return dict() else: #TODO Other key-value file storage style raise NotImplementedError() # Configuration fields are likely to have several parameter categories # If specified, return only 'select_field' one return content[select_field] if select_field else content def parse_commandline(self): ''' Read command lines arguments and map them to a more usuable dictionnary _________________________________________ Return self.config_backtest: dict(): Backtest simulator itself parameters ''' log.debug('Reading commandline arguments') parser = argparse.ArgumentParser(description='Backtester module, the terrific financial simukation') parser.add_argument('-v', '--version', action='version', version='%(prog)s v0.8.1 Licence rien du tout', help='Print program version') parser.add_argument('-f', '--frequency', type=str, action='store', default='daily', required=False, help='(pandas) frequency in days betweend two quotes fetch') parser.add_argument('-a', '--algorithm', action='store', required=True, help='Trading algorithm to be used') parser.add_argument('-m', '--manager', action='store', required=True, help='Portfolio strategie to be used') parser.add_argument('-b', '--database', action='store', default='test', required=False, help='Table to considere in database') parser.add_argument('-i', '--initialcash', type=float, action='store', default=100000.0, required=False, help='Initial cash portfolio value') parser.add_argument('-t', '--tickers', action='store', default='random', required=False, help='target names to process') parser.add_argument('-s', '--start', action='store', default='1/1/2006', required=False, help='Start date of the backtester') parser.add_argument('-e', '--end', action='store', default='1/12/2010', required=False, help='Stop date of the backtester') parser.add_argument('-ex', '--exchange', action='store', default='', required=False, help='list of markets where trade, separated with a coma') parser.add_argument('-r', '--remote', action='store_true', help='Indicates if the program was ran manually or not') parser.add_argument('-l', '--live', action='store_true', help='makes the engine work in real-time !') parser.add_argument('-p', '--port', action='store', default=5570, required=False, help='Activates the diffusion of the universe on the network, on the port provided') args = parser.parse_args() #TODO Same as zipline in datasource, a mapping function with type and conversion function tuple #NOTE self.config_backtest = args.__dict__ # For generic use, further modules will need a dictionnary of parameters, not the namespace provided by argparse log.debug('Mapping arguments to backtest parameters dictionnary') self.config_backtest = {'algorithm' : args.algorithm, 'frequency' : args.frequency, 'manager' : args.manager, 'database' : args.database, 'tickers' : self._smart_tickers_select(args.tickers, exchange=args.exchange), 'start' : self._normalize_date_format(args.start), 'end' : self._normalize_date_format(args.end), 'live' : args.live, 'port' : args.port, 'exchange' : args.exchange, 'cash' : args.initialcash, 'remote' : args.remote} return self.config_backtest def get_strategie_configuration(self, *args, **kwargs): ''' Read localy or receive remotely strategie's parameters ''' if kwargs.get('remote', self.config_backtest['remote']): # Get configuration through ZMQ socket self.config_strategie = self._get_remote_data(port=self.config_backtest['port']) else: log.info('Reading strategie configuration from json files') self.config_strategie['manager'] = \ self._read_structured_file('managers.json', config_folder=True, select_field=self.config_backtest['manager']) self.config_strategie['algorithm'] = \ self._read_structured_file('algorithms.json', config_folder=True, select_field=self.config_backtest['algorithm']) # The manager can use the same socket during simulation to emit portfolio informations self.config_strategie['manager']['server'] = self.server log.info('Configuration is Done.') return self.config_strategie def _get_remote_data(self, port, host='localhost'): ''' Listen on backend ZMQ socket configuration data ''' # We need data from network, start the server assert not self.offline self.server.run(host=host, port=port) # In remote mode, client sends missing configuration through zmq socket log.info('Fetching backtest configuration from client') msg = self.server.receive(json=True) log.debug('Got it !') # Check message format and fields assert isinstance(msg, dict) assert 'algorithm' in msg assert 'manager' in msg return msg def _smart_tickers_select(self, tickers_description, exchange=''): ''' Take tickers string description and return an array of explicit and usuable symbols ''' # Informations are coma separated within the string tickers_description = tickers_description.split(',') # Useful way of stocks selection in order to test algorithm strength if tickers_description[0] == 'random': # Basic check: the second argument is the the number, integer, of stocks to pick up randomly assert len(tickers_description) == 2 assert int(tickers_description[1]) # Pick up stocks on specified (or not) market exchange tickers_description = self.datafeed.random_stocks(int(tickers_description[1]), exchange=exchange.split(',')) return tickers_description #TODO Handle in-day dates, with hours and minutes def _normalize_date_format(self, date): ''' Dates can be defined in many ways, but zipline use aware datetime objects only __________________________________________________ Parameters date: str String date like YYYY-MM-DD ''' assert isinstance(date, str) return pytz.utc.localize(datetime.strptime(date, '%Y-%m-%d'))
class QuandlSource(DataSource): """ Yields all events in event_list that match the given sid_filter. If no event_list is specified, generates an internal stream of events to filter. Returns all events if filter is None. Configuration options: sids : list of values representing simulated internal sids start : start date delta : timedelta between internal events filter : filter to remove the sids """ def __init__(self, data_descriptor, **kwargs): assert isinstance(data_descriptor['index'], pd.tseries.index.DatetimeIndex) self.data_descriptor = data_descriptor # Unpack config dictionary with default values. self.sids = kwargs.get('sids', data_descriptor['tickers']) self.start = kwargs.get('start', data_descriptor['index'][0]) self.end = kwargs.get('end', data_descriptor['index'][-1]) # Hash_value for downstream sorting. self.arg_string = hash_args(data_descriptor, **kwargs) self._raw_data = None self.feed = DataFeed() @property def mapping(self): mapping = { 'dt': (lambda x: x, 'dt'), 'sid': (lambda x: x, 'sid'), 'price': (float, 'Adjusted Close'), 'volume': (int, 'Volume'), } # Add additional fields. for field_name in self.data: if field_name in ['Adjusted Close', 'Volume', 'dt', 'sid']: continue mapping[field_name] = (lambda x: x, field_name) return mapping @property def instance_hash(self): return self.arg_string def _get(self): #TODO Test here for one value, make it later a panel assert len(self.sids) == 1 # Try to set quandl api key, stored in default config file self.feed._search_quandlkey() return self.feed.fetch_quandl(self.sids[0], start_date=self.start, end_date=self.end, returns='pandas') def raw_data_gen(self): self.data = self._get() sid = self.sids[0] for dt, series in self.data.iterrows(): event = { 'dt': dt, 'sid': sid, } for field_name, value in series.iteritems(): event[field_name] = value yield event @property def raw_data(self): if not self._raw_data: self._raw_data = self.raw_data_gen() return self._raw_data
import pandas.rpy.common as com from neuronquant.data.datafeed import DataFeed r = robjects.r svmforecast_lib = 'e1071.R' r('source("%s")' % svmforecast_lib) #quantmod = importr('quantmod') symbol = 'GOOG' history = 500 today = datetime.datetime.now() start_date = today - pd.datetools.Day(history) #r('require(quantmod)') #tt = r('get( getSymbols("{}", from="{}"))'.format(symbol, start_date.strftime(format='%Y-%m-%d'))) returns = DataFeed().quotes(symbol, start_date=start_date, end_date=today) #returns = data.pct_change().fillna(0.0) returns = returns.rename(columns={symbol: "Close"}) r_quotes = com.convert_to_r_dataframe(returns) import ipdb ipdb.set_trace() data_matrix = r('svmFeatures')(r_quotes) #rets = r('na.trim( ROC(Cl({}), type="discrete"))'.format('tt')) #data_matrix = r('svmFeatures')(tt) #data_df = pd.rpy.common.convert_robj(data_matrix)
import pandas.rpy.common as com from neuronquant.data.datafeed import DataFeed r = robjects.r svmforecast_lib = 'e1071.R' r('source("%s")' % svmforecast_lib) #quantmod = importr('quantmod') symbol = 'GOOG' history = 500 today = datetime.datetime.now() start_date = today - pd.datetools.Day(history) #r('require(quantmod)') #tt = r('get( getSymbols("{}", from="{}"))'.format(symbol, start_date.strftime(format='%Y-%m-%d'))) returns = DataFeed().quotes(symbol, start_date=start_date, end_date=today) #returns = data.pct_change().fillna(0.0) returns = returns.rename(columns={symbol: "Close"}) r_quotes = com.convert_to_r_dataframe(returns) import ipdb; ipdb.set_trace() data_matrix = r('svmFeatures')(r_quotes) #rets = r('na.trim( ROC(Cl({}), type="discrete"))'.format('tt')) #data_matrix = r('svmFeatures')(tt) #data_df = pd.rpy.common.convert_robj(data_matrix)
class PortfolioManager(object): ''' Manages portfolio during simulation, and stays aware of the situation through the update() method. It is configured through zmq message (manager field) or QuanTrade/config/managers.json file. User strategies call it with a dictionnnary of detected opportunities (i.e. buy or sell signals). Then the optimize function computes assets allocation, returning a dictionnary of symbols with their weigths or amount to reallocate. __________________________ _____________ signals {'google': 745.5} --> | | --> | | | trade_signals_handler() | | optimize() | orders {'google': 34} <-- |_________________________| <-- |____________| In addition, portfolio objects can be saved in database and reloaded later, and user on-the-fly orders are catched and executed in remote mode. Finally portfolios are connected to the server broker and, if requested, send state messages to client. This is abstract class, inheretid class will eventally overwrite optmize() to expose their own asset allocation strategy. ''' __metaclass__ = abc.ABCMeta #TODO Add in the constructor or setup parameters some general settings like maximum weights, positions, frequency,... #TODO Better to return 0 stocks to trade: remove the field #NOTE Regarding portfolio constraints: from a set of user-defined #parameters, a unnique set should be constructed. Then the solution provided #by optimize function would have to be a subset of it. (classic mathematical solution) #Finally it should be defined how to handle non-correct solutions def __init__(self, configuration): ''' Parameters configuration : dict Named parameters used either for general portfolio settings (server and constraints), and for user optimizer function ''' super(PortfolioManager, self).__init__() # Easy mysql access self.datafeed = DataFeed() # Zipline portfolio object, updated during simulation with self.date self.portfolio = None self.date = None # Portfolio owner, mainly used for database saving and client communication self.name = configuration.get('name', 'ChuckNorris') self.log = logbook.Logger('Manager::' + self.name) # Other parameters are used in user optimize() method self._optimizer_parameters = configuration # Make the manager talk to connected clients self.connected = configuration.get('connected', False) # Send android notifications when orders are processed # It's only possible with a running server self.android = configuration.get('android', False) & self.connected # Delete from database data with the same portfolio name if configuration.get('clean', True): self.log.info('Cleaning previous trades.') clean_previous_trades(self.name) # Run the server if the engine didn't, while it is asked for if 'server' in configuration and self.connected: # Getting server object instanciated anyway before (by Setup object) self.server = configuration.pop('server') # Web based dashboard where real time results are monitored #FIXME With dynamic generation, dashboad never exists at this point #self.dashboard = Dashboard(self.name) # In case user optimization would need to retrieve more data self.remote = Remote() @abc.abstractmethod def optimize(self): ''' Users should overwrite this method ''' pass def update(self, portfolio, date, metrics=None, save=False, widgets=False): ''' Actualizes the portfolio universe and if connected, sends it through the wires ________________________________ Parameters portfolio: zipline.portfolio ndict object storing portfolio values at the given date date: datetime.datetime Current date in zipline simulation save: boolean If true, save the portfolio in database under self.name key ''' # Make the manager aware of current simulation portfolio and date self.portfolio = portfolio self.date = date if save: self.save_portfolio(portfolio) if metrics is not None: save_metrics_snapshot(self.name, self.date, metrics) # Delete sold items and add new ones on dashboard #if widgets: #self.dashboard.update_position_widgets(self.portfolio.positions) # Send portfolio object to client if self.connected: #NOTE Something smarter ? # We need to translate zipline portfolio and position objects into json data (i.e. dict) packet_portfolio = to_dict(portfolio) for pos in packet_portfolio['positions']: packet_portfolio['positions'][pos] = to_dict(packet_portfolio['positions'][pos]) self.server.send(packet_portfolio, type ='portfolio', channel='dashboard') # Check user remote messages and return it return self.catch_messages() return dict() def trade_signals_handler(self, signals, extras={}): ''' Process buy and sell signals from the simulation ___________________________________________________________ Parameters signals: dict hold stocks of interest, format like {"google": 0.8, "apple": -0.2} If the value is negative -> sell signal, otherwize buy one Values are ranged between [-1 1] regarding signal confidence extras: whatever Object sent from algorithm for certain managers ___________________________________________________________ Return dict orderBook, like {"google": 34, "apple": -56} ''' self._optimizer_parameters['algo'] = extras orderBook = dict() # If value < 0, it's a sell signal on the key, else buy signal to_buy = dict(filter(lambda (sid, strength): strength > 0, signals.iteritems())) to_sell = dict(filter(lambda (sid, strength): strength < 0, signals.iteritems())) #to_buy = [t for t in signals if signals[t] > 0] #NOTE With this line we can't go short #to_sell = set(self.portfolio.positions.keys()).intersection( #[t for t in signals if signals[t] < 0]) if not to_buy and not to_sell: # Nothing to do return dict() # Compute the optimal portfolio allocation, using user defined function alloc, e_ret, e_risk = self.optimize(self.date, to_buy, to_sell, self._optimizer_parameters) #TODO Check about selling in available money and handle 250 stocks limit #TODO Handle max_* as well, ! already actif stocks ## Building orders for zipline #NOTE The follonwing in a separate function that could be used when catching message from user for t in alloc: ## Handle allocation returned as number of stocks to order if isinstance(alloc[t], int): orderBook[t] = alloc[t] ## Handle allocation returned as stock weights to order elif isinstance(alloc[t], float): # Sell orders if alloc[t] <= 0: orderBook[t] = int(alloc[t] * self.portfolio.positions[t].amount) ## Buy orders else: ## If we already trade this ticker, substract owned amount before computing number of stock to buy if self.portfolio.positions[t].amount: price = self.portfolio.positions[t].last_sale_price else: price = signals[t] orderBook[t] = (int(alloc[t] * self.portfolio.portfolio_value / price) - self.portfolio.positions[t].amount) if self.android and orderBook: # Alert user of the orders about to be processed # Ok... kind of fancy method ords = {'-1': 'sell', '1': 'buy'} msg = 'QuanTrade suggests you to ' msg += ', '.join(['{} {} stocks of {}' .format(ords[str(amount / abs(amount))], amount, ticker) for ticker, amount in orderBook.iteritems()]) self.server.send_to_android({'title': 'Portfolio manager notification', 'priority': 1, 'description': msg}) return orderBook def setup_strategie(self, parameters): ''' General parameters or user settings (maw_weigth, max_assets, max_frequency, commission cost) ________________________________________________________ Parameters parameters: dict Arbitrary values to change general constraints, or for user algorithm settings ''' assert isinstance(parameters, dict) for name, value in parameters.iteritems(): self._optimizer_parameters[name] = value def save_portfolio(self, portfolio): ''' Store in database given portfolio, for reuse later or further analysis puropose ____________________________________________ Parameters portfolio: zipline.protocol.Portfolio(1) ndict portfolio object to store ___________________________________________ ''' self.log.debug('Saving portfolio in database') self.datafeed.stock_db.save_portfolio(portfolio, self.name, self.date) def load_portfolio(self, name): ''' Load a complete portfolio object from database ______________________________________________ Parameters name: str(...) name used as primary key in db for the portfolio ______________________________________________ Return The portfolio with the given name if found, None otherwize ''' self.log.info('Loading portfolio {} from database'.foramt(name)) # Get the portfolio as a pandas Serie db_pf = self.datafeed.saved_portfolios(name) # Create empty Portfolio object to be filled portfolio = zp.Portfolio() # The function returns an empty dataframe if it didn't find a portfolio with id 'name' in db if len(db_pf): # Fill new portfolio data structure portfolio.capital_used = db_pf['Capital'] portfolio.starting_cash = db_pf['StartingCash'] portfolio.portfolio_value = db_pf['PortfolioValue'] portfolio.pnl = db_pf['PNL'] portfolio.returns = db_pf['Returns'] portfolio.cash = db_pf['Cash'] portfolio.start_date = db_pf['StartDate'] portfolio.positions = self._adapt_positions_type(db_pf['Positions']) portfolio.positions_value = db_pf['PositionsValue'] return portfolio def _adapt_positions_type(self, db_pos): ''' From array of sql Positions data model To Zipline Positions object ''' # Create empty Positions object to be filled positions = zp.Positions() for pos in db_pos: if pos.Ticker not in positions: positions[pos.Ticker] = zp.Position(pos.Ticker) position = positions[pos.Ticker] position.amount = pos.Amount position.cost_basis = pos.CostBasis position.last_sale_price = pos.LastSalePrice return positions def catch_messages(self, timeout=1): ''' Listen for user messages, process usual orders ''' msg = self.server.noblock_recv(timeout=timeout, json=True) #TODO msg is a command or an information, process it if msg: self.log.info('Got message from user: {}'.format(msg)) try: msg = json.loads(msg) except: msg = '' self.log.error('Unable to parse user message') return msg
class PortfolioManager: ''' Observes the trader universe and produces orders to be excuted within zipline Abstract method meant to be used by user to construct their portfolio optimizer ''' __metaclass__ = abc.ABCMeta #TODO Add in the constructor or setup parameters some general settings like maximum weights, positions, frequency,... def __init__(self, parameters): ''' Parameters parameters : dict(...) Named parameters used either for general portfolio settings (server and constraints), and for user optimizer function ''' super(PortfolioManager, self).__init__() self.log = Logger('Manager') self.datafeed = DataFeed() self.portfolio = None self.date = None self.name = parameters.get('name', 'Chuck Norris') self._optimizer_parameters = parameters self.connected = False #TODO Should send stuff anyway, and accept new connections while running self.connected = parameters.get('connected', False) # Run the server if the engine didn't while it is asked if 'server' in parameters: self.server = parameters.pop('server') if self.server.port is None and self.connected: self.log.info('Binding manager on default port...') self.server.run(host='127.0.0.1', port=5570) @abc.abstractmethod def optimize(self): ''' Users must overwrite this method ''' pass def update(self, portfolio, date): ''' Actualizes the portfolio universe and if connected, sends it through the wires ________________________________ Parameters portfolio: zipline.portfolio(1) ndict object storing portfolio values at the given date date: datetime.datetime(1) Current date in zipline simulation ''' self.portfolio = portfolio self.date = date if self.connected: self.server.send(to_dict(portfolio), type = 'portfolio', channel = 'dashboard') return self.catch_messages() return dict() def trade_signals_handler(self, signals): ''' Process buy and sell signals from backtester or live trader @param signals: dict holding stocks of interest, format like {"google": 567.89, "apple": -345.98} If the value is negative -> sell signal, otherwize buy one @return: dict orderBook, like {"google": 34, "apple": -56} ''' orderBook = dict() # If value < 0, it's a sell signal on the key, else buy signal to_buy = [t for t in signals if signals[t] > 0] to_sell = set(self.portfolio.positions.keys()).intersection([t for t in signals if signals[t] < 0]) if not to_buy and not to_sell: # Nothing to do return dict() # Compute the optimal portfolio allocation, using user defined function alloc, e_ret, e_risk = self.optimize(self.date, to_buy, to_sell, self._optimizer_parameters) #TODO Check about selling in available money and handle 250 stocks limit #TODO Handle max_* as well, ! already actif stocks ## Building orders for zipline #NOTE The follonwing in a separate function that could be used when catching message from user for t in alloc: ## Handle allocation returned as number of stocks to order if isinstance(alloc[t], int): orderBook[t] = alloc[t] ## Handle allocation returned as stock weights to order elif isinstance(alloc[t], float): # Sell orders if alloc[t] <= 0: orderBook[t] = int(alloc[t] * self.portfolio.positions[t].amount) ## Buy orders else: ## If we already trade this ticker, substract owned amount before computing number of stock to buy if self.portfolio.positions[t].amount: price = self.portfolio.positions[t].last_sale_price else: price = signals[t] orderBook[t] = (int(alloc[t] * self.portfolio.portfolio_value / price) - self.portfolio.positions[t].amount) return orderBook def setup_strategie(self, parameters): ''' General parameters or user ones setting (maw_weigth, max_assets, max_frequency, commission cost) ________________________________________________________ Parameters parameters: dict(...) Arbitrary values to change general constraints, or for user algorithm settings ''' for name, value in parameters.iteritems(): self._optimizer_parameters[name] = value #TODO Still need here this dict = f(ndict) def save_portfolio(self, portfolio): ''' Store in database given portfolio, for reuse later or further analysis puropose ____________________________________________ Parameters portfolio: zipline.protocol.Portfolio(1) ndict portfolio object to store ___________________________________________ ''' self.log.info('Saving portfolio in database') self.datafeed.stock_db.save_portfolio(portfolio, self.name, self.date) def load_portfolio(self, name): ''' Load a complete portfolio object from database ______________________________________________ Parameters name: str(...) name used as primary key in db for the portfolio ______________________________________________ Return The portfolio with the given name if found, None otherwize ''' self.log.info('Loading portfolio from database') ## Get the portfolio as a pandas Serie db_pf = self.datafeed.saved_portfolios(name) ## The function returns None if it didn't find a portfolio with id 'name' in db if db_pf is None: return None # Creating portfolio object portfolio = zp.Portfolio() portfolio.capital_used = db_pf['Capital'] portfolio.starting_cash = db_pf['StartingCash'] portfolio.portfolio_value = db_pf['PortfolioValue'] portfolio.pnl = db_pf['PNL'] portfolio.returns = db_pf['Returns'] portfolio.cash = db_pf['Cash'] portfolio.start_date = db_pf['StartDate'] portfolio.positions = self._adapt_positions_format(db_pf['Positions']) portfolio.positions_value = db_pf['PositionsValue'] return portfolio def _adapt_positions_type(self, db_pos): ''' From array of sql Positions data model To Zipline Positions object ''' positions = zp.Positions() for pos in db_pos: if pos.Ticker not in positions: positions[pos.Ticker] = zp.Position(pos.Ticker) position = positions[pos.Ticker] position.amount = pos.Amount position.cost_basis = pos.CostBasis position.last_sale_price = pos.LastSalePrice return positions def catch_messages(self, timeout=1): ''' Listen for user messages, process usual orders ''' msg = self.server.noblock_recv(timeout=timeout, json=True) #TODO msg is a command or an information, process it if msg: self.log.info('Got message from user: {}'.format(msg)) return msg
def __init__(self, data=None): #NOTE Allowing different data access ? #self.metrics = None #self.server = ZMQ_Dealer(id=self.__class__.__name__) self.datafeed = DataFeed()
class Simulation(object): ''' Take a trading strategie and evalute its results ''' def __init__(self, data=None): #NOTE Allowing different data access ? #self.metrics = None #self.server = ZMQ_Dealer(id=self.__class__.__name__) self.datafeed = DataFeed() #TODO For both, timezone configuration def configure(self, configuration): ''' Prepare dates, data, trading environment for simulation _______________________________________________________ Parameters configuration: dict() Structure with previously defined backtest behavior ''' data = self._configure_data(tickers=configuration['tickers'], start_time=configuration['start'], end_time=configuration['end'], freq=configuration['frequency'], exchange=configuration['exchange'], live=configuration['live']) context = self._configure_context(configuration['exchange']) return data, context #NOTE Should the data be loaded in zipline sourcedata class ? #FIXME data default not suitable for live mode def _configure_data(self, tickers, start_time=None, end_time=pd.datetime.now(pytz.utc), freq='daily', exchange='', live=False): if live: # Default end_date is now, suitable for live trading self.load_market_data = LiveBenchmark( end_time, frequency=freq).load_market_data #dates = pd.date_range(start_time, end_time, freq=freq) #NOTE A temporary hack to avoid zipline dirty modification #periods = end_time - start_time #dates = datautils.filter_market_hours(pd.date_range(pd.datetime.now(), periods=periods.days + 1, dates = datautils.filter_market_hours( pd.date_range(pytz.utc.localize(pd.datetime.now()), end_time, freq='1min'), #TODO ...hard coded, later: --frequency daily,3 exchange) import ipdb ipdb.set_trace() #dates = datautils.filter_market_hours(dates, exchange) if len(dates) == 0: log.warning('! Market closed.') sys.exit(0) #TODO Wrap it in a dataframe (always same return type) data = { 'stream_source': exchange, 'tickers': tickers, 'index': dates } else: # Use default zipline load_market_data, i.e. data from msgpack files in ~/.zipline/data/ self.load_market_data = None #TODO if start_time is None get default start_time in ~/.quantrade/default.json assert start_time # Fetch data from mysql database data = self.datafeed.quotes(tickers, start_date=start_time, end_date=end_time) if len(data) == 0: log.warning('Got nothing from database') data = pd.DataFrame() else: assert isinstance(data, pd.DataFrame) assert data.index.tzinfo return data def set_becnhmark_loader(self, load_function): self.load_market_data = load_function #TODO Use of futur localisation database criteria #TODO A list of markets to check if this one is in def _configure_context(self, exchange=''): ''' Setup from exchange traded on benchmarks used, location and method to load data market while simulating _______________________________________________ Parameters exchange: str Trading exchange market ''' # Environment configuration if exchange in datautils.Exchange: finance_context = TradingEnvironment( bm_symbol=datautils.Exchange[exchange]['index'], exchange_tz=datautils.Exchange[exchange]['timezone'], load=self.load_market_data) else: raise NotImplementedError( 'Because of computation limitation, trading worldwide not permitted currently' ) return finance_context def run(self, data, configuration, strategie, context): #___________________________________________________________________________ Running ________ log.info('\n-- Running backetester...\nUsing algorithm: {}\n'.format( configuration['algorithm'])) log.info('\n-- Using portfolio manager: {}\n'.format( configuration['manager'])) backtester = BacktesterEngine(configuration['algorithm'], configuration['manager'], strategie) #NOTE This method does not change anything #backtester.set_sources([DataLiveSource(data_tmp)]) #TODO A new command line parameter ? only minutely and daily (and hourly normally) Use filter parameter of datasource ? backtester.set_data_frequency(configuration['frequency']) # Running simulation with it with context: sim_params = create_simulation_parameters( capital_base=configuration['cash'], start=configuration['start'], end=configuration['end']) results, monthly_perfs = backtester.run(data, sim_params=sim_params) #return self.results return Analyze(results=results, metrics=monthly_perfs, datafeed=self.datafeed, configuration=configuration)