def __init__(self, config_dir='', offline=False):
'''
Parameters
configuration_folder: str
default is current location for configuration files
offline: boolean
Can force to switch off internal server
'''
#NOTE timezone as parameter
super(Setup, self).__init__()
self.configuration_folder = config_dir if config_dir else '/'.join((os.environ['QTRADE'], 'config'))
# Config data structures
self.config_backtest = dict()
self.config_strategie = dict()
self.config_environment = self._inspect_environment()
# Client for easy mysql database access
self.datafeed = DataFeed()
self.offline = offline
if not offline:
# It makes the entire simulator able to receive configuration and
# send informations to remote users and other processes like web frontend
self.server = network.ZMQ_Dealer(id=self.__class__.__name__)
def __init__(self, configuration):
#NOTE Allowing different data access ?
#self.metrics = None
#self.server = ZMQ_Dealer(id=self.__class__.__name__)
self.configuration = configuration
if 'quandl' in configuration['env']:
self.datafeed = DataFeed(configuration['env']['quandl'])
else:
self.datafeed = DataFeed()
def __init__(self, parameters):
PortfolioManager.__init__(self, parameters)
# Database access for symbols retrieving
self.feeds = DataFeed()
# R stuff: R functions file and rpy interface
self.r = robjects.r
portfolio_opt_file = '/'.join(
(os.environ['QTRADE'], 'neuronquant/ai/opt_utils.R'))
self.r('source("{}")'.format(portfolio_opt_file))
def __init__(self, country_code=None):
'''
Parameters
country_code: str
This information is used to setup International object
and get the right local conventions for lang,
dates and currencies. None will stand for 'fr'
'''
#self.locatioon = world.International(country_code)
self.datafeed = DataFeed()
def __init__(self, data_descriptor, **kwargs):
assert isinstance(data_descriptor['index'],
pd.tseries.index.DatetimeIndex)
self.data_descriptor = data_descriptor
# Unpack config dictionary with default values.
self.sids = kwargs.get('sids', data_descriptor['tickers'])
self.start = kwargs.get('start', data_descriptor['index'][0])
self.end = kwargs.get('end', data_descriptor['index'][-1])
# Hash_value for downstream sorting.
self.arg_string = hash_args(data_descriptor, **kwargs)
self._raw_data = None
self.feed = DataFeed()
def __init__(self, parameters):
'''
Parameters
parameters : dict(...)
Named parameters used either for general portfolio settings
(server and constraints), and for user optimizer function
'''
super(PortfolioManager, self).__init__()
self.log = Logger('Manager')
self.datafeed = DataFeed()
self.portfolio = None
self.date = None
self.name = parameters.get('name', 'Chuck Norris')
self._optimizer_parameters = parameters
self.connected = False
#TODO Should send stuff anyway, and accept new connections while running
self.connected = parameters.get('connected', False)
# Run the server if the engine didn't while it is asked
if 'server' in parameters:
self.server = parameters.pop('server')
if self.server.port is None and self.connected:
self.log.info('Binding manager on default port...')
self.server.run(host='127.0.0.1', port=5570)
class Remote(object):
'''
Entry point to remote access to data
Mainly offer a simpler interface and a dataaccess harmonisation
Plus symbol check and complete, timezone, currency conversion
Plus reindexage ?
'''
def __init__(self, country_code=None):
'''
Parameters
country_code: str
This information is used to setup International object
and get the right local conventions for lang, dates and currencies
None will stand for 'fr'
'''
#self.locatioon = world.International(country_code)
self.datafeed = DataFeed()
pass
def fetch_equities_snapshot(self, *args, **kwargs):
'''
Use Yahoo and google finance service to fetch
current infromations about given equitiy names
______________________________________________
Parameters
args: tuple
company names to consider
kwargs['level']: int
Quantity of information level
______________________________________________
Return
snapshot: pandas.DataFrame
with names as columns and informations as index
'''
equities = args
if not equities:
equities = kwargs.get('symbols', [])
level = kwargs.get('level', 0)
symbols = [self.datafeed.guess_name(equity) for equity in equities]
if not level:
# default level, the lightest
snapshot = snapshot_yahoo_pandas(symbols)
elif level == 1:
snapshot = snapshot_google_light(symbols)
elif level == 2:
snapshot = snapshot_google_heavy(symbols)
else:
raise ValueError('Invalid level of information requested')
# Give columns back equities names requested
snapshot.columns = equities
return snapshot
def _localize_data(self, data):
'''
Inspect data and applie localisation
on date and monnaie values
'''
assert isinstance(data, pd.Dataframe) or isinstance(data, dict)
class Remote(object):
'''
Entry point to remote access to data
Mainly offer a simpler interface and a dataaccess harmonisation
Plus symbol check and complete, timezone, currency conversion
Plus reindexage ?
'''
def __init__(self, country_code=None):
'''
Parameters
country_code: str
This information is used to setup International object
and get the right local conventions for lang, dates and currencies
None will stand for 'fr'
'''
#self.locatioon = world.International(country_code)
self.datafeed = DataFeed()
pass
def fetch_equities_snapshot(self, *args, **kwargs):
'''
Use Yahoo and google finance service to fetch
current infromations about given equitiy names
______________________________________________
Parameters
args: tuple
company names to consider
kwargs['level']: int
Quantity of information level
______________________________________________
Return
snapshot: pandas.DataFrame
with names as columns and informations as index
'''
equities = args
if not equities:
equities = kwargs.get('symbols', [])
level = kwargs.get('level', 0)
symbols = [self.datafeed.guess_name(equity) for equity in equities]
if not level:
# default level, the lightest
snapshot = snapshot_yahoo_pandas(symbols)
elif level == 1:
snapshot = snapshot_google_light(symbols)
elif level == 2:
snapshot = snapshot_google_heavy(symbols)
else:
raise ValueError('Invalid level of information requested')
# Give columns back equities names requested
snapshot.columns = equities
return snapshot
def _localize_data(self, data):
'''
Inspect data and applie localisation
on date and monnaie values
'''
assert isinstance(data, pd.Dataframe) or isinstance(data, dict)
class OptimalFrontier(PortfolioManager):
'''
Compute with R the efficient frontier and pick up the optimize point on it
'''
def __init__(self, parameters):
PortfolioManager.__init__(self, parameters)
# Database access for symbols retrieving
self.feeds = DataFeed()
# R stuff: R functions file and rpy interface
self.r = robjects.r
portfolio_opt_file = '/'.join((os.environ['QTRADE'], 'neuronquant/ai/opt_utils.R'))
self.r('source("{}")'.format(portfolio_opt_file))
def optimize(self, date, to_buy, to_sell, parameters):
symbols = []
allocations = dict()
# Considere only portfolio positions + future positions - positions about to be sold
positions = set([t for t in self.portfolio.positions.keys() if self.portfolio.positions[t].amount]).union(to_buy).difference(to_sell)
if not positions and to_sell:
for t in to_sell:
allocations[t] = - parameters.get('perc_sell', 1.0)
return allocations, 0, 1
try:
assert(positions)
except:
self.log.error('** No positions determined')
if len(positions) == 1:
return {positions.pop(): parameters.get('max_weigths', 0.2)}, 0, 1
for p in positions:
symbols.append(self.feeds.guess_name(p).lower())
loopback = parameters.get('loopback', 50)
source = parameters.get('source', 'yahoo')
start = pd.datetime.strftime(date - pd.datetools.BDay(loopback), format = '%Y-%m-%d')
date = pd.datetime.strftime(date, format = '%Y-%m-%d')
r_symbols = self.r('c("{}")'.format('", "'.join(symbols)))
r_names = self.r('c("{}.Return")'.format('.Return", "'.join(symbols)))
data = self.r('importSeries')(r_symbols, start, date, source=source)
frontier = self.r('getEfficientFrontier')(data, r_names, points = 500, Debug = False, graph = False)
if not frontier:
self.log.warning('No optimal frontier found')
return dict(), None, None
try:
mp = self.r('marketPortfolio')(frontier, 0.02, Debug = False, graph = False)
except:
self.log.error('** Error running R optimizer')
return dict(), None, None
self.log.debug('Allocation: {}'.format(mp))
#FIXME Some key errors survive so far
for s, t in zip(symbols, positions):
allocations[t] = round(mp.rx('.'.join((s, 'Return')))[0][0], 2)
er = round(mp.rx('er')[0][0], 2)
eStd = round(mp.rx('eStd')[0][0], 2)
self.log.debug('Allocation: {}\nWith expected return: {}\tand expected risk: {}'.format(allocations, er, eStd))
return allocations, er, eStd
def __init__(self, parameters):
PortfolioManager.__init__(self, parameters)
# Database access for symbols retrieving
self.feeds = DataFeed()
# R stuff: R functions file and rpy interface
self.r = robjects.r
portfolio_opt_file = '/'.join((os.environ['QTRADE'], 'neuronquant/ai/opt_utils.R'))
self.r('source("{}")'.format(portfolio_opt_file))
def __init__(self, data=None, flavor='mysql', lvl='debug'):
#NOTE Allowing different data access ?
self.data = data
if not data:
self.feeds = DataFeed()
self.backtest_cfg = None
self.algo_cfg = None
self.manager_cfg = None
self.monthly_perfs = None
self.server = ZMQ_Dealer(id=self.__class__.__name__)
def __init__(self, data, **kwargs):
assert isinstance(data['index'], pd.tseries.index.DatetimeIndex)
self.data = data
# Unpack config dictionary with default values.
self.sids = kwargs.get('sids', data['tickers'])
self.start = kwargs.get('start', data['index'][0])
self.end = kwargs.get('end', data['index'][-1])
self.fake_index = pd.date_range(self.start,
self.end,
freq=pd.datetools.BDay())
# Hash_value for downstream sorting.
self.arg_string = hash_args(data, **kwargs)
self._raw_data = None
self.remote = Fetcher()
self.feed = DataFeed()
def __init__(self, data_descriptor, **kwargs):
assert isinstance(data_descriptor['index'], pd.tseries.index.DatetimeIndex)
self.data_descriptor = data_descriptor
# Unpack config dictionary with default values.
self.sids = kwargs.get('sids', data_descriptor['tickers'])
self.start = kwargs.get('start', data_descriptor['index'][0])
self.end = kwargs.get('end', data_descriptor['index'][-1])
# Hash_value for downstream sorting.
self.arg_string = hash_args(data_descriptor, **kwargs)
self._raw_data = None
self.feed = DataFeed()
def __init__(self, *args, **kwargs):
super(Analyze, self).__init__()
# MySQL Database client
self.datafeed = kwargs.pop('datafeed') if 'datafeed' in kwargs else DataFeed()
# R analysis file only need portfolio returns
self.returns = kwargs.pop('returns') if 'returns' in kwargs else None
# Final risk measurments as returned by the backtester
self.results = kwargs.pop('results') if 'results' in kwargs else None
# Simulation rolling performance
self.metrics = kwargs.pop('metrics') if 'metrics' in kwargs else None
# You better should know what was simulation's parameters
self.configuration = kwargs.pop('configuration') if 'configuration' in kwargs else None
def __init__(self, configuration):
'''
Parameters
configuration : dict
Named parameters used either for general portfolio settings
(server and constraints), and for user optimizer function
'''
super(PortfolioManager, self).__init__()
# Easy mysql access
self.datafeed = DataFeed()
# Zipline portfolio object, updated during simulation with self.date
self.portfolio = None
self.date = None
# Portfolio owner, mainly used for database saving and client communication
self.name = configuration.get('name', 'ChuckNorris')
self.log = logbook.Logger('Manager::' + self.name)
# Other parameters are used in user optimize() method
self._optimizer_parameters = configuration
# Make the manager talk to connected clients
self.connected = configuration.get('connected', False)
# Send android notifications when orders are processed
# It's only possible with a running server
self.android = configuration.get('android', False) & self.connected
# Delete from database data with the same portfolio name
if configuration.get('clean', True):
self.log.info('Cleaning previous trades.')
clean_previous_trades(self.name)
# Run the server if the engine didn't, while it is asked for
if 'server' in configuration and self.connected:
# Getting server object instanciated anyway before (by Setup object)
self.server = configuration.pop('server')
# Web based dashboard where real time results are monitored
#FIXME With dynamic generation, dashboad never exists at this point
#self.dashboard = Dashboard(self.name)
# In case user optimization would need to retrieve more data
self.remote = Remote()
def __init__(self, data, **kwargs):
assert isinstance(data['index'], pd.tseries.index.DatetimeIndex)
self.data = data
# Unpack config dictionary with default values.
self.sids = kwargs.get('sids', data['tickers'])
self.start = kwargs.get('start', data['index'][0])
self.end = kwargs.get('end', data['index'][-1])
self.fake_index = pd.date_range(self.start, self.end, freq=pd.datetools.BDay())
# Hash_value for downstream sorting.
self.arg_string = hash_args(data, **kwargs)
self._raw_data = None
self.remote = Fetcher()
self.feed = DataFeed()
def smart_tickers_select(tickers_description, exchange=''):
'''
Take tickers string description and return
an array of explicit and usuable symbols
'''
# Informations are coma separated within the string
tickers_description = tickers_description.split(',')
# Useful way of stocks selection in order to test algorithm strength
if tickers_description[0] == 'random':
# Basic check: the second argument is the the number, integer, of stocks to pick up randomly
assert len(tickers_description) == 2
assert int(tickers_description[1])
# Pick up stocks on specified (or not) market exchange
tickers_description = DataFeed().random_stocks(
int(tickers_description[1]), exchange=exchange.split(','))
return tickers_description
class OptimalFrontier(PortfolioManager):
'''
Compute with R the efficient frontier and pick up the optimize point on it
'''
def __init__(self, parameters):
PortfolioManager.__init__(self, parameters)
# Database access for symbols retrieving
self.feeds = DataFeed()
# R stuff: R functions file and rpy interface
self.r = robjects.r
portfolio_opt_file = '/'.join(
(os.environ['QTRADE'], 'neuronquant/ai/opt_utils.R'))
self.r('source("{}")'.format(portfolio_opt_file))
def optimize(self, date, to_buy, to_sell, parameters):
symbols = []
allocations = dict()
# Considere only portfolio positions + future positions - positions about to be sold
positions = set([
t for t in self.portfolio.positions.keys()
if self.portfolio.positions[t].amount
]).union(to_buy).difference(to_sell)
if not positions and to_sell:
for t in to_sell:
allocations[t] = -parameters.get('perc_sell', 1.0)
return allocations, 0, 1
try:
assert (positions)
except:
self.log.error('** No positions determined')
if len(positions) == 1:
return {positions.pop(): parameters.get('max_weigths', 0.2)}, 0, 1
for p in positions:
symbols.append(self.feeds.guess_name(p).lower())
loopback = parameters.get('loopback', 50)
source = parameters.get('source', 'yahoo')
start = pd.datetime.strftime(date - pd.datetools.BDay(loopback),
format='%Y-%m-%d')
date = pd.datetime.strftime(date, format='%Y-%m-%d')
r_symbols = self.r('c("{}")'.format('", "'.join(symbols)))
r_names = self.r('c("{}.Return")'.format('.Return", "'.join(symbols)))
data = self.r('importSeries')(r_symbols, start, date, source=source)
frontier = self.r('getEfficientFrontier')(data,
r_names,
points=500,
Debug=False,
graph=False)
if not frontier:
self.log.warning('No optimal frontier found')
return dict(), None, None
try:
mp = self.r('marketPortfolio')(frontier,
0.02,
Debug=False,
graph=False)
except:
self.log.error('** Error running R optimizer')
return dict(), None, None
self.log.debug('Allocation: {}'.format(mp))
#FIXME Some key errors survive so far
for s, t in zip(symbols, positions):
allocations[t] = round(mp.rx('.'.join((s, 'Return')))[0][0], 2)
er = round(mp.rx('er')[0][0], 2)
eStd = round(mp.rx('eStd')[0][0], 2)
self.log.debug(
'Allocation: {}\nWith expected return: {}\tand expected risk: {}'.
format(allocations, er, eStd))
return allocations, er, eStd
class DataLiveSource(DataSource):
"""
Yields all events in event_list that match the given sid_filter.
If no event_list is specified, generates an internal stream of events
to filter. Returns all events if filter is None.
Configuration options:
sids : list of values representing simulated internal sids
start : start date
delta : timedelta between internal events
filter : filter to remove the sids
"""
def __init__(self, data, **kwargs):
assert isinstance(data['index'], pd.tseries.index.DatetimeIndex)
self.data = data
# Unpack config dictionary with default values.
self.sids = kwargs.get('sids', data['tickers'])
self.start = kwargs.get('start', data['index'][0])
self.end = kwargs.get('end', data['index'][-1])
self.fake_index = pd.date_range(self.start,
self.end,
freq=pd.datetools.BDay())
# Hash_value for downstream sorting.
self.arg_string = hash_args(data, **kwargs)
self._raw_data = None
self.remote = Fetcher()
self.feed = DataFeed()
@property
def mapping(self):
return {
'dt': (lambda x: x, 'dt'),
'sid': (lambda x: x, 'sid'),
'price': (float, 'price'),
'currency': (str, 'currency'),
'perc_change': (float, 'perc_change'),
'volume': (int, 'volume'),
}
@property
def instance_hash(self):
return self.arg_string
def raw_data_gen(self):
current_dt = datetime.datetime.now()
index = self.data['index']
selector = (index.day > current_dt.day) \
| ((index.day == current_dt.day) & (index.hour > current_dt.hour)) \
| ((index.day == current_dt.day) & (index.hour == current_dt.hour) & (index.minute >= current_dt.minute))
#NOTE Not an equal size issue ?
for fake_dt, dt in zip(self.fake_index, index[selector]):
while (current_dt.minute != dt.minute) or (current_dt.hour !=
dt.hour):
time.sleep(15)
current_dt = datetime.datetime.now()
print('Waiting {} / {}'.format(current_dt, dt))
#for fake_dt, dt in zip(self.fake_index, self.data['index']):
for sid in self.data['tickers']:
if sid in self.sids:
symbol = self.feed.guess_name(sid).lower()
#FIXME Erros because no markets specifie, use light=True and add market
#snapshot = self.remote.get_stock_snapshot(symbol, light=False)
snapshot = self.remote.get_stock_snapshot(symbol,
light=True)
import ipdb
ipdb.set_trace()
log.debug('Data available:\n{}'.format(
json.dumps(snapshot,
sort_keys=True,
indent=4,
separators=(',', ': '))))
if not snapshot:
log.error(
'** No data snapshot available, maybe stopped by google ?'
)
sys.exit(2)
event = {
'dt': fake_dt,
'trade_time': dt,
'sid': sid,
'price': float(snapshot[symbol]['last']),
'currency': snapshot[symbol]['currency'],
'perc_change': float(snapshot[symbol]['perc_change']),
'volume': int(snapshot[symbol]['volume']),
}
yield event
@property
def raw_data(self):
if not self._raw_data:
self._raw_data = self.raw_data_gen()
return self._raw_data
class DBPriceSource(DataSource):
"""
Yields all events in event_list that match the given sid_filter.
If no event_list is specified, generates an internal stream of events
to filter. Returns all events if filter is None.
Configuration options:
sids : list of values representing simulated internal sids
start : start date
delta : timedelta between internal events
filter : filter to remove the sids
"""
def __init__(self, data_descriptor, **kwargs):
assert isinstance(data_descriptor['index'], pd.tseries.index.DatetimeIndex)
self.data_descriptor = data_descriptor
# Unpack config dictionary with default values.
self.sids = kwargs.get('sids', data_descriptor['tickers'])
self.start = kwargs.get('start', data_descriptor['index'][0])
self.end = kwargs.get('end', data_descriptor['index'][-1])
# Hash_value for downstream sorting.
self.arg_string = hash_args(data_descriptor, **kwargs)
self._raw_data = None
self.feed = DataFeed()
@property
def mapping(self):
return {
'dt': (lambda x: x, 'dt'),
'sid': (lambda x: x, 'sid'),
'price': (float, 'price'),
'volume': (int, 'volume'),
}
@property
def instance_hash(self):
return self.arg_string
def _get(self):
return self.feed.quotes(self.sids,
start_date=self.start, end_date=self.end)
def raw_data_gen(self):
self.data = self._get()
for dt, series in self.data.iterrows():
for sid, price in series.iterkv():
if sid in self.sids:
event = {
'dt': dt,
'sid': sid,
'price': price,
'volume': 1000,
}
yield event
@property
def raw_data(self):
if not self._raw_data:
self._raw_data = self.raw_data_gen()
return self._raw_data
class DBPriceSource(DataSource):
"""
Yields all events in event_list that match the given sid_filter.
If no event_list is specified, generates an internal stream of events
to filter. Returns all events if filter is None.
Configuration options:
sids : list of values representing simulated internal sids
start : start date
delta : timedelta between internal events
filter : filter to remove the sids
"""
def __init__(self, data_descriptor, **kwargs):
assert isinstance(data_descriptor['index'],
pd.tseries.index.DatetimeIndex)
self.data_descriptor = data_descriptor
# Unpack config dictionary with default values.
self.sids = kwargs.get('sids', data_descriptor['tickers'])
self.start = kwargs.get('start', data_descriptor['index'][0])
self.end = kwargs.get('end', data_descriptor['index'][-1])
# Hash_value for downstream sorting.
self.arg_string = hash_args(data_descriptor, **kwargs)
self._raw_data = None
self.feed = DataFeed()
@property
def mapping(self):
return {
'dt': (lambda x: x, 'dt'),
'sid': (lambda x: x, 'sid'),
'price': (float, 'price'),
'volume': (int, 'volume'),
}
@property
def instance_hash(self):
return self.arg_string
def _get(self):
return self.feed.quotes(self.sids,
start_date=self.start,
end_date=self.end)
def raw_data_gen(self):
self.data = self._get()
for dt, series in self.data.iterrows():
for sid, price in series.iterkv():
if sid in self.sids:
event = {
'dt': dt,
'sid': sid,
'price': price,
'volume': 1000,
}
yield event
@property
def raw_data(self):
if not self._raw_data:
self._raw_data = self.raw_data_gen()
return self._raw_data
class DataLiveSource(DataSource):
"""
Yields all events in event_list that match the given sid_filter.
If no event_list is specified, generates an internal stream of events
to filter. Returns all events if filter is None.
Configuration options:
sids : list of values representing simulated internal sids
start : start date
delta : timedelta between internal events
filter : filter to remove the sids
"""
def __init__(self, data, **kwargs):
assert isinstance(data['index'], pd.tseries.index.DatetimeIndex)
self.data = data
# Unpack config dictionary with default values.
self.sids = kwargs.get('sids', data['tickers'])
self.start = kwargs.get('start', data['index'][0])
self.end = kwargs.get('end', data['index'][-1])
self.fake_index = pd.date_range(self.start, self.end, freq=pd.datetools.BDay())
# Hash_value for downstream sorting.
self.arg_string = hash_args(data, **kwargs)
self._raw_data = None
self.remote = Fetcher()
self.feed = DataFeed()
@property
def mapping(self):
return {
'dt': (lambda x: x, 'dt'),
'sid': (lambda x: x, 'sid'),
'price': (float, 'price'),
'currency': (str, 'currency'),
'perc_change': (float, 'perc_change'),
'volume': (int, 'volume'),
}
@property
def instance_hash(self):
return self.arg_string
def raw_data_gen(self):
current_dt = datetime.datetime.now()
index = self.data['index']
selector = (index.day > current_dt.day) \
| ((index.day == current_dt.day) & (index.hour > current_dt.hour)) \
| ((index.day == current_dt.day) & (index.hour == current_dt.hour) & (index.minute >= current_dt.minute))
#NOTE Not an equal size issue ?
for fake_dt, dt in zip(self.fake_index, index[selector]):
while (current_dt.minute != dt.minute) or (current_dt.hour != dt.hour) :
time.sleep(15)
current_dt = datetime.datetime.now()
print('Waiting {} / {}'.format(current_dt, dt))
#for fake_dt, dt in zip(self.fake_index, self.data['index']):
for sid in self.data['tickers']:
if sid in self.sids:
symbol = self.feed.guess_name(sid).lower()
#FIXME Erros because no markets specifie, use light=True and add market
#snapshot = self.remote.get_stock_snapshot(symbol, light=False)
snapshot = self.remote.get_stock_snapshot(symbol, light=True)
import ipdb; ipdb.set_trace()
log.debug('Data available:\n{}'.format(json.dumps(snapshot,
sort_keys=True, indent=4, separators=(',', ': '))))
if not snapshot:
log.error('** No data snapshot available, maybe stopped by google ?')
sys.exit(2)
event = {
'dt': fake_dt,
'trade_time': dt,
'sid': sid,
'price': float(snapshot[symbol]['last']),
'currency': snapshot[symbol]['currency'],
'perc_change': float(snapshot[symbol]['perc_change']),
'volume': int(snapshot[symbol]['volume']),
}
yield event
@property
def raw_data(self):
if not self._raw_data:
self._raw_data = self.raw_data_gen()
return self._raw_data
class Simulation(object):
''' Take a trading strategie and evalute its results '''
def __init__(self, data=None, flavor='mysql', lvl='debug'):
#NOTE Allowing different data access ?
self.data = data
if not data:
self.feeds = DataFeed()
self.backtest_cfg = None
self.algo_cfg = None
self.manager_cfg = None
self.monthly_perfs = None
self.server = ZMQ_Dealer(id=self.__class__.__name__)
def configure(self, bt_cfg=None, a_cfg=None, m_cfg=None):
''' Reads and provides a clean configuration for the simulation '''
if not bt_cfg:
parser = argparse.ArgumentParser(description='Backtester module, the terrific financial simukation')
parser.add_argument('-v', '--version', action='version', version='%(prog)s v0.8.1 Licence rien du tout', help='Print program version')
parser.add_argument('-d', '--delta', type=str, action='store', default='D', required=False, help='Delta in days betweend two quotes fetch')
parser.add_argument('-a', '--algorithm', action='store', required=True, help='Trading algorithm to be used')
parser.add_argument('-m', '--manager', action='store', required=True, help='Portfolio strategie to be used')
parser.add_argument('-b', '--database', action='store', default='test', required=False, help='Table to considere in database')
parser.add_argument('-t', '--tickers', action='store', required=True, help='target names to process')
parser.add_argument('-s', '--start', action='store', default='1/1/2006', required=False, help='Start date of the backtester')
parser.add_argument('-e', '--end', action='store', default='1/12/2010', required=False, help='Stop date of the backtester')
parser.add_argument('-r', '--remote', action='store_true', help='Indicates if the program was ran manually or not')
parser.add_argument('-l', '--live', action='store_true', help='makes the engine work in real-time !')
parser.add_argument('-p', '--port', action='store', default=5570, required=False, help='Activates the diffusion of the universe on the network, on the port provided')
args = parser.parse_args()
self.backtest_cfg = {'algorithm' : args.algorithm,
'delta' : args.delta,
'manager' : args.manager,
'database' : args.database,
'tickers' : args.tickers.split(','),
'start' : args.start,
'end' : args.end,
'live' : args.live,
'port' : args.port,
'remote' : args.remote}
else:
self.backtest_cfg = bt_cfg
if isinstance(self.backtest_cfg['start'], str) and isinstance(self.backtest_cfg['end'], str):
###############test = pd.datetime.strptime('2012-02-01:14:00', '%Y-%m-%d:%H:%M')
self.backtest_cfg['start'] = pytz.utc.localize(pd.datetime.strptime(self.backtest_cfg['start'], '%Y-%m-%d'))
self.backtest_cfg['end'] = pytz.utc.localize(pd.datetime.strptime(self.backtest_cfg['end'], '%Y-%m-%d'))
elif isinstance(self.backtest_cfg['start'], pd.datetime) and isinstance(self.backtest_cfg['end'], pd.datetime):
if not self.backtest_cfg['start'].tzinfo:
self.backtest_cfg['start'] = pytz.utc.localize(self.backtest_cfg['start'])
if not self.backtest_cfg['end'].tzinfo:
self.backtest_cfg['end'] = pytz.utc.localize(self.backtest_cfg['end'])
else:
raise NotImplementedError()
#if self.backtest_cfg['interactive']:
self.read_config(remote=self.backtest_cfg['remote'], manager=m_cfg, algorithm=a_cfg)
#else:
#self.server.run_forever(port=self.backtest_cfg['port'], on_recv=self.read_config)
return self.backtest_cfg
def read_config(self, *args, **kwargs):
self.manager_cfg = kwargs.get('manager', None)
self.algo_cfg = kwargs.get('algorithm', None)
if kwargs.get('remote', False):
self.server.run(host='127.0.0.1', port=self.backtest_cfg['port'])
# In remote mode, client sends missing configuration through zmq socket
if (not self.manager_cfg) or (not self.algo_cfg):
log.info('Fetching backtest configuration from client')
msg = self.server.receive(json=True)
log.debug('Got it !')
# Set simulation parameters with it
assert isinstance(msg, dict)
if self.manager_cfg is None:
self.manager_cfg = msg['manager']
if self.algo_cfg is None:
self.algo_cfg = msg['algorithm']
else:
# Reading configuration from json files
config_dir = '/'.join((os.environ['QTRADE'], 'config/'))
try:
# Files store many algos and manager parameters,
# use backtest configuration to pick up the right one
if self.manager_cfg is None:
self.manager_cfg = json.load(open('{}/managers.cfg'.format(config_dir), 'r'))[self.backtest_cfg['manager']]
if self.algo_cfg is None:
self.algo_cfg = json.load(open('{}/algos.cfg'.format(config_dir), 'r'))[self.backtest_cfg['algorithm']]
except:
log.error('** loading json configuration.')
sys.exit(1)
# The manager can use the same socket during simulation to emit portfolio informations
self.manager_cfg['server'] = self.server
log.info('Configuration is Done.')
def run_backtest(self):
# No configuration, no backtest man
if self.backtest_cfg is None or self.algo_cfg is None or self.manager_cfg is None:
log.error('** Backtester not configured properly')
return 1
#'''_____________________________________________________________________ Parameters ________
if self.backtest_cfg['tickers'][0] == 'random':
assert(len(self.backtest_cfg['tickers']) == 2)
assert(int(self.backtest_cfg['tickers'][1]))
self.backtest_cfg['tickers'] = self.feeds.random_stocks(int(self.backtest_cfg['tickers'][1]))
if self.backtest_cfg['live']:
#dates = pd.date_range(self.backtest_cfg['start'], self.backtest_cfg['end'], freq=self.backtest_cfg['delta'])
#NOTE A temporary hack to avoid zipline dirty modification
periods = self.backtest_cfg['end'] - self.backtest_cfg['start']
dates = pd.date_range(pd.datetime.now(), periods=periods.days + 1, freq=self.backtest_cfg['delta'])
'''
if dates.freq > pd.datetools.Day():
#fr_selector = ((dates.hour > 8) & (dates.hour < 17)) | ((dates.hour == 17) & (dates.minute < 31))
us_selector = ((dates.hour > 15) & (dates.hour < 22)) | ((dates.hour == 15) & (dates.minute > 31))
dates = dates[us_selector]
if not dates:
log.warning('! Market closed.')
sys.exit(0)
'''
data = {'tickers': self.backtest_cfg['tickers'], 'index': dates.tz_localize(pytz.utc)}
else:
data = self.feeds.quotes(self.backtest_cfg['tickers'],
start_date = self.backtest_cfg['start'],
end_date = self.backtest_cfg['end'])
assert isinstance(data, pd.DataFrame)
assert data.index.tzinfo
#___________________________________________________________________________ Running ________
log.info('\n-- Running backetester...\nUsing algorithm: {}\n'.format(self.backtest_cfg['algorithm']))
log.info('\n-- Using portfolio manager: {}\n'.format(self.backtest_cfg['manager']))
backtester = BacktesterEngine(self.backtest_cfg['algorithm'],
self.backtest_cfg['manager'],
self.algo_cfg,
self.manager_cfg)
self.results, self.monthly_perfs = backtester.run(data)
#self.backtest_cfg['start'],
#self.backtest_cfg['end'])
return self.results
def rolling_performances(self, timestamp='one_month', save=False, db_id=None):
''' Filters self.perfs and, if asked, save it to database '''
#TODO Study the impact of month choice
#TODO Check timestamp in an enumeration
#NOTE Script args define default database table name (test), make it consistent
if db_id is None:
db_id = self.backtest_cfg['algorithm'] + pd.datetime.strftime(pd.datetime.now(), format='%Y%m%d')
if self.monthly_perfs:
perfs = dict()
length = range(len(self.monthly_perfs[timestamp]))
index = self._get_index(self.monthly_perfs[timestamp])
perfs['Name'] = np.array([db_id] * len(self.monthly_perfs[timestamp]))
#perfs['Period'] = np.array([self.monthly_perfs[timestamp][i]['period_label'] for i in length])
perfs['Period'] = np.array([pd.datetime.date(date) for date in index])
perfs['Sharpe.Ratio'] = np.array([self.monthly_perfs[timestamp][i]['sharpe'] for i in length])
perfs['Returns'] = np.array([self.monthly_perfs[timestamp][i]['algorithm_period_return'] for i in length])
perfs['Max.Drawdown'] = np.array([self.monthly_perfs[timestamp][i]['max_drawdown'] for i in length])
perfs['Volatility'] = np.array([self.monthly_perfs[timestamp][i]['algo_volatility'] for i in length])
perfs['Beta'] = np.array([self.monthly_perfs[timestamp][i]['beta'] for i in length])
perfs['Alpha'] = np.array([self.monthly_perfs[timestamp][i]['alpha'] for i in length])
perfs['Excess.Returns'] = np.array([self.monthly_perfs[timestamp][i]['excess_return'] for i in length])
perfs['Benchmark.Returns'] = np.array([self.monthly_perfs[timestamp][i]['benchmark_period_return'] for i in length])
perfs['Benchmark.Volatility'] = np.array([self.monthly_perfs[timestamp][i]['benchmark_volatility'] for i in length])
perfs['Treasury.Returns'] = np.array([self.monthly_perfs[timestamp][i]['treasury_period_return'] for i in length])
else:
#TODO Get it from DB if it exists
raise NotImplementedError()
try:
data = pd.DataFrame(perfs, index=index)
except:
import ipdb; ipdb.set_trace()
if save:
self.feeds.stock_db.save_metrics(data)
return data
def overall_metrics(self, timestamp='one_month', save=False, db_id=None):
''' Use zipline results to compute some performance indicators and store it in database '''
perfs = dict()
metrics = self.rolling_performances(timestamp=timestamp, save=False, db_id=db_id)
riskfree = np.mean(metrics['Treasury.Returns'])
#NOTE Script args define default database table name (test), make it consistent
if db_id is None:
db_id = self.backtest_cfg['algorithm'] + pd.datetime.strftime(pd.datetime.now(), format='%Y%m%d')
perfs['Name'] = db_id
perfs['Sharpe.Ratio'] = tsu.get_sharpe_ratio(metrics['Returns'].values, risk_free = riskfree)
perfs['Returns'] = (((metrics['Returns'] + 1).cumprod()) - 1)[-1]
perfs['Max.Drawdown'] = max(metrics['Max.Drawdown'])
perfs['Volatility'] = np.mean(metrics['Volatility'])
perfs['Beta'] = np.mean(metrics['Beta'])
perfs['Alpha'] = np.mean(metrics['Alpha'])
perfs['Benchmark.Returns'] = (((metrics['Benchmark.Returns'] + 1).cumprod()) - 1)[-1]
if save:
self.feeds.stock_db.save_performances(perfs)
return perfs
#TODO Save returns
def get_returns(self, benchmark=None, timestamp='one_month', save=False, db_id=None):
returns = dict()
if benchmark:
#TODO Benchmark fields in database for guessing name like for stocks
benchmark_symbol = '^GSPC'
benchmark_data = get_benchmark_returns(benchmark_symbol, self.backtest_cfg['start'], self.backtest_cfg['end'])
else:
raise NotImplementedError()
#benchmark_data = [d for d in benchmark_data if (d.date >= self.backtest_cfg['start']) and (d.date <= self.backtest_cfg['end'])]
dates = pd.DatetimeIndex([d.date for d in benchmark_data])
returns['Benchmark.Returns'] = pd.Series([d.returns for d in benchmark_data], index=dates)
returns['Benchmark.CReturns'] = ((returns['Benchmark.Returns'] + 1).cumprod()) - 1
returns['Returns'] = pd.Series(self.results.returns, index=dates)
returns['CReturns'] = pd.Series(((self.results.returns + 1).cumprod()) - 1, index=dates)
df = pd.DataFrame(returns, index=dates)
if save:
raise NotImplementedError()
self.feeds.stock_db.saveDFToDB(df, table=db_id)
if benchmark is None:
df = df.drop(['Benchmark.Returns', 'Benchmark.CReturns'], axis=1)
return df
def _get_index(self, perfs):
#NOTE No frequency infos or just period number ?
start = pytz.utc.localize(pd.datetime.strptime(perfs[0]['period_label'] + '-01', '%Y-%m-%d'))
end = pytz.utc.localize(pd.datetime.strptime(perfs[-1]['period_label'] + '-01', '%Y-%m-%d'))
return pd.date_range(start - pd.datetools.BDay(10), end, freq=pd.datetools.MonthBegin())
def _extract_perf(self, perfs, field):
index = self._get_index(perfs)
values = [perfs[i][field] for i in range(len(perfs))]
return pd.Series(values, index=index)
def __del__(self):
del self.server
class Remote(object):
'''
Entry point to remote access to data
Mainly offer a simpler interface and a dataaccess harmonisation
Plus symbol check and complete, timezone, currency conversion
Plus reindexage ?
'''
def __init__(self, country_code=None):
'''
Parameters
country_code: str
This information is used to setup International object
and get the right local conventions for lang,
dates and currencies. None will stand for 'fr'
'''
#self.locatioon = world.International(country_code)
self.datafeed = DataFeed()
#NOTE with args and kwargs ?
def fetch_equities_daily(self, equities, ohlc=False,
r_type=False, returns=False, **kwargs):
if len(equities) == 0:
return pd.DataFrame()
if isinstance(equities, str):
equities = equities.split(',')
symbols = [self.datafeed.guess_name(equity) for equity in equities]
if ohlc:
data = load_bars_from_yahoo(stocks=symbols, **kwargs)
data.items = equities
else:
data = load_from_yahoo(stocks=symbols, **kwargs)
data.columns = equities
#NOTE Would it work with a pandas panel ?
if returns:
data = ((data - data.shift(1)) / data).fillna(method='bfill')
if r_type:
data = convert_to_r_matrix(data)
return data
def fetch_equities_snapshot(self, *args, **kwargs):
'''
Use Yahoo and google finance service to fetch
current infromations about given equitiy names
______________________________________________
Parameters
args: tuple
company names to consider
kwargs['level']: int
Quantity of information level
______________________________________________
Return
snapshot: pandas.DataFrame
with names as columns and informations as index
'''
equities = args
if not equities:
equities = kwargs.get('symbols', [])
level = kwargs.get('level', 0)
#TODO Symbols are usualy reused, cach them
symbols = [self.datafeed.guess_name(equity) for equity in equities]
if not level:
# default level, the lightest
snapshot = snapshot_yahoo_pandas(symbols)
elif level == 1:
snapshot = snapshot_google_light(symbols)
elif level == 2:
snapshot = snapshot_google_heavy(symbols)
else:
raise ValueError('Invalid level of information requested')
# Give columns back equities names requested
snapshot.columns = equities
return snapshot
def _localize_data(self, data):
'''
Inspect data and applie localisation
on date and monnaie values
'''
assert isinstance(data, pd.Dataframe) or isinstance(data, dict)
class Setup(object):
''' Configuration object for the Trading engine'''
def __init__(self, config_dir='', offline=False):
'''
Parameters
configuration_folder: str
default is current location for configuration files
offline: boolean
Can force to switch off internal server
'''
#NOTE timezone as parameter
super(Setup, self).__init__()
self.configuration_folder = config_dir if config_dir else '/'.join((os.environ['QTRADE'], 'config'))
# Config data structures
self.config_backtest = dict()
self.config_strategie = dict()
self.config_environment = self._inspect_environment()
# Client for easy mysql database access
self.datafeed = DataFeed()
self.offline = offline
if not offline:
# It makes the entire simulator able to receive configuration and
# send informations to remote users and other processes like web frontend
self.server = network.ZMQ_Dealer(id=self.__class__.__name__)
def _inspect_environment(self, local_file='~/.quantrade/default.json'):
'''
Read common user and project configuration files
for usual environment parameters
'''
context = dict()
if os.path.exists(os.path.expanduser(local_file)):
log.info('Found local configuration file, loading {}'.format(local_file))
context = self._read_structured_file(os.path.expanduser(local_file))
return context
def _read_structured_file(self, formatfile, config_folder=False, select_field=None, format='json'):
'''
Map well structured, i.e. common file format like key-value storage, csv, ..., file content into a dictionnary
'''
if format == 'json':
try:
# Read given file in specified format from default or given config directory
if config_folder:
content = json.load(open('/'.join([self.configuration_folder, formatfile]), 'r'))
else:
content = json.load(open(formatfile, 'r'))
except:
log.error('** loading json configuration.')
return dict()
else:
#TODO Other key-value file storage style
raise NotImplementedError()
# Configuration fields are likely to have several parameter categories
# If specified, return only 'select_field' one
return content[select_field] if select_field else content
def parse_commandline(self):
'''
Read command lines arguments and map them
to a more usuable dictionnary
_________________________________________
Return
self.config_backtest: dict():
Backtest simulator itself parameters
'''
log.debug('Reading commandline arguments')
parser = argparse.ArgumentParser(description='Backtester module, the terrific financial simukation')
parser.add_argument('-v', '--version',
action='version',
version='%(prog)s v0.8.1 Licence rien du tout', help='Print program version')
parser.add_argument('-f', '--frequency',
type=str, action='store', default='daily',
required=False, help='(pandas) frequency in days betweend two quotes fetch')
parser.add_argument('-a', '--algorithm',
action='store',
required=True, help='Trading algorithm to be used')
parser.add_argument('-m', '--manager',
action='store',
required=True, help='Portfolio strategie to be used')
parser.add_argument('-b', '--database',
action='store', default='test',
required=False, help='Table to considere in database')
parser.add_argument('-i', '--initialcash',
type=float, action='store', default=100000.0,
required=False, help='Initial cash portfolio value')
parser.add_argument('-t', '--tickers',
action='store', default='random',
required=False, help='target names to process')
parser.add_argument('-s', '--start',
action='store', default='1/1/2006',
required=False, help='Start date of the backtester')
parser.add_argument('-e', '--end',
action='store', default='1/12/2010',
required=False, help='Stop date of the backtester')
parser.add_argument('-ex', '--exchange',
action='store', default='',
required=False, help='list of markets where trade, separated with a coma')
parser.add_argument('-r', '--remote',
action='store_true',
help='Indicates if the program was ran manually or not')
parser.add_argument('-l', '--live',
action='store_true',
help='makes the engine work in real-time !')
parser.add_argument('-p', '--port',
action='store', default=5570,
required=False, help='Activates the diffusion of the universe on the network, on the port provided')
args = parser.parse_args()
#TODO Same as zipline in datasource, a mapping function with type and conversion function tuple
#NOTE self.config_backtest = args.__dict__
# For generic use, further modules will need a dictionnary of parameters, not the namespace provided by argparse
log.debug('Mapping arguments to backtest parameters dictionnary')
self.config_backtest = {'algorithm' : args.algorithm,
'frequency' : args.frequency,
'manager' : args.manager,
'database' : args.database,
'tickers' : self._smart_tickers_select(args.tickers, exchange=args.exchange),
'start' : self._normalize_date_format(args.start),
'end' : self._normalize_date_format(args.end),
'live' : args.live,
'port' : args.port,
'exchange' : args.exchange,
'cash' : args.initialcash,
'remote' : args.remote}
return self.config_backtest
def get_strategie_configuration(self, *args, **kwargs):
'''
Read localy or receive remotely
strategie's parameters
'''
if kwargs.get('remote', self.config_backtest['remote']):
# Get configuration through ZMQ socket
self.config_strategie = self._get_remote_data(port=self.config_backtest['port'])
else:
log.info('Reading strategie configuration from json files')
self.config_strategie['manager'] = \
self._read_structured_file('managers.json',
config_folder=True,
select_field=self.config_backtest['manager'])
self.config_strategie['algorithm'] = \
self._read_structured_file('algorithms.json',
config_folder=True,
select_field=self.config_backtest['algorithm'])
# The manager can use the same socket during simulation to emit portfolio informations
self.config_strategie['manager']['server'] = self.server
log.info('Configuration is Done.')
return self.config_strategie
def _get_remote_data(self, port, host='localhost'):
'''
Listen on backend ZMQ socket configuration data
'''
# We need data from network, start the server
assert not self.offline
self.server.run(host=host, port=port)
# In remote mode, client sends missing configuration through zmq socket
log.info('Fetching backtest configuration from client')
msg = self.server.receive(json=True)
log.debug('Got it !')
# Check message format and fields
assert isinstance(msg, dict)
assert 'algorithm' in msg
assert 'manager' in msg
return msg
def _smart_tickers_select(self, tickers_description, exchange=''):
'''
Take tickers string description and return
an array of explicit and usuable symbols
'''
# Informations are coma separated within the string
tickers_description = tickers_description.split(',')
# Useful way of stocks selection in order to test algorithm strength
if tickers_description[0] == 'random':
# Basic check: the second argument is the the number, integer, of stocks to pick up randomly
assert len(tickers_description) == 2
assert int(tickers_description[1])
# Pick up stocks on specified (or not) market exchange
tickers_description = self.datafeed.random_stocks(int(tickers_description[1]), exchange=exchange.split(','))
return tickers_description
#TODO Handle in-day dates, with hours and minutes
def _normalize_date_format(self, date):
'''
Dates can be defined in many ways, but zipline use
aware datetime objects only
__________________________________________________
Parameters
date: str
String date like YYYY-MM-DD
'''
assert isinstance(date, str)
return pytz.utc.localize(datetime.strptime(date, '%Y-%m-%d'))
class QuandlSource(DataSource):
"""
Yields all events in event_list that match the given sid_filter.
If no event_list is specified, generates an internal stream of events
to filter. Returns all events if filter is None.
Configuration options:
sids : list of values representing simulated internal sids
start : start date
delta : timedelta between internal events
filter : filter to remove the sids
"""
def __init__(self, data_descriptor, **kwargs):
assert isinstance(data_descriptor['index'],
pd.tseries.index.DatetimeIndex)
self.data_descriptor = data_descriptor
# Unpack config dictionary with default values.
self.sids = kwargs.get('sids', data_descriptor['tickers'])
self.start = kwargs.get('start', data_descriptor['index'][0])
self.end = kwargs.get('end', data_descriptor['index'][-1])
# Hash_value for downstream sorting.
self.arg_string = hash_args(data_descriptor, **kwargs)
self._raw_data = None
self.feed = DataFeed()
@property
def mapping(self):
mapping = {
'dt': (lambda x: x, 'dt'),
'sid': (lambda x: x, 'sid'),
'price': (float, 'Adjusted Close'),
'volume': (int, 'Volume'),
}
# Add additional fields.
for field_name in self.data:
if field_name in ['Adjusted Close', 'Volume', 'dt', 'sid']:
continue
mapping[field_name] = (lambda x: x, field_name)
return mapping
@property
def instance_hash(self):
return self.arg_string
def _get(self):
#TODO Test here for one value, make it later a panel
assert len(self.sids) == 1
# Try to set quandl api key, stored in default config file
self.feed._search_quandlkey()
return self.feed.fetch_quandl(self.sids[0],
start_date=self.start,
end_date=self.end,
returns='pandas')
def raw_data_gen(self):
self.data = self._get()
sid = self.sids[0]
for dt, series in self.data.iterrows():
event = {
'dt': dt,
'sid': sid,
}
for field_name, value in series.iteritems():
event[field_name] = value
yield event
@property
def raw_data(self):
if not self._raw_data:
self._raw_data = self.raw_data_gen()
return self._raw_data
import pandas.rpy.common as com
from neuronquant.data.datafeed import DataFeed
r = robjects.r
svmforecast_lib = 'e1071.R'
r('source("%s")' % svmforecast_lib)
#quantmod = importr('quantmod')
symbol = 'GOOG'
history = 500
today = datetime.datetime.now()
start_date = today - pd.datetools.Day(history)
#r('require(quantmod)')
#tt = r('get( getSymbols("{}", from="{}"))'.format(symbol, start_date.strftime(format='%Y-%m-%d')))
returns = DataFeed().quotes(symbol, start_date=start_date, end_date=today)
#returns = data.pct_change().fillna(0.0)
returns = returns.rename(columns={symbol: "Close"})
r_quotes = com.convert_to_r_dataframe(returns)
import ipdb
ipdb.set_trace()
data_matrix = r('svmFeatures')(r_quotes)
#rets = r('na.trim( ROC(Cl({}), type="discrete"))'.format('tt'))
#data_matrix = r('svmFeatures')(tt)
#data_df = pd.rpy.common.convert_robj(data_matrix)
import pandas.rpy.common as com
from neuronquant.data.datafeed import DataFeed
r = robjects.r
svmforecast_lib = 'e1071.R'
r('source("%s")' % svmforecast_lib)
#quantmod = importr('quantmod')
symbol = 'GOOG'
history = 500
today = datetime.datetime.now()
start_date = today - pd.datetools.Day(history)
#r('require(quantmod)')
#tt = r('get( getSymbols("{}", from="{}"))'.format(symbol, start_date.strftime(format='%Y-%m-%d')))
returns = DataFeed().quotes(symbol, start_date=start_date, end_date=today)
#returns = data.pct_change().fillna(0.0)
returns = returns.rename(columns={symbol: "Close"})
r_quotes = com.convert_to_r_dataframe(returns)
import ipdb; ipdb.set_trace()
data_matrix = r('svmFeatures')(r_quotes)
#rets = r('na.trim( ROC(Cl({}), type="discrete"))'.format('tt'))
#data_matrix = r('svmFeatures')(tt)
#data_df = pd.rpy.common.convert_robj(data_matrix)
class PortfolioManager(object):
'''
Manages portfolio during simulation, and stays aware of the situation
through the update() method. It is configured through zmq message (manager
field) or QuanTrade/config/managers.json file.
User strategies call it with a dictionnnary of detected opportunities (i.e.
buy or sell signals). Then the optimize function computes assets
allocation, returning a dictionnary of symbols with their weigths or amount
to reallocate.
__________________________ _____________
signals {'google': 745.5} --> | | --> | |
| trade_signals_handler() | | optimize() |
orders {'google': 34} <-- |_________________________| <-- |____________|
In addition, portfolio objects can be saved in database and reloaded later,
and user on-the-fly orders are catched and executed in remote mode. Finally
portfolios are connected to the server broker and, if requested, send state
messages to client.
This is abstract class, inheretid class will eventally overwrite optmize()
to expose their own asset allocation strategy.
'''
__metaclass__ = abc.ABCMeta
#TODO Add in the constructor or setup parameters some general settings like maximum weights, positions, frequency,...
#TODO Better to return 0 stocks to trade: remove the field
#NOTE Regarding portfolio constraints: from a set of user-defined
#parameters, a unnique set should be constructed. Then the solution provided
#by optimize function would have to be a subset of it. (classic mathematical solution)
#Finally it should be defined how to handle non-correct solutions
def __init__(self, configuration):
'''
Parameters
configuration : dict
Named parameters used either for general portfolio settings
(server and constraints), and for user optimizer function
'''
super(PortfolioManager, self).__init__()
# Easy mysql access
self.datafeed = DataFeed()
# Zipline portfolio object, updated during simulation with self.date
self.portfolio = None
self.date = None
# Portfolio owner, mainly used for database saving and client communication
self.name = configuration.get('name', 'ChuckNorris')
self.log = logbook.Logger('Manager::' + self.name)
# Other parameters are used in user optimize() method
self._optimizer_parameters = configuration
# Make the manager talk to connected clients
self.connected = configuration.get('connected', False)
# Send android notifications when orders are processed
# It's only possible with a running server
self.android = configuration.get('android', False) & self.connected
# Delete from database data with the same portfolio name
if configuration.get('clean', True):
self.log.info('Cleaning previous trades.')
clean_previous_trades(self.name)
# Run the server if the engine didn't, while it is asked for
if 'server' in configuration and self.connected:
# Getting server object instanciated anyway before (by Setup object)
self.server = configuration.pop('server')
# Web based dashboard where real time results are monitored
#FIXME With dynamic generation, dashboad never exists at this point
#self.dashboard = Dashboard(self.name)
# In case user optimization would need to retrieve more data
self.remote = Remote()
@abc.abstractmethod
def optimize(self):
'''
Users should overwrite this method
'''
pass
def update(self, portfolio, date, metrics=None, save=False, widgets=False):
'''
Actualizes the portfolio universe
and if connected, sends it through the wires
________________________________
Parameters
portfolio: zipline.portfolio
ndict object storing portfolio values at the given date
date: datetime.datetime
Current date in zipline simulation
save: boolean
If true, save the portfolio in database under self.name key
'''
# Make the manager aware of current simulation portfolio and date
self.portfolio = portfolio
self.date = date
if save:
self.save_portfolio(portfolio)
if metrics is not None:
save_metrics_snapshot(self.name, self.date, metrics)
# Delete sold items and add new ones on dashboard
#if widgets:
#self.dashboard.update_position_widgets(self.portfolio.positions)
# Send portfolio object to client
if self.connected:
#NOTE Something smarter ?
# We need to translate zipline portfolio and position objects into json data (i.e. dict)
packet_portfolio = to_dict(portfolio)
for pos in packet_portfolio['positions']:
packet_portfolio['positions'][pos] = to_dict(packet_portfolio['positions'][pos])
self.server.send(packet_portfolio,
type ='portfolio',
channel='dashboard')
# Check user remote messages and return it
return self.catch_messages()
return dict()
def trade_signals_handler(self, signals, extras={}):
'''
Process buy and sell signals from the simulation
___________________________________________________________
Parameters
signals: dict
hold stocks of interest, format like {"google": 0.8, "apple": -0.2}
If the value is negative -> sell signal, otherwize buy one
Values are ranged between [-1 1] regarding signal confidence
extras: whatever
Object sent from algorithm for certain managers
___________________________________________________________
Return
dict orderBook, like {"google": 34, "apple": -56}
'''
self._optimizer_parameters['algo'] = extras
orderBook = dict()
# If value < 0, it's a sell signal on the key, else buy signal
to_buy = dict(filter(lambda (sid, strength): strength > 0, signals.iteritems()))
to_sell = dict(filter(lambda (sid, strength): strength < 0, signals.iteritems()))
#to_buy = [t for t in signals if signals[t] > 0]
#NOTE With this line we can't go short
#to_sell = set(self.portfolio.positions.keys()).intersection(
#[t for t in signals if signals[t] < 0])
if not to_buy and not to_sell:
# Nothing to do
return dict()
# Compute the optimal portfolio allocation, using user defined function
alloc, e_ret, e_risk = self.optimize(self.date, to_buy, to_sell, self._optimizer_parameters)
#TODO Check about selling in available money and handle 250 stocks limit
#TODO Handle max_* as well, ! already actif stocks
## Building orders for zipline
#NOTE The follonwing in a separate function that could be used when catching message from user
for t in alloc:
## Handle allocation returned as number of stocks to order
if isinstance(alloc[t], int):
orderBook[t] = alloc[t]
## Handle allocation returned as stock weights to order
elif isinstance(alloc[t], float):
# Sell orders
if alloc[t] <= 0:
orderBook[t] = int(alloc[t] * self.portfolio.positions[t].amount)
## Buy orders
else:
## If we already trade this ticker, substract owned amount before computing number of stock to buy
if self.portfolio.positions[t].amount:
price = self.portfolio.positions[t].last_sale_price
else:
price = signals[t]
orderBook[t] = (int(alloc[t] * self.portfolio.portfolio_value / price)
- self.portfolio.positions[t].amount)
if self.android and orderBook:
# Alert user of the orders about to be processed
# Ok... kind of fancy method
ords = {'-1': 'sell', '1': 'buy'}
msg = 'QuanTrade suggests you to '
msg += ', '.join(['{} {} stocks of {}'
.format(ords[str(amount / abs(amount))], amount, ticker) for
ticker, amount in orderBook.iteritems()])
self.server.send_to_android({'title': 'Portfolio manager notification',
'priority': 1,
'description': msg})
return orderBook
def setup_strategie(self, parameters):
'''
General parameters or user settings
(maw_weigth, max_assets, max_frequency, commission cost)
________________________________________________________
Parameters
parameters: dict
Arbitrary values to change general constraints,
or for user algorithm settings
'''
assert isinstance(parameters, dict)
for name, value in parameters.iteritems():
self._optimizer_parameters[name] = value
def save_portfolio(self, portfolio):
'''
Store in database given portfolio,
for reuse later or further analysis puropose
____________________________________________
Parameters
portfolio: zipline.protocol.Portfolio(1)
ndict portfolio object to store
___________________________________________
'''
self.log.debug('Saving portfolio in database')
self.datafeed.stock_db.save_portfolio(portfolio, self.name, self.date)
def load_portfolio(self, name):
'''
Load a complete portfolio object from database
______________________________________________
Parameters
name: str(...)
name used as primary key in db for the portfolio
______________________________________________
Return
The portfolio with the given name if found,
None otherwize
'''
self.log.info('Loading portfolio {} from database'.foramt(name))
# Get the portfolio as a pandas Serie
db_pf = self.datafeed.saved_portfolios(name)
# Create empty Portfolio object to be filled
portfolio = zp.Portfolio()
# The function returns an empty dataframe if it didn't find a portfolio with id 'name' in db
if len(db_pf):
# Fill new portfolio data structure
portfolio.capital_used = db_pf['Capital']
portfolio.starting_cash = db_pf['StartingCash']
portfolio.portfolio_value = db_pf['PortfolioValue']
portfolio.pnl = db_pf['PNL']
portfolio.returns = db_pf['Returns']
portfolio.cash = db_pf['Cash']
portfolio.start_date = db_pf['StartDate']
portfolio.positions = self._adapt_positions_type(db_pf['Positions'])
portfolio.positions_value = db_pf['PositionsValue']
return portfolio
def _adapt_positions_type(self, db_pos):
'''
From array of sql Positions data model
To Zipline Positions object
'''
# Create empty Positions object to be filled
positions = zp.Positions()
for pos in db_pos:
if pos.Ticker not in positions:
positions[pos.Ticker] = zp.Position(pos.Ticker)
position = positions[pos.Ticker]
position.amount = pos.Amount
position.cost_basis = pos.CostBasis
position.last_sale_price = pos.LastSalePrice
return positions
def catch_messages(self, timeout=1):
'''
Listen for user messages,
process usual orders
'''
msg = self.server.noblock_recv(timeout=timeout, json=True)
#TODO msg is a command or an information, process it
if msg:
self.log.info('Got message from user: {}'.format(msg))
try:
msg = json.loads(msg)
except:
msg = ''
self.log.error('Unable to parse user message')
return msg
class PortfolioManager:
'''
Observes the trader universe and produces
orders to be excuted within zipline
Abstract method meant to be used by user
to construct their portfolio optimizer
'''
__metaclass__ = abc.ABCMeta
#TODO Add in the constructor or setup parameters some general settings like maximum weights, positions, frequency,...
def __init__(self, parameters):
'''
Parameters
parameters : dict(...)
Named parameters used either for general portfolio settings
(server and constraints), and for user optimizer function
'''
super(PortfolioManager, self).__init__()
self.log = Logger('Manager')
self.datafeed = DataFeed()
self.portfolio = None
self.date = None
self.name = parameters.get('name', 'Chuck Norris')
self._optimizer_parameters = parameters
self.connected = False
#TODO Should send stuff anyway, and accept new connections while running
self.connected = parameters.get('connected', False)
# Run the server if the engine didn't while it is asked
if 'server' in parameters:
self.server = parameters.pop('server')
if self.server.port is None and self.connected:
self.log.info('Binding manager on default port...')
self.server.run(host='127.0.0.1', port=5570)
@abc.abstractmethod
def optimize(self):
'''
Users must overwrite this method
'''
pass
def update(self, portfolio, date):
'''
Actualizes the portfolio universe
and if connected, sends it through the wires
________________________________
Parameters
portfolio: zipline.portfolio(1)
ndict object storing portfolio values at the given date
date: datetime.datetime(1)
Current date in zipline simulation
'''
self.portfolio = portfolio
self.date = date
if self.connected:
self.server.send(to_dict(portfolio),
type = 'portfolio',
channel = 'dashboard')
return self.catch_messages()
return dict()
def trade_signals_handler(self, signals):
'''
Process buy and sell signals from backtester or live trader
@param signals: dict holding stocks of interest, format like {"google": 567.89, "apple": -345.98}
If the value is negative -> sell signal, otherwize buy one
@return: dict orderBook, like {"google": 34, "apple": -56}
'''
orderBook = dict()
# If value < 0, it's a sell signal on the key, else buy signal
to_buy = [t for t in signals if signals[t] > 0]
to_sell = set(self.portfolio.positions.keys()).intersection([t for t in signals if signals[t] < 0])
if not to_buy and not to_sell:
# Nothing to do
return dict()
# Compute the optimal portfolio allocation, using user defined function
alloc, e_ret, e_risk = self.optimize(self.date, to_buy, to_sell, self._optimizer_parameters)
#TODO Check about selling in available money and handle 250 stocks limit
#TODO Handle max_* as well, ! already actif stocks
## Building orders for zipline
#NOTE The follonwing in a separate function that could be used when catching message from user
for t in alloc:
## Handle allocation returned as number of stocks to order
if isinstance(alloc[t], int):
orderBook[t] = alloc[t]
## Handle allocation returned as stock weights to order
elif isinstance(alloc[t], float):
# Sell orders
if alloc[t] <= 0:
orderBook[t] = int(alloc[t] * self.portfolio.positions[t].amount)
## Buy orders
else:
## If we already trade this ticker, substract owned amount before computing number of stock to buy
if self.portfolio.positions[t].amount:
price = self.portfolio.positions[t].last_sale_price
else:
price = signals[t]
orderBook[t] = (int(alloc[t] * self.portfolio.portfolio_value / price)
- self.portfolio.positions[t].amount)
return orderBook
def setup_strategie(self, parameters):
'''
General parameters or user ones setting
(maw_weigth, max_assets, max_frequency, commission cost)
________________________________________________________
Parameters
parameters: dict(...)
Arbitrary values to change general constraints,
or for user algorithm settings
'''
for name, value in parameters.iteritems():
self._optimizer_parameters[name] = value
#TODO Still need here this dict = f(ndict)
def save_portfolio(self, portfolio):
'''
Store in database given portfolio,
for reuse later or further analysis puropose
____________________________________________
Parameters
portfolio: zipline.protocol.Portfolio(1)
ndict portfolio object to store
___________________________________________
'''
self.log.info('Saving portfolio in database')
self.datafeed.stock_db.save_portfolio(portfolio, self.name, self.date)
def load_portfolio(self, name):
'''
Load a complete portfolio object from database
______________________________________________
Parameters
name: str(...)
name used as primary key in db for the portfolio
______________________________________________
Return
The portfolio with the given name if found,
None otherwize
'''
self.log.info('Loading portfolio from database')
## Get the portfolio as a pandas Serie
db_pf = self.datafeed.saved_portfolios(name)
## The function returns None if it didn't find a portfolio with id 'name' in db
if db_pf is None:
return None
# Creating portfolio object
portfolio = zp.Portfolio()
portfolio.capital_used = db_pf['Capital']
portfolio.starting_cash = db_pf['StartingCash']
portfolio.portfolio_value = db_pf['PortfolioValue']
portfolio.pnl = db_pf['PNL']
portfolio.returns = db_pf['Returns']
portfolio.cash = db_pf['Cash']
portfolio.start_date = db_pf['StartDate']
portfolio.positions = self._adapt_positions_format(db_pf['Positions'])
portfolio.positions_value = db_pf['PositionsValue']
return portfolio
def _adapt_positions_type(self, db_pos):
'''
From array of sql Positions data model
To Zipline Positions object
'''
positions = zp.Positions()
for pos in db_pos:
if pos.Ticker not in positions:
positions[pos.Ticker] = zp.Position(pos.Ticker)
position = positions[pos.Ticker]
position.amount = pos.Amount
position.cost_basis = pos.CostBasis
position.last_sale_price = pos.LastSalePrice
return positions
def catch_messages(self, timeout=1):
'''
Listen for user messages,
process usual orders
'''
msg = self.server.noblock_recv(timeout=timeout, json=True)
#TODO msg is a command or an information, process it
if msg:
self.log.info('Got message from user: {}'.format(msg))
return msg
def __init__(self, data=None):
#NOTE Allowing different data access ?
#self.metrics = None
#self.server = ZMQ_Dealer(id=self.__class__.__name__)
self.datafeed = DataFeed()
class Simulation(object):
''' Take a trading strategie and evalute its results '''
def __init__(self, data=None):
#NOTE Allowing different data access ?
#self.metrics = None
#self.server = ZMQ_Dealer(id=self.__class__.__name__)
self.datafeed = DataFeed()
#TODO For both, timezone configuration
def configure(self, configuration):
'''
Prepare dates, data, trading environment for simulation
_______________________________________________________
Parameters
configuration: dict()
Structure with previously defined backtest behavior
'''
data = self._configure_data(tickers=configuration['tickers'],
start_time=configuration['start'],
end_time=configuration['end'],
freq=configuration['frequency'],
exchange=configuration['exchange'],
live=configuration['live'])
context = self._configure_context(configuration['exchange'])
return data, context
#NOTE Should the data be loaded in zipline sourcedata class ?
#FIXME data default not suitable for live mode
def _configure_data(self,
tickers,
start_time=None,
end_time=pd.datetime.now(pytz.utc),
freq='daily',
exchange='',
live=False):
if live:
# Default end_date is now, suitable for live trading
self.load_market_data = LiveBenchmark(
end_time, frequency=freq).load_market_data
#dates = pd.date_range(start_time, end_time, freq=freq)
#NOTE A temporary hack to avoid zipline dirty modification
#periods = end_time - start_time
#dates = datautils.filter_market_hours(pd.date_range(pd.datetime.now(), periods=periods.days + 1,
dates = datautils.filter_market_hours(
pd.date_range(pytz.utc.localize(pd.datetime.now()),
end_time,
freq='1min'),
#TODO ...hard coded, later: --frequency daily,3
exchange)
import ipdb
ipdb.set_trace()
#dates = datautils.filter_market_hours(dates, exchange)
if len(dates) == 0:
log.warning('! Market closed.')
sys.exit(0)
#TODO Wrap it in a dataframe (always same return type)
data = {
'stream_source': exchange,
'tickers': tickers,
'index': dates
}
else:
# Use default zipline load_market_data, i.e. data from msgpack files in ~/.zipline/data/
self.load_market_data = None
#TODO if start_time is None get default start_time in ~/.quantrade/default.json
assert start_time
# Fetch data from mysql database
data = self.datafeed.quotes(tickers,
start_date=start_time,
end_date=end_time)
if len(data) == 0:
log.warning('Got nothing from database')
data = pd.DataFrame()
else:
assert isinstance(data, pd.DataFrame)
assert data.index.tzinfo
return data
def set_becnhmark_loader(self, load_function):
self.load_market_data = load_function
#TODO Use of futur localisation database criteria
#TODO A list of markets to check if this one is in
def _configure_context(self, exchange=''):
'''
Setup from exchange traded on benchmarks used, location
and method to load data market while simulating
_______________________________________________
Parameters
exchange: str
Trading exchange market
'''
# Environment configuration
if exchange in datautils.Exchange:
finance_context = TradingEnvironment(
bm_symbol=datautils.Exchange[exchange]['index'],
exchange_tz=datautils.Exchange[exchange]['timezone'],
load=self.load_market_data)
else:
raise NotImplementedError(
'Because of computation limitation, trading worldwide not permitted currently'
)
return finance_context
def run(self, data, configuration, strategie, context):
#___________________________________________________________________________ Running ________
log.info('\n-- Running backetester...\nUsing algorithm: {}\n'.format(
configuration['algorithm']))
log.info('\n-- Using portfolio manager: {}\n'.format(
configuration['manager']))
backtester = BacktesterEngine(configuration['algorithm'],
configuration['manager'], strategie)
#NOTE This method does not change anything
#backtester.set_sources([DataLiveSource(data_tmp)])
#TODO A new command line parameter ? only minutely and daily (and hourly normally) Use filter parameter of datasource ?
backtester.set_data_frequency(configuration['frequency'])
# Running simulation with it
with context:
sim_params = create_simulation_parameters(
capital_base=configuration['cash'],
start=configuration['start'],
end=configuration['end'])
results, monthly_perfs = backtester.run(data,
sim_params=sim_params)
#return self.results
return Analyze(results=results,
metrics=monthly_perfs,
datafeed=self.datafeed,
configuration=configuration)
class QuandlSource(DataSource):
"""
Yields all events in event_list that match the given sid_filter.
If no event_list is specified, generates an internal stream of events
to filter. Returns all events if filter is None.
Configuration options:
sids : list of values representing simulated internal sids
start : start date
delta : timedelta between internal events
filter : filter to remove the sids
"""
def __init__(self, data_descriptor, **kwargs):
assert isinstance(data_descriptor['index'], pd.tseries.index.DatetimeIndex)
self.data_descriptor = data_descriptor
# Unpack config dictionary with default values.
self.sids = kwargs.get('sids', data_descriptor['tickers'])
self.start = kwargs.get('start', data_descriptor['index'][0])
self.end = kwargs.get('end', data_descriptor['index'][-1])
# Hash_value for downstream sorting.
self.arg_string = hash_args(data_descriptor, **kwargs)
self._raw_data = None
self.feed = DataFeed()
@property
def mapping(self):
mapping = {
'dt': (lambda x: x, 'dt'),
'sid': (lambda x: x, 'sid'),
'price': (float, 'Adjusted Close'),
'volume': (int, 'Volume'),
}
# Add additional fields.
for field_name in self.data:
if field_name in ['Adjusted Close', 'Volume', 'dt', 'sid']:
continue
mapping[field_name] = (lambda x: x, field_name)
return mapping
@property
def instance_hash(self):
return self.arg_string
def _get(self):
#TODO Test here for one value, make it later a panel
assert len(self.sids) == 1
# Try to set quandl api key, stored in default config file
self.feed._search_quandlkey()
return self.feed.fetch_quandl(self.sids[0],
start_date=self.start,
end_date=self.end,
returns='pandas')
def raw_data_gen(self):
self.data = self._get()
sid = self.sids[0]
for dt, series in self.data.iterrows():
event = {
'dt': dt,
'sid': sid,
}
for field_name, value in series.iteritems():
event[field_name] = value
yield event
@property
def raw_data(self):
if not self._raw_data:
self._raw_data = self.raw_data_gen()
return self._raw_data
