class Remote(object):
'''
Entry point to remote access to data
Mainly offer a simpler interface and a dataaccess harmonisation
Plus symbol check and complete, timezone, currency conversion
Plus reindexage ?
'''
def __init__(self, country_code=None):
'''
Parameters
country_code: str
This information is used to setup International object
and get the right local conventions for lang, dates and currencies
None will stand for 'fr'
'''
#self.locatioon = world.International(country_code)
self.datafeed = DataFeed()
pass
def fetch_equities_snapshot(self, *args, **kwargs):
'''
Use Yahoo and google finance service to fetch
current infromations about given equitiy names
______________________________________________
Parameters
args: tuple
company names to consider
kwargs['level']: int
Quantity of information level
______________________________________________
Return
snapshot: pandas.DataFrame
with names as columns and informations as index
'''
equities = args
if not equities:
equities = kwargs.get('symbols', [])
level = kwargs.get('level', 0)
symbols = [self.datafeed.guess_name(equity) for equity in equities]
if not level:
# default level, the lightest
snapshot = snapshot_yahoo_pandas(symbols)
elif level == 1:
snapshot = snapshot_google_light(symbols)
elif level == 2:
snapshot = snapshot_google_heavy(symbols)
else:
raise ValueError('Invalid level of information requested')
# Give columns back equities names requested
snapshot.columns = equities
return snapshot
def _localize_data(self, data):
'''
Inspect data and applie localisation
on date and monnaie values
'''
assert isinstance(data, pd.Dataframe) or isinstance(data, dict)
class Remote(object):
'''
Entry point to remote access to data
Mainly offer a simpler interface and a dataaccess harmonisation
Plus symbol check and complete, timezone, currency conversion
Plus reindexage ?
'''
def __init__(self, country_code=None):
'''
Parameters
country_code: str
This information is used to setup International object
and get the right local conventions for lang, dates and currencies
None will stand for 'fr'
'''
#self.locatioon = world.International(country_code)
self.datafeed = DataFeed()
pass
def fetch_equities_snapshot(self, *args, **kwargs):
'''
Use Yahoo and google finance service to fetch
current infromations about given equitiy names
______________________________________________
Parameters
args: tuple
company names to consider
kwargs['level']: int
Quantity of information level
______________________________________________
Return
snapshot: pandas.DataFrame
with names as columns and informations as index
'''
equities = args
if not equities:
equities = kwargs.get('symbols', [])
level = kwargs.get('level', 0)
symbols = [self.datafeed.guess_name(equity) for equity in equities]
if not level:
# default level, the lightest
snapshot = snapshot_yahoo_pandas(symbols)
elif level == 1:
snapshot = snapshot_google_light(symbols)
elif level == 2:
snapshot = snapshot_google_heavy(symbols)
else:
raise ValueError('Invalid level of information requested')
# Give columns back equities names requested
snapshot.columns = equities
return snapshot
def _localize_data(self, data):
'''
Inspect data and applie localisation
on date and monnaie values
'''
assert isinstance(data, pd.Dataframe) or isinstance(data, dict)
class OptimalFrontier(PortfolioManager):
'''
Compute with R the efficient frontier and pick up the optimize point on it
'''
def __init__(self, parameters):
PortfolioManager.__init__(self, parameters)
# Database access for symbols retrieving
self.feeds = DataFeed()
# R stuff: R functions file and rpy interface
self.r = robjects.r
portfolio_opt_file = '/'.join((os.environ['QTRADE'], 'neuronquant/ai/opt_utils.R'))
self.r('source("{}")'.format(portfolio_opt_file))
def optimize(self, date, to_buy, to_sell, parameters):
symbols = []
allocations = dict()
# Considere only portfolio positions + future positions - positions about to be sold
positions = set([t for t in self.portfolio.positions.keys() if self.portfolio.positions[t].amount]).union(to_buy).difference(to_sell)
if not positions and to_sell:
for t in to_sell:
allocations[t] = - parameters.get('perc_sell', 1.0)
return allocations, 0, 1
try:
assert(positions)
except:
self.log.error('** No positions determined')
if len(positions) == 1:
return {positions.pop(): parameters.get('max_weigths', 0.2)}, 0, 1
for p in positions:
symbols.append(self.feeds.guess_name(p).lower())
loopback = parameters.get('loopback', 50)
source = parameters.get('source', 'yahoo')
start = pd.datetime.strftime(date - pd.datetools.BDay(loopback), format = '%Y-%m-%d')
date = pd.datetime.strftime(date, format = '%Y-%m-%d')
r_symbols = self.r('c("{}")'.format('", "'.join(symbols)))
r_names = self.r('c("{}.Return")'.format('.Return", "'.join(symbols)))
data = self.r('importSeries')(r_symbols, start, date, source=source)
frontier = self.r('getEfficientFrontier')(data, r_names, points = 500, Debug = False, graph = False)
if not frontier:
self.log.warning('No optimal frontier found')
return dict(), None, None
try:
mp = self.r('marketPortfolio')(frontier, 0.02, Debug = False, graph = False)
except:
self.log.error('** Error running R optimizer')
return dict(), None, None
self.log.debug('Allocation: {}'.format(mp))
#FIXME Some key errors survive so far
for s, t in zip(symbols, positions):
allocations[t] = round(mp.rx('.'.join((s, 'Return')))[0][0], 2)
er = round(mp.rx('er')[0][0], 2)
eStd = round(mp.rx('eStd')[0][0], 2)
self.log.debug('Allocation: {}\nWith expected return: {}\tand expected risk: {}'.format(allocations, er, eStd))
return allocations, er, eStd
class DataLiveSource(DataSource):
"""
Yields all events in event_list that match the given sid_filter.
If no event_list is specified, generates an internal stream of events
to filter. Returns all events if filter is None.
Configuration options:
sids : list of values representing simulated internal sids
start : start date
delta : timedelta between internal events
filter : filter to remove the sids
"""
def __init__(self, data, **kwargs):
assert isinstance(data['index'], pd.tseries.index.DatetimeIndex)
self.data = data
# Unpack config dictionary with default values.
self.sids = kwargs.get('sids', data['tickers'])
self.start = kwargs.get('start', data['index'][0])
self.end = kwargs.get('end', data['index'][-1])
self.fake_index = pd.date_range(self.start, self.end, freq=pd.datetools.BDay())
# Hash_value for downstream sorting.
self.arg_string = hash_args(data, **kwargs)
self._raw_data = None
self.remote = Fetcher()
self.feed = DataFeed()
@property
def mapping(self):
return {
'dt': (lambda x: x, 'dt'),
'sid': (lambda x: x, 'sid'),
'price': (float, 'price'),
'currency': (str, 'currency'),
'perc_change': (float, 'perc_change'),
'volume': (int, 'volume'),
}
@property
def instance_hash(self):
return self.arg_string
def raw_data_gen(self):
current_dt = datetime.datetime.now()
index = self.data['index']
selector = (index.day > current_dt.day) \
| ((index.day == current_dt.day) & (index.hour > current_dt.hour)) \
| ((index.day == current_dt.day) & (index.hour == current_dt.hour) & (index.minute >= current_dt.minute))
#NOTE Not an equal size issue ?
for fake_dt, dt in zip(self.fake_index, index[selector]):
while (current_dt.minute != dt.minute) or (current_dt.hour != dt.hour) :
time.sleep(15)
current_dt = datetime.datetime.now()
print('Waiting {} / {}'.format(current_dt, dt))
#for fake_dt, dt in zip(self.fake_index, self.data['index']):
for sid in self.data['tickers']:
if sid in self.sids:
symbol = self.feed.guess_name(sid).lower()
#FIXME Erros because no markets specifie, use light=True and add market
#snapshot = self.remote.get_stock_snapshot(symbol, light=False)
snapshot = self.remote.get_stock_snapshot(symbol, light=True)
import ipdb; ipdb.set_trace()
log.debug('Data available:\n{}'.format(json.dumps(snapshot,
sort_keys=True, indent=4, separators=(',', ': '))))
if not snapshot:
log.error('** No data snapshot available, maybe stopped by google ?')
sys.exit(2)
event = {
'dt': fake_dt,
'trade_time': dt,
'sid': sid,
'price': float(snapshot[symbol]['last']),
'currency': snapshot[symbol]['currency'],
'perc_change': float(snapshot[symbol]['perc_change']),
'volume': int(snapshot[symbol]['volume']),
}
yield event
@property
def raw_data(self):
if not self._raw_data:
self._raw_data = self.raw_data_gen()
return self._raw_data
class Remote(object):
'''
Entry point to remote access to data
Mainly offer a simpler interface and a dataaccess harmonisation
Plus symbol check and complete, timezone, currency conversion
Plus reindexage ?
'''
def __init__(self, country_code=None):
'''
Parameters
country_code: str
This information is used to setup International object
and get the right local conventions for lang,
dates and currencies. None will stand for 'fr'
'''
#self.locatioon = world.International(country_code)
self.datafeed = DataFeed()
#NOTE with args and kwargs ?
def fetch_equities_daily(self, equities, ohlc=False,
r_type=False, returns=False, **kwargs):
if len(equities) == 0:
return pd.DataFrame()
if isinstance(equities, str):
equities = equities.split(',')
symbols = [self.datafeed.guess_name(equity) for equity in equities]
if ohlc:
data = load_bars_from_yahoo(stocks=symbols, **kwargs)
data.items = equities
else:
data = load_from_yahoo(stocks=symbols, **kwargs)
data.columns = equities
#NOTE Would it work with a pandas panel ?
if returns:
data = ((data - data.shift(1)) / data).fillna(method='bfill')
if r_type:
data = convert_to_r_matrix(data)
return data
def fetch_equities_snapshot(self, *args, **kwargs):
'''
Use Yahoo and google finance service to fetch
current infromations about given equitiy names
______________________________________________
Parameters
args: tuple
company names to consider
kwargs['level']: int
Quantity of information level
______________________________________________
Return
snapshot: pandas.DataFrame
with names as columns and informations as index
'''
equities = args
if not equities:
equities = kwargs.get('symbols', [])
level = kwargs.get('level', 0)
#TODO Symbols are usualy reused, cach them
symbols = [self.datafeed.guess_name(equity) for equity in equities]
if not level:
# default level, the lightest
snapshot = snapshot_yahoo_pandas(symbols)
elif level == 1:
snapshot = snapshot_google_light(symbols)
elif level == 2:
snapshot = snapshot_google_heavy(symbols)
else:
raise ValueError('Invalid level of information requested')
# Give columns back equities names requested
snapshot.columns = equities
return snapshot
def _localize_data(self, data):
'''
Inspect data and applie localisation
on date and monnaie values
'''
assert isinstance(data, pd.Dataframe) or isinstance(data, dict)
class OptimalFrontier(PortfolioManager):
'''
Compute with R the efficient frontier and pick up the optimize point on it
'''
def __init__(self, parameters):
PortfolioManager.__init__(self, parameters)
# Database access for symbols retrieving
self.feeds = DataFeed()
# R stuff: R functions file and rpy interface
self.r = robjects.r
portfolio_opt_file = '/'.join(
(os.environ['QTRADE'], 'neuronquant/ai/opt_utils.R'))
self.r('source("{}")'.format(portfolio_opt_file))
def optimize(self, date, to_buy, to_sell, parameters):
symbols = []
allocations = dict()
# Considere only portfolio positions + future positions - positions about to be sold
positions = set([
t for t in self.portfolio.positions.keys()
if self.portfolio.positions[t].amount
]).union(to_buy).difference(to_sell)
if not positions and to_sell:
for t in to_sell:
allocations[t] = -parameters.get('perc_sell', 1.0)
return allocations, 0, 1
try:
assert (positions)
except:
self.log.error('** No positions determined')
if len(positions) == 1:
return {positions.pop(): parameters.get('max_weigths', 0.2)}, 0, 1
for p in positions:
symbols.append(self.feeds.guess_name(p).lower())
loopback = parameters.get('loopback', 50)
source = parameters.get('source', 'yahoo')
start = pd.datetime.strftime(date - pd.datetools.BDay(loopback),
format='%Y-%m-%d')
date = pd.datetime.strftime(date, format='%Y-%m-%d')
r_symbols = self.r('c("{}")'.format('", "'.join(symbols)))
r_names = self.r('c("{}.Return")'.format('.Return", "'.join(symbols)))
data = self.r('importSeries')(r_symbols, start, date, source=source)
frontier = self.r('getEfficientFrontier')(data,
r_names,
points=500,
Debug=False,
graph=False)
if not frontier:
self.log.warning('No optimal frontier found')
return dict(), None, None
try:
mp = self.r('marketPortfolio')(frontier,
0.02,
Debug=False,
graph=False)
except:
self.log.error('** Error running R optimizer')
return dict(), None, None
self.log.debug('Allocation: {}'.format(mp))
#FIXME Some key errors survive so far
for s, t in zip(symbols, positions):
allocations[t] = round(mp.rx('.'.join((s, 'Return')))[0][0], 2)
er = round(mp.rx('er')[0][0], 2)
eStd = round(mp.rx('eStd')[0][0], 2)
self.log.debug(
'Allocation: {}\nWith expected return: {}\tand expected risk: {}'.
format(allocations, er, eStd))
return allocations, er, eStd
class Simulation(object):
''' Take a trading strategie and evalute its results '''
def __init__(self, data=None, flavor='mysql', lvl='debug'):
#NOTE Allowing different data access ?
self.data = data
if not data:
self.feeds = DataFeed()
self.backtest_cfg = None
self.algo_cfg = None
self.manager_cfg = None
self.monthly_perfs = None
self.server = ZMQ_Dealer(id=self.__class__.__name__)
def configure(self, bt_cfg=None, a_cfg=None, m_cfg=None):
''' Reads and provides a clean configuration for the simulation '''
if not bt_cfg:
parser = argparse.ArgumentParser(description='Backtester module, the terrific financial simukation')
parser.add_argument('-v', '--version',
action='version',
version='%(prog)s v0.8.1 Licence rien du tout', help='Print program version')
parser.add_argument('-d', '--delta',
type=str, action='store', default='D',
required=False, help='Delta in days betweend two quotes fetch')
parser.add_argument('-a', '--algorithm',
action='store',
required=True, help='Trading algorithm to be used')
parser.add_argument('-m', '--manager',
action='store',
required=True, help='Portfolio strategie to be used')
parser.add_argument('-b', '--database',
action='store', default='test',
required=False, help='Table to considere in database')
parser.add_argument('-i', '--initialcash',
type=float, action='store', default=100000.0,
required=False, help='Initial cash portfolio value')
parser.add_argument('-t', '--tickers',
action='store', default='random',
required=False, help='target names to process')
parser.add_argument('-s', '--start',
action='store', default='1/1/2006',
required=False, help='Start date of the backtester')
parser.add_argument('-e', '--end',
action='store', default='1/12/2010',
required=False, help='Stop date of the backtester')
parser.add_argument('-ex', '--exchange',
action='store', default='s&p500',
required=False, help='list of markets where trade, separated with a coma')
parser.add_argument('-r', '--remote',
action='store_true',
help='Indicates if the program was ran manually or not')
parser.add_argument('-l', '--live',
action='store_true',
help='makes the engine work in real-time !')
parser.add_argument('-p', '--port',
action='store', default=5570,
required=False, help='Activates the diffusion of the universe on the network, on the port provided')
args = parser.parse_args()
self.backtest_cfg = {'algorithm' : args.algorithm,
'delta' : args.delta,
'manager' : args.manager,
'database' : args.database,
'tickers' : args.tickers.split(','),
'start' : args.start,
'end' : args.end,
'live' : args.live,
'port' : args.port,
'exchange' : args.exchange,
'cash' : args.initialcash,
'remote' : args.remote}
else:
self.backtest_cfg = bt_cfg
if isinstance(self.backtest_cfg['start'], str) and isinstance(self.backtest_cfg['end'], str):
###############test = pd.datetime.strptime('2012-02-01:14:00', '%Y-%m-%d:%H:%M')
self.backtest_cfg['start'] = pytz.utc.localize(pd.datetime.strptime(self.backtest_cfg['start'], '%Y-%m-%d'))
self.backtest_cfg['end'] = pytz.utc.localize(pd.datetime.strptime(self.backtest_cfg['end'], '%Y-%m-%d'))
elif isinstance(self.backtest_cfg['start'], pd.datetime) and isinstance(self.backtest_cfg['end'], pd.datetime):
if not self.backtest_cfg['start'].tzinfo:
self.backtest_cfg['start'] = pytz.utc.localize(self.backtest_cfg['start'])
if not self.backtest_cfg['end'].tzinfo:
self.backtest_cfg['end'] = pytz.utc.localize(self.backtest_cfg['end'])
else:
raise NotImplementedError()
self.read_config(remote=self.backtest_cfg['remote'], manager=m_cfg, algorithm=a_cfg)
return self.backtest_cfg
def read_config(self, *args, **kwargs):
self.manager_cfg = kwargs.get('manager', None)
self.algo_cfg = kwargs.get('algorithm', None)
if kwargs.get('remote', False):
self.server.run(host='127.0.0.1', port=self.backtest_cfg['port'])
# In remote mode, client sends missing configuration through zmq socket
if (not self.manager_cfg) or (not self.algo_cfg):
log.info('Fetching backtest configuration from client')
msg = self.server.receive(json=True)
log.debug('Got it !')
# Set simulation parameters with it
assert isinstance(msg, dict)
if self.manager_cfg is None:
self.manager_cfg = msg['manager']
if self.algo_cfg is None:
self.algo_cfg = msg['algorithm']
else:
# Reading configuration from json files
config_dir = '/'.join((os.environ['QTRADE'], 'config/'))
try:
# Files store many algos and manager parameters,
# use backtest configuration to pick up the right one
if self.manager_cfg is None:
self.manager_cfg = json.load(open('{}/managers.cfg'.format(config_dir), 'r'))[self.backtest_cfg['manager']]
if self.algo_cfg is None:
self.algo_cfg = json.load(open('{}/algos.cfg'.format(config_dir), 'r'))[self.backtest_cfg['algorithm']]
except:
log.error('** loading json configuration.')
sys.exit(1)
# The manager can use the same socket during simulation to emit portfolio informations
self.manager_cfg['server'] = self.server
log.info('Configuration is Done.')
def run_backtest(self):
# No configuration, no backtest man
if self.backtest_cfg is None or self.algo_cfg is None or self.manager_cfg is None:
log.error('** Backtester not configured properly')
return 1
#'''_____________________________________________________________________ Parameters ________
if self.backtest_cfg['tickers'][0] == 'random':
assert(len(self.backtest_cfg['tickers']) == 2)
assert(int(self.backtest_cfg['tickers'][1]))
self.backtest_cfg['tickers'] = self.feeds.random_stocks(int(self.backtest_cfg['tickers'][1]), exchange=self.backtest_cfg['exchange'].split(','))
if self.backtest_cfg['live']:
#dates = pd.date_range(self.backtest_cfg['start'], self.backtest_cfg['end'], freq=self.backtest_cfg['delta'])
#NOTE A temporary hack to avoid zipline dirty modification
periods = self.backtest_cfg['end'] - self.backtest_cfg['start']
dates = pd.date_range(pd.datetime.now(), periods=periods.days + 1, freq=self.backtest_cfg['delta'])
'''
if dates.freq > pd.datetools.Day():
#fr_selector = ((dates.hour > 8) & (dates.hour < 17)) | ((dates.hour == 17) & (dates.minute < 31))
us_selector = ((dates.hour > 15) & (dates.hour < 22)) | ((dates.hour == 15) & (dates.minute > 31))
dates = dates[us_selector]
if not dates:
log.warning('! Market closed.')
sys.exit(0)
'''
data = {'tickers': self.backtest_cfg['tickers'], 'index': dates.tz_localize(pytz.utc)}
else:
data = self.feeds.quotes(self.backtest_cfg['tickers'],
start_date = self.backtest_cfg['start'],
end_date = self.backtest_cfg['end'])
assert isinstance(data, pd.DataFrame)
assert data.index.tzinfo
#___________________________________________________________________________ Running ________
log.info('\n-- Running backetester...\nUsing algorithm: {}\n'.format(self.backtest_cfg['algorithm']))
log.info('\n-- Using portfolio manager: {}\n'.format(self.backtest_cfg['manager']))
backtester = BacktesterEngine(self.backtest_cfg['algorithm'],
self.backtest_cfg['manager'],
self.algo_cfg,
self.manager_cfg)
self.results, self.monthly_perfs = backtester.run(data,
SimulationParameters(capital_base=self.backtest_cfg['cash'],
period_start=self.backtest_cfg['start'],
period_end=self.backtest_cfg['end']))
return self.results
def rolling_performances(self, timestamp='one_month', save=False, db_id=None):
''' Filters self.perfs and, if asked, save it to database '''
#TODO Study the impact of month choice
#TODO Check timestamp in an enumeration
#TODO Implement other benchmarks for perf computation (zipline issue, maybe expected)
#NOTE Script args define default database table name (test), make it consistent
if db_id is None:
db_id = self.backtest_cfg['algorithm'] + pd.datetime.strftime(pd.datetime.now(), format='%Y%m%d')
if self.monthly_perfs:
#TODO New fields from zipline: information, sortino
perfs = dict()
length = range(len(self.monthly_perfs[timestamp]))
index = self._get_index(self.monthly_perfs[timestamp])
perfs['Name'] = np.array([db_id] * len(self.monthly_perfs[timestamp]))
#perfs['Period'] = np.array([self.monthly_perfs[timestamp][i]['period_label'] for i in length])
perfs['Period'] = np.array([pd.datetime.date(date) for date in index])
perfs['Sharpe.Ratio'] = np.array([self.monthly_perfs[timestamp][i]['sharpe'] for i in length])
perfs['Sortino.Ratio'] = np.array([self.monthly_perfs[timestamp][i]['sortino'] for i in length])
perfs['Information'] = np.array([self.monthly_perfs[timestamp][i]['information'] for i in length])
perfs['Returns'] = np.array([self.monthly_perfs[timestamp][i]['algorithm_period_return'] for i in length])
perfs['Max.Drawdown'] = np.array([self.monthly_perfs[timestamp][i]['max_drawdown'] for i in length])
perfs['Volatility'] = np.array([self.monthly_perfs[timestamp][i]['algo_volatility'] for i in length])
perfs['Beta'] = np.array([self.monthly_perfs[timestamp][i]['beta'] for i in length])
perfs['Alpha'] = np.array([self.monthly_perfs[timestamp][i]['alpha'] for i in length])
perfs['Excess.Returns'] = np.array([self.monthly_perfs[timestamp][i]['excess_return'] for i in length])
perfs['Benchmark.Returns'] = np.array([self.monthly_perfs[timestamp][i]['benchmark_period_return'] for i in length])
perfs['Benchmark.Volatility'] = np.array([self.monthly_perfs[timestamp][i]['benchmark_volatility'] for i in length])
perfs['Treasury.Returns'] = np.array([self.monthly_perfs[timestamp][i]['treasury_period_return'] for i in length])
else:
#TODO Get it from DB if it exists
raise NotImplementedError()
try:
data = pd.DataFrame(perfs, index=index)
except:
import ipdb; ipdb.set_trace()
if save:
self.feeds.stock_db.save_metrics(data)
return data
def overall_metrics(self, timestamp='one_month', metrics=None, save=False, db_id=None):
'''
Use zipline results to compute some performance indicators and store it in database
'''
perfs = dict()
# If no rolling perfs provided, computes it
if metrics is None:
metrics = self.rolling_performances(timestamp=timestamp, save=False, db_id=db_id)
riskfree = np.mean(metrics['Treasury.Returns'])
#NOTE Script args define default database table name (test), make it consistent
if db_id is None:
db_id = self.backtest_cfg['algorithm'] + pd.datetime.strftime(pd.datetime.now(), format='%Y%m%d')
perfs['Name'] = db_id
perfs['Sharpe.Ratio'] = tsu.get_sharpe_ratio(metrics['Returns'].values, risk_free = riskfree)
perfs['Returns'] = (((metrics['Returns'] + 1).cumprod()) - 1)[-1]
perfs['Max.Drawdown'] = max(metrics['Max.Drawdown'])
perfs['Volatility'] = np.mean(metrics['Volatility'])
perfs['Beta'] = np.mean(metrics['Beta'])
perfs['Alpha'] = np.mean(metrics['Alpha'])
perfs['Benchmark.Returns'] = (((metrics['Benchmark.Returns'] + 1).cumprod()) - 1)[-1]
if save:
self.feeds.stock_db.save_performances(perfs)
return perfs
#TODO Save returns
def get_returns(self, benchmark=None, timestamp='one_month', save=False, db_id=None):
returns = dict()
if benchmark:
benchmark_symbol = self.feeds.guess_name(benchmark)
if benchmark_symbol:
benchmark_data = get_benchmark_returns(benchmark_symbol, self.backtest_cfg['start'], self.backtest_cfg['end'])
else:
raise KeyError()
else:
#TODO Automatic detection given exchange market (on command line) ? Or s&p500 as only implemented in zipline currently (but not for long !)
raise NotImplementedError()
#NOTE Could be more efficient. But len(benchmark_data.date) != len(self.results.returns.index). Maybe because of different markets
dates = pd.DatetimeIndex([d.date for d in benchmark_data])
returns['Benchmark.Returns'] = pd.Series([d.returns for d in benchmark_data], index=dates)
returns['Benchmark.CReturns'] = ((returns['Benchmark.Returns'] + 1).cumprod()) - 1
returns['Returns'] = pd.Series(self.results.returns, index=dates)
returns['CReturns'] = pd.Series(((self.results.returns + 1).cumprod()) - 1, index=dates)
df = pd.DataFrame(returns, index=dates)
if save:
raise NotImplementedError()
self.feeds.stock_db.saveDFToDB(df, table=db_id)
if benchmark is None:
df = df.drop(['Benchmark.Returns', 'Benchmark.CReturns'], axis=1)
return df
def _get_index(self, perfs):
#NOTE No frequency infos or just period number ?
start = pytz.utc.localize(pd.datetime.strptime(perfs[0]['period_label'] + '-01', '%Y-%m-%d'))
end = pytz.utc.localize(pd.datetime.strptime(perfs[-1]['period_label'] + '-01', '%Y-%m-%d'))
return pd.date_range(start - pd.datetools.BDay(10), end, freq=pd.datetools.MonthBegin())
def _extract_perf(self, perfs, field):
index = self._get_index(perfs)
values = [perfs[i][field] for i in range(len(perfs))]
return pd.Series(values, index=index)
def __del__(self):
del self.server
class DataLiveSource(DataSource):
"""
Yields all events in event_list that match the given sid_filter.
If no event_list is specified, generates an internal stream of events
to filter. Returns all events if filter is None.
Configuration options:
sids : list of values representing simulated internal sids
start : start date
delta : timedelta between internal events
filter : filter to remove the sids
"""
def __init__(self, data, **kwargs):
assert isinstance(data['index'], pd.tseries.index.DatetimeIndex)
self.data = data
# Unpack config dictionary with default values.
self.sids = kwargs.get('sids', data['tickers'])
self.start = kwargs.get('start', data['index'][0])
self.end = kwargs.get('end', data['index'][-1])
self.fake_index = pd.date_range(self.start,
self.end,
freq=pd.datetools.BDay())
# Hash_value for downstream sorting.
self.arg_string = hash_args(data, **kwargs)
self._raw_data = None
self.remote = Fetcher()
self.feed = DataFeed()
@property
def mapping(self):
return {
'dt': (lambda x: x, 'dt'),
'sid': (lambda x: x, 'sid'),
'price': (float, 'price'),
'currency': (str, 'currency'),
'perc_change': (float, 'perc_change'),
'volume': (int, 'volume'),
}
@property
def instance_hash(self):
return self.arg_string
def raw_data_gen(self):
current_dt = datetime.datetime.now()
index = self.data['index']
selector = (index.day > current_dt.day) \
| ((index.day == current_dt.day) & (index.hour > current_dt.hour)) \
| ((index.day == current_dt.day) & (index.hour == current_dt.hour) & (index.minute >= current_dt.minute))
#NOTE Not an equal size issue ?
for fake_dt, dt in zip(self.fake_index, index[selector]):
while (current_dt.minute != dt.minute) or (current_dt.hour !=
dt.hour):
time.sleep(15)
current_dt = datetime.datetime.now()
print('Waiting {} / {}'.format(current_dt, dt))
#for fake_dt, dt in zip(self.fake_index, self.data['index']):
for sid in self.data['tickers']:
if sid in self.sids:
symbol = self.feed.guess_name(sid).lower()
#FIXME Erros because no markets specifie, use light=True and add market
#snapshot = self.remote.get_stock_snapshot(symbol, light=False)
snapshot = self.remote.get_stock_snapshot(symbol,
light=True)
import ipdb
ipdb.set_trace()
log.debug('Data available:\n{}'.format(
json.dumps(snapshot,
sort_keys=True,
indent=4,
separators=(',', ': '))))
if not snapshot:
log.error(
'** No data snapshot available, maybe stopped by google ?'
)
sys.exit(2)
event = {
'dt': fake_dt,
'trade_time': dt,
'sid': sid,
'price': float(snapshot[symbol]['last']),
'currency': snapshot[symbol]['currency'],
'perc_change': float(snapshot[symbol]['perc_change']),
'volume': int(snapshot[symbol]['volume']),
}
yield event
@property
def raw_data(self):
if not self._raw_data:
self._raw_data = self.raw_data_gen()
return self._raw_data
