def get_returns(sid, pattern=None):
try:
returns = index_quote_fetcher.fetch_window('returns',
shift_dates,
index=sid)
except:
assert pattern is not None
path0 = generate_path(pattern, sid, dates[0])
path1 = generate_path(pattern, sid, dates[-1])
if path0 == path1:
df = read_frame(path0)
df.index = pd.to_datetime(shift_dates)
returns = (df * stock_returns[df.columns]).sum(axis=1)
else:
returns = {}
for date, shift_date in zip(dates, shift_dates):
path = generate_path(pattern, sid, date)
df = pd.read_csv(path, header=0, dtype={0: str})
df.index = df['sid']
returns[shift_date] = (
df['weight'] *
stock_returns.ix[shift_date].ix[df.index]).sum()
returns = pd.Series(returns)
returns.index = pd.to_datetime(shift_dates)
return returns
def get_returns(sid, pattern=None):
try:
returns = index_quote_fetcher.fetch_window('returns', shift_dates, index=sid)
except:
assert pattern is not None
path0 = generate_path(pattern, sid, dates[0])
path1 = generate_path(pattern, sid, dates[-1])
if path0 == path1:
df = read_frame(path0)
df.index = pd.to_datetime(shift_dates)
returns = (df * stock_returns[df.columns]).sum(axis=1)
else:
returns = {}
for date, shift_date in zip(dates, shift_dates):
path = generate_path(pattern, sid, date)
df = pd.read_csv(path, header=0, dtype={0: str})
df.index = df['sid']
returns[shift_date] = (df['weight'] * stock_returns.ix[shift_date].ix[df.index]).sum()
returns = pd.Series(returns)
returns.index = pd.to_datetime(shift_dates)
return returns
parser = argparse.ArgumentParser()
parser.add_argument('position', type=str, nargs='+')
parser.add_argument('--composite', type=str)
parser.add_argument('--cost', type=float, default=0.001)
parser.add_argument('--pdf', type=str, required=True)
parser.add_argument('--legend', type=str, nargs='*')
args = parser.parse_args()
if not args.legend or len(args.legend) != len(args.position):
args.legend = range(len(args.position))
positions = []
dates, sids = None, None
for fname in args.position:
position = read_frame(fname)
if dates is None:
dates = position.index
else:
dates = dates.intersection(position.index)
if sids is None:
sids = set(position.columns)
else:
sids = sids | set(position.columns)
positions.append(position)
for i, position in enumerate(positions):
position = position.ix[dates]
positions[i] = position
dates = [date.strftime('%Y%m%d') for date in dates]
parser = argparse.ArgumentParser()
parser.add_argument('file', help='A Python configuration script file')
parser.add_argument('-s', '--start', help='IS startdate', type=str)
parser.add_argument('-e', '--end', help='IS enddate', type=str)
parser.add_argument('-o', '--output', help='Output file name', type=str)
parser.add_argument('-t', '--filetype', choices=('csv', 'msgpack', 'pickle'), default='msgpack')
args = parser.parse_args()
mod = imp.load_source('config', args.file)
groups = {}
for group, config in mod.groups.iteritems():
combiner = config['combiner']
for name, alpha_config in config['alphas'].iteritems():
alpha = read_frame(alpha_config['path'], alpha_config.get('filetype', None))
combiner.add_alpha(name, alpha, preprocess=alpha_config.get('preprocess', False))
groups[group] = combiner
if 'group_combiner' not in mod.__dict__:
try:
assert len(groups) == 1
except:
print 'For multiple groups, you should specify "group_combiner" in', args.file
raise
group_combiner = None
output = args.output
filetype = args.filetype
else:
if len(groups) > 1:
group_combiner = mod.group_combiner['combiner']
parser.add_argument('-e', '--ext', type=str, nargs='*')
parser.add_argument('-l', '--label', type=str, nargs='*')
parser.add_argument('--lshift', type=int, default=-1)
parser.add_argument('--rshift', type=int, default=20)
parser.add_argument('--pdf', type=str, required=True)
args = parser.parse_args()
if args.ext is None:
args.ext = []
if args.label is None:
args.label = []
if len(args.label) != len(args.ext):
args.label = ['factor_' + str(i) for i in range(len(args.ext))]
univ = read_frame(args.univ).fillna(False).astype(bool)
exts = {}
for label, ext in zip(args.label, args.ext):
exts[label] = read_frame(ext)
returns = fetch_returns(univ.index, args.rshift, args.lshift)
univ = univ.ix[returns.index]
ret = returns[univ].mean(axis=1)
ret.name = 'returns'
fig1 = plot_pnl(ret, by=None)
for k, v in exts.iteritems():
exts[k] = fetch_dates(v, univ.index)
exps = barra_exposure_fetcher.fetch_dates('style', univ.index)
exps.major_axis = [m[6:] for m in exps.major_axis]
parser.add_argument('-e', '--end', type=str)
parser.add_argument('--shift', type=int, default=0)
parser.add_argument('--source', type=str, choices=('lance', 'alpha', 'other'))
parser.add_argument('-a', '--account', type=str)
parser.add_argument('-i', '--input', type=str)
parser.add_argument('-o', '--output', type=str, default='assets.${YYYYMMDD}')
parser.add_argument('-p', '--price', type=str, choices=('prev_close', 'close', 'open', 'latest'), default='close')
args = parser.parse_args()
dates = args.date in DATES and [args.date] or []
if args.start:
dates = [date for date in DATES if date >= args.start]
if args.end:
dates = [date for date in dates if date <= args.end]
if args.shift:
dates = [DATES[DATES.index(date)-args.shift] for date in dates]
if args.account:
args.source = 'lance'
if args.input:
args.source = 'other'
assert args.source
if args.source == 'lance':
for date in dates:
prep_assets_lance(args.account, date, args.output)
elif args.source == 'alpha':
alpha = read_frame('/home/liulc/model/product/ols/alpha.univ0')
for date in dates:
prep_assets_alpha(alpha, date, args.output)
parser.add_argument('-i', '--input', type=str)
parser.add_argument('-o',
'--output',
type=str,
default='universe.${YYYYMMDD}')
args = parser.parse_args()
dates = args.date in DATES and [args.date] or []
if args.start:
dates = [date for date in DATES if date >= args.start]
if args.end:
dates = [date for date in dates if date <= args.end]
if args.shift:
dates = [DATES[DATES.index(date) - args.shift] for date in dates]
if args.account:
args.source = 'lance'
if args.input:
args.source = 'other'
if args.univ:
args.source = 'univ'
assert args.source
if args.source == 'lance':
for date in dates:
prep_universe_lance(args.account, date, args.output)
elif args.source == 'univ':
univ = read_frame('/home/liulc/model/universe/universe_' + args.univ)
for date in dates:
prep_universe_univ(univ, date, args.output)
type=str,
default='assets.${YYYYMMDD}')
parser.add_argument('-p',
'--price',
type=str,
choices=('prev_close', 'close', 'open', 'latest'),
default='close')
args = parser.parse_args()
dates = args.date in DATES and [args.date] or []
if args.start:
dates = [date for date in DATES if date >= args.start]
if args.end:
dates = [date for date in dates if date <= args.end]
if args.shift:
dates = [DATES[DATES.index(date) - args.shift] for date in dates]
if args.account:
args.source = 'lance'
if args.input:
args.source = 'other'
assert args.source
if args.source == 'lance':
for date in dates:
prep_assets_lance(args.account, date, args.output)
elif args.source == 'alpha':
alpha = read_frame('/home/liulc/model/product/ols/alpha.univ0')
for date in dates:
prep_assets_alpha(alpha, date, args.output)
parser.add_argument('-e', '--ext', type=str, nargs='*')
parser.add_argument('-l', '--label', type=str, nargs='*')
parser.add_argument('--lshift', type=int, default=-1)
parser.add_argument('--rshift', type=int, default=20)
parser.add_argument('--pdf', type=str, required=True)
args = parser.parse_args()
if args.ext is None:
args.ext = []
if args.label is None:
args.label = []
if len(args.label) != len(args.ext):
args.label = ['factor_'+str(i) for i in range(len(args.ext))]
univ = read_frame(args.univ).fillna(False).astype(bool)
exts = {}
for label, ext in zip(args.label, args.ext):
exts[label] = read_frame(ext)
returns = fetch_returns(univ.index, args.rshift, args.lshift)
univ = univ.ix[returns.index]
ret = returns[univ].mean(axis=1)
ret.name = 'returns'
fig1 = plot_pnl(ret, by=None)
for k, v in exts.iteritems():
exts[k] = fetch_dates(v, univ.index)
exps = barra_exposure_fetcher.fetch_dates('style', univ.index)
exps.major_axis = [m[6:] for m in exps.major_axis]
parser = argparse.ArgumentParser()
parser.add_argument('position', type=str, nargs='+')
parser.add_argument('--composite', type=str)
parser.add_argument('--cost', type=float, default=0.001)
parser.add_argument('--pdf', type=str, required=True)
parser.add_argument('--legend', type=str, nargs='*')
args = parser.parse_args()
if not args.legend or len(args.legend) != len(args.position):
args.legend = range(len(args.position))
positions= []
dates, sids = None, None
for fname in args.position:
position = read_frame(fname)
if dates is None:
dates = position.index
else:
dates = dates.intersection(position.index)
if sids is None:
sids = set(position.columns)
else:
sids = sids | set(position.columns)
positions.append(position)
for i, position in enumerate(positions):
position = position.ix[dates]
positions[i] = position
dates = [date.strftime('%Y%m%d') for date in dates]
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('metric', type=str, nargs='+')
parser.add_argument('--which', type=str, nargs='*')
parser.add_argument('--all', action='store_true')
parser.add_argument('--legend', type=str, nargs='*')
parser.add_argument('--pdf', type=str, required=True)
args = parser.parse_args()
if not args.legend or len(args.legend) != len(args.position):
args.legend = range(len(args.position))
metrics = []
for fname in args.metric:
metric = read_frame(fname)
if args.all:
args.which = metric.columns
metrics.append(metric)
figs = []
for col in args.which:
if col not in metrics[0].columns:
continue
df = []
for metric in metrics:
df.append(metric[col])
if len(df) > 1:
df = pd.concat(df, axis=1)
df.columns = args.legend
df.columns.name = col
parser = argparse.ArgumentParser()
parser.add_argument('-a', '--account', type=str, required=True)
parser.add_argument('-d', '--date', type=str, required=True)
parser.add_argument('-o', '--output', type=str, required=True)
parser.add_argument('-r', '--risk', type=float)
parser.add_argument('--lambda_f', type=float, default=0.05)
parser.add_argument('--lambda_d', type=float, default=0.025)
parser.add_argument('-x', '--exclude', type=str, required=True)
parser.add_argument('--alpha', type=str)
args = parser.parse_args()
if args.date not in DATES:
exit(1)
if args.alpha:
args.alpha = read_frame(args.alpha).ix[args.date]
if not os.path.exists(args.output):
os.makedirs(args.output)
bid_sid, sid_bid = get_idmaps(args.date)
regulars = read_regulars(args.date, args.account, args.alpha)
regulars['bid'] = [sid_bid[sid] for sid in regulars.index]
composites = read_composites(args.date, args.account)
assert args.exclude in composites.index and len(composites.index) > 1
dump_assets(regulars, args.output, config)
for sid in composites.index:
dump_composite(args.date, sid, args.output, config)
metrics = ('returns', ) + tuple([('r' if rank else '') + 'IC' + str(n)
for rank in [True, False]
for n in (1, 5, 20)])
parser.add_argument('--metric',
choices=metrics,
default='rIC1',
help='What type of correlations is of interest?')
parser.add_argument('--limit', type=int, default=10)
parser.add_argument('-n', '--negate', action='store_true')
args = parser.parse_args()
alpha_metric, dates = {}, None
if args.alpha:
for path in args.alpha:
for name in glob(path):
df = read_frame(name, args.ftype)
perf = Performance(df)
name = os.path.basename(name)
alpha_metric[name] = get_metric(perf, args.mode, args.metric)
if dates is None:
dates = alpha_metric[name].index
else:
dates = dates.union(alpha_metric[name].index)
ext_alphas = {}
if args.file:
with open(args.file) as file:
for line in file:
name, fpath = line.strip().split()
ext_alphas[name] = read_frame(fpath, args.ftype)
if args.dir:
parser.add_argument('--dir', help='Input directory, each file contained is assumed to be an alpha file', type=str)
parser.add_argument('--file', help='Input file, each row in the format: name path_to_a_csv_file', type=str)
parser.add_argument('--days', type=int, default=2*DAYS_IN_YEAR, help='How many points to be included in correlation calculation')
parser.add_argument('--db', help='Check correlation with alphas in alphadb', action='store_true')
parser.add_argument('--mode', choices=('longshort', 'quantile30', 'BTOP70Q', 'top30'), default='BTOP70Q')
metrics = ('returns',) + tuple([('r' if rank else '') + 'IC' + str(n) for rank in [True, False] for n in (1, 5, 20)])
parser.add_argument('--metric', choices=metrics, default='rIC1', help='What type of correlations is of interest?')
parser.add_argument('--limit', type=int, default=10)
parser.add_argument('-n', '--negate', action='store_true')
args = parser.parse_args()
alpha_metric, dates = {}, None
if args.alpha:
for path in args.alpha:
for name in glob(path):
df = read_frame(name, args.ftype)
perf = Performance(df)
name = os.path.basename(name)
alpha_metric[name] = get_metric(perf, args.mode, args.metric)
if dates is None:
dates = alpha_metric[name].index
else:
dates = dates.union(alpha_metric[name].index)
ext_alphas = {}
if args.file:
with open(args.file) as file:
for line in file:
name, fpath = line.strip().split()
ext_alphas[name] = read_frame(fpath, args.ftype)
if args.dir:
parser.add_argument('file', help='A Python configuration script file')
parser.add_argument('-s', '--start', help='IS startdate', type=str)
parser.add_argument('-e', '--end', help='IS enddate', type=str)
parser.add_argument('-o', '--output', help='Output file name', type=str)
parser.add_argument('-t',
'--filetype',
choices=('csv', 'msgpack', 'pickle'),
default='msgpack')
args = parser.parse_args()
mod = imp.load_source('config', args.file)
groups = {}
for group, config in mod.groups.iteritems():
combiner = config['combiner']
for name, alpha_config in config['alphas'].iteritems():
alpha = read_frame(alpha_config['path'],
alpha_config.get('filetype', None))
combiner.add_alpha(name,
alpha,
preprocess=alpha_config.get(
'preprocess', False))
groups[group] = combiner
if 'group_combiner' not in mod.__dict__:
try:
assert len(groups) == 1
except:
print 'For multiple groups, you should specify "group_combiner" in', args.file
raise
group_combiner = None
output = args.output
filetype = args.filetype
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('metric', type=str, nargs='+')
parser.add_argument('--which', type=str, nargs='*')
parser.add_argument('--all', action='store_true')
parser.add_argument('--legend', type=str, nargs='*')
parser.add_argument('--pdf', type=str, required=True)
args = parser.parse_args()
if not args.legend or len(args.legend) != len(args.position):
args.legend = range(len(args.position))
metrics = []
for fname in args.metric:
metric = read_frame(fname)
if args.all:
args.which = metric.columns
metrics.append(metric)
figs = []
for col in args.which:
if col not in metrics[0].columns:
continue
df = []
for metric in metrics:
df.append(metric[col])
if len(df) > 1:
df = pd.concat(df, axis=1)
df.columns = args.legend
df.columns.name = col
if args.atype == 'perf':
for alpha in args.alphas:
try:
with open(alpha) as file:
perf = cPickle.load(file)
alphas[alpha] = perf
if hasattr(perf, 'freq'):
args.atype = 'intraday'
else:
args.atype = 'daily'
except:
print '[WARNING] Failed to parse file', alpha
else:
for alpha in args.alphas:
try:
alphadf = read_frame(alpha, args.ftype)
if args.atype is None:
if len(alphadf.index) == len(np.unique(
alphadf.index.date)):
args.atype = 'daily'
else:
args.atype = 'intraday'
if args.atype == 'intraday':
perf = IntPerformance(alphadf)
else:
perf = Performance(alphadf)
alphas[alpha] = perf
except:
print '[WARNING] Failed to parse file', alpha
parser.add_argument('--source', type=str, choices=('lance', 'univ', 'other'))
parser.add_argument('-a', '--account', type=str)
parser.add_argument('-u', '--univ', type=str)
parser.add_argument('-i', '--input', type=str)
parser.add_argument('-o', '--output', type=str, default='universe.${YYYYMMDD}')
args = parser.parse_args()
dates = args.date in DATES and [args.date] or []
if args.start:
dates = [date for date in DATES if date >= args.start]
if args.end:
dates = [date for date in dates if date <= args.end]
if args.shift:
dates = [DATES[DATES.index(date)-args.shift] for date in dates]
if args.account:
args.source = 'lance'
if args.input:
args.source = 'other'
if args.univ:
args.source = 'univ'
assert args.source
if args.source == 'lance':
for date in dates:
prep_universe_lance(args.account, date, args.output)
elif args.source == 'univ':
univ = read_frame('/home/liulc/model/universe/universe_'+args.univ)
for date in dates:
prep_universe_univ(univ, date, args.output)
parser = argparse.ArgumentParser()
parser.add_argument('-a', '--alpha', required=True, type=str)
parser.add_argument('-c', '--config', required=True, type=str)
parser.add_argument('-d', '--dir', required=True, type=str)
parser.add_argument('-u', '--univ', required=True, type=str)
parser.add_argument('-s', '--start', type=str)
parser.add_argument('-e', '--end', type=str)
parser.add_argument('-f', '--freq', default=1, type=int)
parser.add_argument('-o', '--offset', default=0, type=int)
parser.add_argument('--debug_on', action='store_true')
args = parser.parse_args()
if args.start:
dates = [date for date in DATES if date >= args.start]
if args.end:
dates = [date for date in dates if date <= args.end]
dates = dates[args.offset::args.freq]
if not os.path.exists(args.dir):
os.makedirs(args.dir)
shutil.copy(args.alpha, args.dir)
shutil.copy(args.config, args.dir)
shutil.copy(args.univ, args.dir)
os.chdir(args.dir)
alpha, univ = read_frame(args.alpha), read_frame(args.univ)
optimizer = BarraOptimizer(args.config, args.debug_on, alpha, univ, dates)
for date in dates:
optimizer.run(date)
if args.atype == 'perf':
for alpha in args.alphas:
try:
with open(alpha) as file:
perf = cPickle.load(file)
alphas[alpha] = perf
if hasattr(perf, 'freq'):
args.atype = 'intraday'
else:
args.atype = 'daily'
except:
print '[WARNING] Failed to parse file', alpha
else:
for alpha in args.alphas:
try:
alphadf = read_frame(alpha, args.ftype)
if args.atype is None:
if len(alphadf.index) == len(np.unique(alphadf.index.date)):
args.atype = 'daily'
else:
args.atype = 'intraday'
if args.atype == 'intraday':
perf = IntPerformance(alphadf)
else:
perf = Performance(alphadf)
alphas[alpha] = perf
except:
print '[WARNING] Failed to parse file', alpha
if args.univ:
import os
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('alpha', help='Alpha file')
parser.add_argument('--ftype', help='File type', choices=('csv', 'pickle', 'msgpack'))
parser.add_argument('-s', '--start', type=str)
parser.add_argument('-e', '--end', type=str)
parser.add_argument('-l', '--lshift', nargs='*', type=int)
parser.add_argument('-r', '--rshift', nargs='*', type=int)
parser.add_argument('-m', '--mode', choices=('raw', 'neutral'), default='neutral')
parser.add_argument('--pdf', type=str, help='PDF file to save the plot')
parser.add_argument('--png', type=str, help='PNG file to save the plot')
args = parser.parse_args()
alpha = read_frame(args.alpha, args.ftype)
alpha = alpha.notnull()
if args.start:
alpha = alpha.ix[args.start:]
if args.end:
alpha = alpha.ix[:args.end]
if args.pdf and os.path.exists(args.pdf):
with magic.Magic() as m:
ftype = m.id_filename(args.pdf)
if ftype[:3] != 'PDF':
print 'The argument --pdf if exists must be a PDF file'
exit(0)
if args.png and os.path.exists(args.png):
with magic.Magic() as m:
ftype = m.id_filename(args.png)
parser.add_argument('--sum_u',
help='Whether to sum up these universes weight',
action='store_true')
parser.add_argument('-s', '--start', help='Startdate', type=str)
parser.add_argument('-e', '--end', help='Enddate', type=str)
parser.add_argument('--shift',
help='Shift of index data w.r.t. weight',
default=0,
type=int)
parser.add_argument('--dump', type=str, help='File to dump weights')
parser.add_argument('--pdf', type=str, help='PDF file to save the plot')
parser.add_argument('--png', type=str, help='PNG file to save the plot')
parser.add_argument('--ylim', nargs=2, help='Y-axis limit')
args = parser.parse_args()
alpha = read_frame(args.alpha, args.ftype)
if args.pdf and os.path.exists(args.pdf):
with magic.Magic() as m:
ftype = m.id_filename(args.pdf)
if ftype[:3] != 'PDF':
print 'The argument --pdf if exists must be a PDF file'
exit(0)
if args.png and os.path.exists(args.png):
with magic.Magic() as m:
ftype = m.id_filename(args.png)
if ftype[:3] != 'PNG':
print 'The argument --png if exists must be a PNG file'
exit(0)
if args.start:
alpha = alpha.ix[args.start:]
parser = argparse.ArgumentParser()
parser.add_argument('-a', '--account', type=str, required=True)
parser.add_argument('-d', '--date', type=str, required=True)
parser.add_argument('-o', '--output', type=str, required=True)
parser.add_argument('-r', '--risk', type=float)
parser.add_argument('--lambda_f', type=float, default=0.05)
parser.add_argument('--lambda_d', type=float, default=0.025)
parser.add_argument('-x', '--exclude', type=str, required=True)
parser.add_argument('--alpha', type=str)
args = parser.parse_args()
if args.date not in DATES:
exit(1)
if args.alpha:
args.alpha = read_frame(args.alpha).ix[args.date]
if not os.path.exists(args.output):
os.makedirs(args.output)
bid_sid, sid_bid = get_idmaps(args.date)
regulars = read_regulars(args.date, args.account, args.alpha)
regulars['bid'] = [sid_bid[sid] for sid in regulars.index]
composites = read_composites(args.date, args.account)
assert args.exclude in composites.index and len(composites.index) > 1
dump_assets(regulars, args.output, config)
for sid in composites.index:
dump_composite(args.date, sid, args.output, config)