def __init__(self, portfolio, name, opendate):
space = portfolio.space
self.date = opendate
self.name = name
self.status = 'opened'
self.mpos = 'flat'
self.quantity = 0
self.price = 0.0
self.value = 0.0
self.profit = 0.0
self.netreturn = 0.0
self.opened = opendate
self.held = 0
self.costbasis = 0.0
self.trades = []
self.ntrades = 0
self.pdata = Frame.frames[frame_name(name, space)].df
self.multiplier = MULTIPLIERS[space.subject]
def vapply(group, vname, vfuncs=None):
r"""Apply a variable to multiple dataframes.
Parameters
----------
group : alphapy.Group
The input group.
vname : str
The variable to apply to the ``group``.
vfuncs : dict, optional
Dictionary of external modules and functions.
Returns
-------
None : None
Other Parameters
----------------
Frame.frames : dict
Global dictionary of dataframes
See Also
--------
vunapply
"""
# get all frame names to apply variables
gnames = [item.lower() for item in group.members]
# get all the precedent variables
allv = vtree(vname)
# apply the variables to each frame
for g in gnames:
fname = frame_name(g, group.space)
if fname in Frame.frames:
f = Frame.frames[fname].df
if not f.empty:
for v in allv:
logger.debug("Applying variable %s to %s", v, g)
f = vexec(f, v, vfuncs)
else:
logger.debug("Frame for %s is empty", g)
else:
logger.debug("Frame not found: %s", fname)
def vunapply(group, vname):
r"""Remove a variable from multiple dataframes.
Parameters
----------
group : alphapy.Group
The input group.
vname : str
The variable to remove from the ``group``.
Returns
-------
None : None
Other Parameters
----------------
Frame.frames : dict
Global dictionary of dataframes
See Also
--------
vapply
"""
# get all frame names to apply variables
gnames = [item.lower() for item in group.all_members()]
# apply the variables to each frame
for g in gnames:
fname = frame_name(g, group.space)
if fname in Frame.frames:
f = Frame.frames[fname].df
logger.info("Unapplying variable %s from %s", vname, g)
if vname not in f.columns:
logger.info("Variable %s not in %s frame", vname, g)
else:
estr = "Frame.frames['%s'].df = f.df.drop('%s', axis=1)" \
% (fname, vname)
exec(estr)
else:
logger.info("Frame not found: %s", fname)
def run_system(model, system, group, intraday=False, quantity=1):
r"""Run a system for a given group, creating a trades frame.
Parameters
----------
model : alphapy.Model
The model object with specifications.
system : alphapy.System
The system to run.
group : alphapy.Group
The group of symbols to trade.
intraday : bool, optional
If true, this is an intraday system.
quantity : float, optional
The amount to trade for each symbol, e.g., number of shares
Returns
-------
tf : pandas.DataFrame
All of the trades for this ``group``.
"""
system_name = system.name
logger.info("Generating Trades for System %s", system_name)
# Unpack the model data.
directory = model.specs['directory']
extension = model.specs['extension']
separator = model.specs['separator']
# Extract the group information.
gname = group.name
gmembers = group.members
gspace = group.space
# Run the system for each member of the group
gtlist = []
for symbol in gmembers:
# generate the trades for this member
tlist = trade_system(model, system, gspace, intraday, symbol, quantity)
if tlist:
# add trades to global trade list
for item in tlist:
gtlist.append(item)
else:
logger.info("No trades for symbol %s", symbol)
# Create group trades frame
tf = None
if gtlist:
tspace = Space(system_name, "trades", group.space.fractal)
gtlist = sorted(gtlist, key=lambda x: x[0])
tf = DataFrame.from_items(gtlist, orient='index', columns=Trade.states)
tfname = frame_name(gname, tspace)
system_dir = SSEP.join([directory, 'systems'])
labels = ['date']
if intraday:
labels.append('time')
write_frame(tf,
system_dir,
tfname,
extension,
separator,
index=True,
index_label=labels)
del tspace
else:
logger.info("No trades were found")
# Return trades frame
return tf
def trade_system(model, system, space, intraday, name, quantity):
r"""Trade the given system.
Parameters
----------
model : alphapy.Model
The model object with specifications.
system : alphapy.System
The long/short system to run.
space : alphapy.Space
Namespace of instrument prices.
intraday : bool
If True, then run an intraday system.
name : str
The symbol to trade.
quantity : float
The amount of the ``name`` to trade, e.g., number of shares
Returns
-------
tradelist : list
List of trade entries and exits.
Other Parameters
----------------
Frame.frames : dict
All of the data frames containing price data.
"""
# Unpack the model data.
directory = model.specs['directory']
extension = model.specs['extension']
separator = model.specs['separator']
# Unpack the system parameters.
longentry = system.longentry
shortentry = system.shortentry
longexit = system.longexit
shortexit = system.shortexit
holdperiod = system.holdperiod
scale = system.scale
# Determine whether or not this is a model-driven system.
entries_and_exits = [longentry, shortentry, longexit, shortexit]
active_signals = [x for x in entries_and_exits if x is not None]
use_model = False
for signal in active_signals:
if any(x in signal for x in ['phigh', 'plow']):
use_model = True
# Read in the price frame
pf = Frame.frames[frame_name(name, space)].df
# Use model output probabilities as input to the system
if use_model:
# get latest probabilities file
probs_dir = SSEP.join([directory, 'output'])
file_path = most_recent_file(probs_dir, 'probabilities*')
file_name = file_path.split(SSEP)[-1].split('.')[0]
# read the probabilities frame and trim the price frame
probs_frame = read_frame(probs_dir, file_name, extension, separator)
pf = pf[-probs_frame.shape[0]:]
probs_frame.index = pf.index
probs_frame.columns = ['probability']
# add probability column to price frame
pf = pd.concat([pf, probs_frame], axis=1)
# Evaluate the long and short events in the price frame
for signal in active_signals:
vexec(pf, signal)
# Initialize trading state variables
inlong = False
inshort = False
h = 0
p = 0
q = quantity
tradelist = []
# Loop through prices and generate trades
for dt, row in pf.iterrows():
# get closing price
c = row['close']
if intraday:
bar_number = row['bar_number']
end_of_day = row['end_of_day']
# evaluate entry and exit conditions
lerow = row[longentry] if longentry else None
serow = row[shortentry] if shortentry else None
lxrow = row[longexit] if longexit else None
sxrow = row[shortexit] if shortexit else None
# process the long and short events
if lerow:
if p < 0:
# short active, so exit short
tradelist.append((dt, [name, Orders.sx, -p, c]))
inshort = False
h = 0
p = 0
if p == 0 or scale:
# go long (again)
tradelist.append((dt, [name, Orders.le, q, c]))
inlong = True
p = p + q
elif serow:
if p > 0:
# long active, so exit long
tradelist.append((dt, [name, Orders.lx, -p, c]))
inlong = False
h = 0
p = 0
if p == 0 or scale:
# go short (again)
tradelist.append((dt, [name, Orders.se, -q, c]))
inshort = True
p = p - q
# check exit conditions
if inlong and h > 0 and lxrow:
# long active, so exit long
tradelist.append((dt, [name, Orders.lx, -p, c]))
inlong = False
h = 0
p = 0
if inshort and h > 0 and sxrow:
# short active, so exit short
tradelist.append((dt, [name, Orders.sx, -p, c]))
inshort = False
h = 0
p = 0
# if a holding period was given, then check for exit
if holdperiod and h >= holdperiod:
if inlong:
tradelist.append((dt, [name, Orders.lh, -p, c]))
inlong = False
if inshort:
tradelist.append((dt, [name, Orders.sh, -p, c]))
inshort = False
h = 0
p = 0
# increment the hold counter
if inlong or inshort:
h += 1
if intraday and end_of_day:
if inlong:
# long active, so exit long
tradelist.append((dt, [name, Orders.lx, -p, c]))
inlong = False
if inshort:
# short active, so exit short
tradelist.append((dt, [name, Orders.sx, -p, c]))
inshort = False
h = 0
p = 0
return tradelist
def run_system(model, system, group, quantity=1):
r"""Run a system for a given group, creating a trades frame.
Parameters
----------
model : alphapy.Model
The model object with specifications.
system : alphapy.System or str
The system to run, either a long/short system or a local one
identified by function name, e.g., 'open_range_breakout'.
group : alphapy.Group
The group of symbols to test.
quantity : float
The amount to trade for each symbol, e.g., number of shares
Returns
-------
tf : pandas.DataFrame
All of the trades for this ``group``.
"""
if system.__class__ == str:
system_name = system
else:
system_name = system.name
logger.info("Generating Trades for System %s", system_name)
# Unpack the model data.
directory = model.specs['directory']
extension = model.specs['extension']
separator = model.specs['separator']
# Extract the group information.
gname = group.name
gmembers = group.members
gspace = group.space
# Run the system for each member of the group
gtlist = []
for symbol in gmembers:
# generate the trades for this member
if system.__class__ == str:
try:
tlist = globals()[system_name](symbol, gspace, quantity)
except:
logger.info("Could not execute system for %s", symbol)
else:
# call default long/short system
tlist = long_short(system, symbol, gspace, quantity)
if tlist:
# create the local trades frame
df = DataFrame.from_items(tlist,
orient='index',
columns=Trade.states)
# add trades to global trade list
for item in tlist:
gtlist.append(item)
else:
logger.info("No trades for symbol %s", symbol)
# Create group trades frame
tf = None
if gtlist:
tspace = Space(system_name, "trades", group.space.fractal)
gtlist = sorted(gtlist, key=lambda x: x[0])
tf = DataFrame.from_items(gtlist, orient='index', columns=Trade.states)
tfname = frame_name(gname, tspace)
system_dir = SSEP.join([directory, 'systems'])
write_frame(tf, system_dir, tfname, extension, separator, index=True)
del tspace
else:
logger.info("No trades were found")
# Return trades frame
return tf
def open_range_breakout(name, space, quantity):
r"""Run an Opening Range Breakout (ORB) system.
An ORB system is an intraday strategy that waits for price to
"break out" in a certain direction after establishing an
initial High-Low range. The timing of the trade is either
time-based (e.g., 30 minutes after the Open) or price-based
(e.g., 20% of the average daily range). Either the position
is held until the end of the trading day, or the position is
closed with a stop loss (e.g., the other side of the opening
range).
Parameters
----------
name : str
The symbol to trade.
space : alphapy.Space
Namespace of instrument prices.
quantity : float
The amount of the ``name`` to trade, e.g., number of shares
Returns
-------
tradelist : list
List of trade entries and exits.
Other Parameters
----------------
Frame.frames : dict
All of the data frames containing price data.
"""
# system parameters
trigger_first = 7
trigger_last = 56
# price frame
pf = Frame.frames[frame_name(name, space)].df
# initialize the trade list
tradelist = []
# generate trade file
for dt, row in pf.iterrows():
# extract data from row
bar_number = row['bar_number']
h = row['high']
l = row['low']
c = row['close']
end_of_day = row['end_of_day']
# open range breakout
if bar_number == 0:
# new day
traded = False
inlong = False
inshort = False
hh = h
ll = l
elif bar_number < trigger_first:
# set opening range
if h > hh:
hh = h
if l < ll:
ll = l
else:
if not traded and bar_number < trigger_last:
# trigger trade
if h > hh:
# long breakout triggers
tradelist.append((dt, [name, Orders.le, quantity, hh]))
inlong = True
traded = True
if l < ll and not traded:
# short breakout triggers
tradelist.append((dt, [name, Orders.se, -quantity, ll]))
inshort = True
traded = True
# test stop loss
if inlong and l < ll:
tradelist.append((dt, [name, Orders.lx, -quantity, ll]))
inlong = False
if inshort and h > hh:
tradelist.append((dt, [name, Orders.sx, quantity, hh]))
inshort = False
# exit any positions at the end of the day
if inlong and end_of_day:
# long active, so exit long
tradelist.append((dt, [name, Orders.lx, -quantity, c]))
if inshort and end_of_day:
# short active, so exit short
tradelist.append((dt, [name, Orders.sx, quantity, c]))
return tradelist
def long_short(system, name, space, quantity):
r"""Run a long/short system.
A long/short system is always in the market. At any given
time, either a long position is active, or a short position
is active.
Parameters
----------
system : alphapy.System
The long/short system to run.
name : str
The symbol to trade.
space : alphapy.Space
Namespace of instrument prices.
quantity : float
The amount of the ``name`` to trade, e.g., number of shares
Returns
-------
tradelist : list
List of trade entries and exits.
Other Parameters
----------------
Frame.frames : dict
All of the data frames containing price data.
"""
# extract the system parameters
longentry = system.longentry
shortentry = system.shortentry
longexit = system.longexit
shortexit = system.shortexit
holdperiod = system.holdperiod
scale = system.scale
# price frame
pf = Frame.frames[frame_name(name, space)].df
# initialize the trade list
tradelist = []
# evaluate the long and short events
if longentry:
vexec(pf, longentry)
if shortentry:
vexec(pf, shortentry)
if longexit:
vexec(pf, longexit)
if shortexit:
vexec(pf, shortexit)
# generate trade file
inlong = False
inshort = False
h = 0
p = 0
q = quantity
for dt, row in pf.iterrows():
# evaluate entry and exit conditions
lerow = None
if longentry:
lerow = row[longentry]
serow = None
if shortentry:
serow = row[shortentry]
lxrow = None
if longexit:
lxrow = row[longexit]
sxrow = None
if shortexit:
sxrow = row[shortexit]
# get closing price
c = row['close']
# process the long and short events
if lerow:
if p < 0:
# short active, so exit short
tradelist.append((dt, [name, Orders.sx, -p, c]))
inshort = False
h = 0
p = 0
if p == 0 or scale:
# go long (again)
tradelist.append((dt, [name, Orders.le, q, c]))
inlong = True
p = p + q
elif serow:
if p > 0:
# long active, so exit long
tradelist.append((dt, [name, Orders.lx, -p, c]))
inlong = False
h = 0
p = 0
if p == 0 or scale:
# go short (again)
tradelist.append((dt, [name, Orders.se, -q, c]))
inshort = True
p = p - q
# check exit conditions
if inlong and h > 0 and lxrow:
# long active, so exit long
tradelist.append((dt, [name, Orders.lx, -p, c]))
inlong = False
h = 0
p = 0
if inshort and h > 0 and sxrow:
# short active, so exit short
tradelist.append((dt, [name, Orders.sx, -p, c]))
inshort = False
h = 0
p = 0
# if a holding period was given, then check for exit
if holdperiod > 0 and h >= holdperiod:
if inlong:
tradelist.append((dt, [name, Orders.lh, -p, c]))
inlong = False
if inshort:
tradelist.append((dt, [name, Orders.sh, -p, c]))
inshort = False
h = 0
p = 0
# increment the hold counter
if inlong or inshort:
h += 1
return tradelist
def get_market_data(model, group, lookback_period, resample_data):
r"""Get data from an external feed.
Parameters
----------
model : alphapy.Model
The model object describing the data.
group : alphapy.Group
The group of symbols.
lookback_period : int
The number of periods of data to retrieve.
Returns
-------
n_periods : int
The maximum number of periods actually retrieved.
"""
# Unpack model specifications
directory = model.specs['directory']
extension = model.specs['extension']
separator = model.specs['separator']
# Unpack group elements
gspace = group.space
schema = gspace.schema
fractal = gspace.fractal
# Determine the feed source
if any(substring in fractal for substring in PD_INTRADAY_OFFSETS):
# intraday data (date and time)
logger.info("Getting Intraday Data [%s] from %s", fractal, schema)
intraday_data = True
index_column = 'datetime'
else:
# daily data or higher (date only)
logger.info("Getting Daily Data [%s] from %s", fractal, schema)
intraday_data = False
index_column = 'date'
# Get the data from the relevant feed
data_dir = SSEP.join([directory, 'data'])
pandas_data = any(substring in schema for substring in PD_WEB_DATA_FEEDS)
n_periods = 0
for item in group.members:
logger.info("Getting %s data for last %d days", item, lookback_period)
# Locate the data source
if schema == 'data':
fname = frame_name(item.lower(), gspace)
df = read_frame(data_dir, fname, extension, separator)
if not intraday_data:
df.set_index(pd.DatetimeIndex(df[index_column]),
drop=True,
inplace=True)
elif schema == 'google' and intraday_data:
df = get_google_data(item, lookback_period, fractal)
elif pandas_data:
df = get_pandas_data(schema, item, lookback_period)
else:
logger.error("Unsupported Data Source: %s", schema)
# Now that we have content, standardize the data
if df is not None and not df.empty:
logger.info("Rows: %d", len(df))
# standardize column names
df = df.rename(columns=lambda x: x.lower().replace(' ', ''))
# add intraday columns if necessary
if intraday_data:
df = enhance_intraday_data(df)
# order by increasing date if necessary
df = df.sort_index()
# resample data
if resample_data:
df = df.resample(fractal).agg({
'open': 'first',
'high': 'max',
'low': 'min',
'close': 'last',
'volume': 'sum'
})
logger.info("Rows after Resampling at %s: %d", fractal,
len(df))
# allocate global Frame
newf = Frame(item.lower(), gspace, df)
if newf is None:
logger.error("Could not allocate Frame for: %s", item)
# calculate maximum number of periods
df_len = len(df)
if df_len > n_periods:
n_periods = df_len
else:
logger.info("No DataFrame for %s", item)
# The number of periods actually retrieved
return n_periods
def exec_trade(p, name, order, quantity, price, tdate):
r"""Execute a trade for a portfolio.
Parameters
----------
p : alphapy.Portfolio
Portfolio in which to trade.
name : str
The symbol to trade.
order : alphapy.Orders
Long or short trade for entry or exit.
quantity : int
The quantity for the order.
price : str
The execution price of the trade.
tdate : datetime
The date and time of the trade.
Returns
-------
tsize : float
The executed trade size.
Other Parameters
----------------
Frame.frames : dict
Dataframe for the price data.
"""
# see if the position already exists
if name in p.positions:
pos = p.positions[name]
newpos = False
else:
pos = Position(p, name, tdate)
newpos = True
# check the dynamic position sizing variable
if not p.posby:
tsize = quantity
else:
if order == Orders.le or order == Orders.se:
pf = Frame.frames[frame_name(name, p.space)].df
cv = float(pf.ix[tdate][p.posby])
tsize = math.trunc((p.value * p.fixedfrac) / cv)
if quantity < 0:
tsize = -tsize
else:
tsize = -pos.quantity
# instantiate and allocate the trade
newtrade = Trade(name, order, tsize, price, tdate)
allocation = allocate_trade(p, pos, newtrade)
if allocation != 0:
# create a new position if necessary
if newpos:
p = add_position(p, name, pos)
p.npos += 1
# update the portfolio
p = update_portfolio(p, pos, newtrade)
# if net position is zero, then close the position
pflat = pos.quantity == 0
if pflat:
p = close_position(p, pos, tdate)
p.npos -= 1
else:
logger.info("Trade Allocation for %s is 0", name)
# return trade size
return tsize
def gen_portfolio(model,
system,
group,
tframe,
startcap=100000,
posby='close'):
r"""Create a portfolio from a trades frame.
Parameters
----------
model : alphapy.Model
The model with specifications.
system : str
Name of the system.
group : alphapy.Group
The group of instruments in the portfolio.
tframe : pandas.DataFrame
The input trade list from running the system.
startcap : float
Starting capital.
posby : str
The position sizing column in the price dataframe.
Returns
-------
p : alphapy.Portfolio
The generated portfolio.
Raises
------
MemoryError
Could not allocate Portfolio.
Notes
-----
This function also generates the files required for analysis
by the *pyfolio* package:
* Returns File
* Positions File
* Transactions File
"""
logger.info("Creating Portfolio for System %s", system)
# Unpack the model data.
directory = model.specs['directory']
extension = model.specs['extension']
separator = model.specs['separator']
# Create the portfolio.
gname = group.name
gspace = group.space
gmembers = group.members
ff = 1.0 / len(gmembers)
p = Portfolio(gname,
system,
gspace,
startcap=startcap,
posby=posby,
restricted=False,
fixedfrac=ff)
if not p:
raise MemoryError("Could not allocate Portfolio")
# Build pyfolio data from the trades frame.
start = tframe.index[0]
end = tframe.index[-1]
trange = np.unique(
tframe.index.map(lambda x: x.date().strftime('%Y-%m-%d'))).tolist()
drange = date_range(start,
end).map(lambda x: x.date().strftime('%Y-%m-%d'))
# Initialize return, position, and transaction data.
rs = []
pcols = list(gmembers)
pcols.extend(['cash'])
pf = DataFrame(index=drange, columns=pcols).fillna(0.0)
ts = []
# Iterate through the date range, updating the portfolio.
for d in drange:
# process today's trades
if d in trange:
trades = tframe.ix[d]
if isinstance(trades, Series):
trades = DataFrame(trades).transpose()
for t in trades.iterrows():
tdate = t[0]
row = t[1]
tsize = exec_trade(p, row['name'], row['order'],
row['quantity'], row['price'], tdate)
if tsize != 0:
ts.append((d, [tsize, row['price'], row['name']]))
else:
logger.info("Trade could not be executed for %s",
row['name'])
# iterate through current positions
positions = p.positions
pfrow = pf.ix[d]
for key in positions:
pos = positions[key]
if pos.quantity > 0:
value = pos.value
else:
value = -pos.value
pfrow[pos.name] = value
pfrow['cash'] = p.cash
# update the portfolio returns
p = valuate_portfolio(p, d)
rs.append((d, [p.netreturn]))
# Create systems directory path
system_dir = SSEP.join([directory, 'systems'])
# Create and record the returns frame for this system.
logger.info("Recording Returns Frame")
rspace = Space(system, 'returns', gspace.fractal)
rf = DataFrame.from_items(rs, orient='index', columns=['return'])
rfname = frame_name(gname, rspace)
write_frame(rf,
system_dir,
rfname,
extension,
separator,
index=True,
index_label='date')
del rspace
# Record the positions frame for this system.
logger.info("Recording Positions Frame")
pspace = Space(system, 'positions', gspace.fractal)
pfname = frame_name(gname, pspace)
write_frame(pf,
system_dir,
pfname,
extension,
separator,
index=True,
index_label='date')
del pspace
# Create and record the transactions frame for this system.
logger.info("Recording Transactions Frame")
tspace = Space(system, 'transactions', gspace.fractal)
tf = DataFrame.from_items(ts,
orient='index',
columns=['amount', 'price', 'symbol'])
tfname = frame_name(gname, tspace)
write_frame(tf,
system_dir,
tfname,
extension,
separator,
index=True,
index_label='date')
del tspace
# Return the portfolio.
return p
def get_market_data(model, group, lookback_period,
data_fractal, intraday_data=False):
r"""Get data from an external feed.
Parameters
----------
model : alphapy.Model
The model object describing the data.
group : alphapy.Group
The group of symbols.
lookback_period : int
The number of periods of data to retrieve.
data_fractal : str
Pandas offset alias.
intraday_data : bool
If True, then get intraday data.
Returns
-------
n_periods : int
The maximum number of periods actually retrieved.
"""
# Unpack model specifications
directory = model.specs['directory']
extension = model.specs['extension']
separator = model.specs['separator']
# Unpack group elements
gspace = group.space
schema = gspace.schema
fractal = gspace.fractal
# Determine the feed source
if intraday_data:
# intraday data (date and time)
logger.info("Getting Intraday Data [%s] from %s", data_fractal, schema)
index_column = 'datetime'
else:
# daily data or higher (date only)
logger.info("Getting Daily Data [%s] from %s", data_fractal, schema)
index_column = 'date'
# Get the data from the relevant feed
data_dir = SSEP.join([directory, 'data'])
pandas_data = any(substring in schema for substring in PD_WEB_DATA_FEEDS)
n_periods = 0
resample_data = True if fractal != data_fractal else False
df = None
to_date = pd.to_datetime('today')
from_date = to_date - pd.to_timedelta(lookback_period, unit='d')
for item in group.members:
logger.info("Getting %s data for last %d days", item, lookback_period)
# Locate the data source
if schema == 'data':
# local intraday or daily
dspace = Space(gspace.subject, gspace.schema, data_fractal)
fname = frame_name(item.lower(), dspace)
df = read_frame(data_dir, fname, extension, separator)
elif schema == 'google' and intraday_data:
# intraday only
df = get_google_data(item, lookback_period, data_fractal)
elif pandas_data:
# daily only
df = get_pandas_data(schema, item, lookback_period)
else:
logger.error("Unsupported Data Source: %s", schema)
# Now that we have content, standardize the data
if df is not None and not df.empty:
logger.info("%d data points from %s to %s", len(df), from_date, to_date)
# convert data to canonical form
df = convert_data(df, index_column, intraday_data)
# resample data and forward fill any NA values
if resample_data:
df = df.resample(fractal).agg({'open' : 'first',
'high' : 'max',
'low' : 'min',
'close' : 'last',
'volume' : 'sum'})
df.dropna(axis=0, how='any', inplace=True)
logger.info("Rows after Resampling at %s: %d",
fractal, len(df))
# add intraday columns if necessary
if intraday_data:
df = enhance_intraday_data(df)
# allocate global Frame
newf = Frame(item.lower(), gspace, df)
if newf is None:
logger.error("Could not allocate Frame for: %s", item)
# calculate maximum number of periods
df_len = len(df)
if df_len > n_periods:
n_periods = df_len
else:
logger.info("No DataFrame for %s", item)
# The number of periods actually retrieved
return n_periods
def get_market_data(model,
market_specs,
group,
lookback_period,
intraday_data=False):
r"""Get data from an external feed.
Parameters
----------
model : alphapy.Model
The model object describing the data.
market_specs : dict
The specifications for controlling the MarketFlow pipeline.
group : alphapy.Group
The group of symbols.
lookback_period : int
The number of periods of data to retrieve.
intraday_data : bool
If True, then get intraday data.
Returns
-------
n_periods : int
The maximum number of periods actually retrieved.
"""
# Unpack market specifications
data_fractal = market_specs['data_fractal']
subschema = market_specs['subschema']
# Unpack model specifications
directory = model.specs['directory']
extension = model.specs['extension']
separator = model.specs['separator']
# Unpack group elements
gspace = group.space
schema = gspace.schema
fractal = gspace.fractal
# Determine the feed source
if intraday_data:
# intraday data (date and time)
logger.info("%s Intraday Data [%s] for %d periods", schema,
data_fractal, lookback_period)
index_column = 'datetime'
else:
# daily data or higher (date only)
logger.info("%s Daily Data [%s] for %d periods", schema, data_fractal,
lookback_period)
index_column = 'date'
# Get the data from the relevant feed
data_dir = SSEP.join([directory, 'data'])
n_periods = 0
resample_data = True if fractal != data_fractal else False
# Date Arithmetic
to_date = pd.to_datetime('today')
from_date = to_date - pd.to_timedelta(lookback_period, unit='d')
to_date = to_date.strftime('%Y-%m-%d')
from_date = from_date.strftime('%Y-%m-%d')
# Get the data from the specified data feed
df = pd.DataFrame()
for symbol in group.members:
logger.info("Getting %s data from %s to %s", symbol.upper(), from_date,
to_date)
# Locate the data source
if schema == 'data':
# local intraday or daily
dspace = Space(gspace.subject, gspace.schema, data_fractal)
fname = frame_name(symbol.lower(), dspace)
df = read_frame(data_dir, fname, extension, separator)
elif schema in data_dispatch_table.keys():
df = data_dispatch_table[schema](schema, subschema, symbol,
intraday_data, data_fractal,
from_date, to_date,
lookback_period)
else:
logger.error("Unsupported Data Source: %s", schema)
# Now that we have content, standardize the data
if not df.empty:
logger.info("Rows: %d [%s]", len(df), data_fractal)
# convert data to canonical form
df = convert_data(df, index_column, intraday_data)
# resample data and forward fill any NA values
if resample_data:
df = df.resample(fractal).agg({
'open': 'first',
'high': 'max',
'low': 'min',
'close': 'last',
'volume': 'sum'
})
df.dropna(axis=0, how='any', inplace=True)
logger.info("Rows after Resampling at %s: %d", fractal,
len(df))
# add intraday columns if necessary
if intraday_data:
df = enhance_intraday_data(df)
# allocate global Frame
newf = Frame(symbol.lower(), gspace, df)
if newf is None:
logger.error("Could not allocate Frame for: %s",
symbol.upper())
# calculate maximum number of periods
df_len = len(df)
if df_len > n_periods:
n_periods = df_len
else:
logger.info("No DataFrame for %s", symbol.upper())
# The number of periods actually retrieved
return n_periods
