def __init__(self, portfolio, name, opendate): space = portfolio.space self.date = opendate self.name = name self.status = 'opened' self.mpos = 'flat' self.quantity = 0 self.price = 0.0 self.value = 0.0 self.profit = 0.0 self.netreturn = 0.0 self.opened = opendate self.held = 0 self.costbasis = 0.0 self.trades = [] self.ntrades = 0 self.pdata = Frame.frames[frame_name(name, space)].df self.multiplier = MULTIPLIERS[space.subject]
def vapply(group, vname, vfuncs=None): r"""Apply a variable to multiple dataframes. Parameters ---------- group : alphapy.Group The input group. vname : str The variable to apply to the ``group``. vfuncs : dict, optional Dictionary of external modules and functions. Returns ------- None : None Other Parameters ---------------- Frame.frames : dict Global dictionary of dataframes See Also -------- vunapply """ # get all frame names to apply variables gnames = [item.lower() for item in group.members] # get all the precedent variables allv = vtree(vname) # apply the variables to each frame for g in gnames: fname = frame_name(g, group.space) if fname in Frame.frames: f = Frame.frames[fname].df if not f.empty: for v in allv: logger.debug("Applying variable %s to %s", v, g) f = vexec(f, v, vfuncs) else: logger.debug("Frame for %s is empty", g) else: logger.debug("Frame not found: %s", fname)
def vunapply(group, vname): r"""Remove a variable from multiple dataframes. Parameters ---------- group : alphapy.Group The input group. vname : str The variable to remove from the ``group``. Returns ------- None : None Other Parameters ---------------- Frame.frames : dict Global dictionary of dataframes See Also -------- vapply """ # get all frame names to apply variables gnames = [item.lower() for item in group.all_members()] # apply the variables to each frame for g in gnames: fname = frame_name(g, group.space) if fname in Frame.frames: f = Frame.frames[fname].df logger.info("Unapplying variable %s from %s", vname, g) if vname not in f.columns: logger.info("Variable %s not in %s frame", vname, g) else: estr = "Frame.frames['%s'].df = f.df.drop('%s', axis=1)" \ % (fname, vname) exec(estr) else: logger.info("Frame not found: %s", fname)
def run_system(model, system, group, intraday=False, quantity=1): r"""Run a system for a given group, creating a trades frame. Parameters ---------- model : alphapy.Model The model object with specifications. system : alphapy.System The system to run. group : alphapy.Group The group of symbols to trade. intraday : bool, optional If true, this is an intraday system. quantity : float, optional The amount to trade for each symbol, e.g., number of shares Returns ------- tf : pandas.DataFrame All of the trades for this ``group``. """ system_name = system.name logger.info("Generating Trades for System %s", system_name) # Unpack the model data. directory = model.specs['directory'] extension = model.specs['extension'] separator = model.specs['separator'] # Extract the group information. gname = group.name gmembers = group.members gspace = group.space # Run the system for each member of the group gtlist = [] for symbol in gmembers: # generate the trades for this member tlist = trade_system(model, system, gspace, intraday, symbol, quantity) if tlist: # add trades to global trade list for item in tlist: gtlist.append(item) else: logger.info("No trades for symbol %s", symbol) # Create group trades frame tf = None if gtlist: tspace = Space(system_name, "trades", group.space.fractal) gtlist = sorted(gtlist, key=lambda x: x[0]) tf = DataFrame.from_items(gtlist, orient='index', columns=Trade.states) tfname = frame_name(gname, tspace) system_dir = SSEP.join([directory, 'systems']) labels = ['date'] if intraday: labels.append('time') write_frame(tf, system_dir, tfname, extension, separator, index=True, index_label=labels) del tspace else: logger.info("No trades were found") # Return trades frame return tf
def trade_system(model, system, space, intraday, name, quantity): r"""Trade the given system. Parameters ---------- model : alphapy.Model The model object with specifications. system : alphapy.System The long/short system to run. space : alphapy.Space Namespace of instrument prices. intraday : bool If True, then run an intraday system. name : str The symbol to trade. quantity : float The amount of the ``name`` to trade, e.g., number of shares Returns ------- tradelist : list List of trade entries and exits. Other Parameters ---------------- Frame.frames : dict All of the data frames containing price data. """ # Unpack the model data. directory = model.specs['directory'] extension = model.specs['extension'] separator = model.specs['separator'] # Unpack the system parameters. longentry = system.longentry shortentry = system.shortentry longexit = system.longexit shortexit = system.shortexit holdperiod = system.holdperiod scale = system.scale # Determine whether or not this is a model-driven system. entries_and_exits = [longentry, shortentry, longexit, shortexit] active_signals = [x for x in entries_and_exits if x is not None] use_model = False for signal in active_signals: if any(x in signal for x in ['phigh', 'plow']): use_model = True # Read in the price frame pf = Frame.frames[frame_name(name, space)].df # Use model output probabilities as input to the system if use_model: # get latest probabilities file probs_dir = SSEP.join([directory, 'output']) file_path = most_recent_file(probs_dir, 'probabilities*') file_name = file_path.split(SSEP)[-1].split('.')[0] # read the probabilities frame and trim the price frame probs_frame = read_frame(probs_dir, file_name, extension, separator) pf = pf[-probs_frame.shape[0]:] probs_frame.index = pf.index probs_frame.columns = ['probability'] # add probability column to price frame pf = pd.concat([pf, probs_frame], axis=1) # Evaluate the long and short events in the price frame for signal in active_signals: vexec(pf, signal) # Initialize trading state variables inlong = False inshort = False h = 0 p = 0 q = quantity tradelist = [] # Loop through prices and generate trades for dt, row in pf.iterrows(): # get closing price c = row['close'] if intraday: bar_number = row['bar_number'] end_of_day = row['end_of_day'] # evaluate entry and exit conditions lerow = row[longentry] if longentry else None serow = row[shortentry] if shortentry else None lxrow = row[longexit] if longexit else None sxrow = row[shortexit] if shortexit else None # process the long and short events if lerow: if p < 0: # short active, so exit short tradelist.append((dt, [name, Orders.sx, -p, c])) inshort = False h = 0 p = 0 if p == 0 or scale: # go long (again) tradelist.append((dt, [name, Orders.le, q, c])) inlong = True p = p + q elif serow: if p > 0: # long active, so exit long tradelist.append((dt, [name, Orders.lx, -p, c])) inlong = False h = 0 p = 0 if p == 0 or scale: # go short (again) tradelist.append((dt, [name, Orders.se, -q, c])) inshort = True p = p - q # check exit conditions if inlong and h > 0 and lxrow: # long active, so exit long tradelist.append((dt, [name, Orders.lx, -p, c])) inlong = False h = 0 p = 0 if inshort and h > 0 and sxrow: # short active, so exit short tradelist.append((dt, [name, Orders.sx, -p, c])) inshort = False h = 0 p = 0 # if a holding period was given, then check for exit if holdperiod and h >= holdperiod: if inlong: tradelist.append((dt, [name, Orders.lh, -p, c])) inlong = False if inshort: tradelist.append((dt, [name, Orders.sh, -p, c])) inshort = False h = 0 p = 0 # increment the hold counter if inlong or inshort: h += 1 if intraday and end_of_day: if inlong: # long active, so exit long tradelist.append((dt, [name, Orders.lx, -p, c])) inlong = False if inshort: # short active, so exit short tradelist.append((dt, [name, Orders.sx, -p, c])) inshort = False h = 0 p = 0 return tradelist
def run_system(model, system, group, quantity=1): r"""Run a system for a given group, creating a trades frame. Parameters ---------- model : alphapy.Model The model object with specifications. system : alphapy.System or str The system to run, either a long/short system or a local one identified by function name, e.g., 'open_range_breakout'. group : alphapy.Group The group of symbols to test. quantity : float The amount to trade for each symbol, e.g., number of shares Returns ------- tf : pandas.DataFrame All of the trades for this ``group``. """ if system.__class__ == str: system_name = system else: system_name = system.name logger.info("Generating Trades for System %s", system_name) # Unpack the model data. directory = model.specs['directory'] extension = model.specs['extension'] separator = model.specs['separator'] # Extract the group information. gname = group.name gmembers = group.members gspace = group.space # Run the system for each member of the group gtlist = [] for symbol in gmembers: # generate the trades for this member if system.__class__ == str: try: tlist = globals()[system_name](symbol, gspace, quantity) except: logger.info("Could not execute system for %s", symbol) else: # call default long/short system tlist = long_short(system, symbol, gspace, quantity) if tlist: # create the local trades frame df = DataFrame.from_items(tlist, orient='index', columns=Trade.states) # add trades to global trade list for item in tlist: gtlist.append(item) else: logger.info("No trades for symbol %s", symbol) # Create group trades frame tf = None if gtlist: tspace = Space(system_name, "trades", group.space.fractal) gtlist = sorted(gtlist, key=lambda x: x[0]) tf = DataFrame.from_items(gtlist, orient='index', columns=Trade.states) tfname = frame_name(gname, tspace) system_dir = SSEP.join([directory, 'systems']) write_frame(tf, system_dir, tfname, extension, separator, index=True) del tspace else: logger.info("No trades were found") # Return trades frame return tf
def open_range_breakout(name, space, quantity): r"""Run an Opening Range Breakout (ORB) system. An ORB system is an intraday strategy that waits for price to "break out" in a certain direction after establishing an initial High-Low range. The timing of the trade is either time-based (e.g., 30 minutes after the Open) or price-based (e.g., 20% of the average daily range). Either the position is held until the end of the trading day, or the position is closed with a stop loss (e.g., the other side of the opening range). Parameters ---------- name : str The symbol to trade. space : alphapy.Space Namespace of instrument prices. quantity : float The amount of the ``name`` to trade, e.g., number of shares Returns ------- tradelist : list List of trade entries and exits. Other Parameters ---------------- Frame.frames : dict All of the data frames containing price data. """ # system parameters trigger_first = 7 trigger_last = 56 # price frame pf = Frame.frames[frame_name(name, space)].df # initialize the trade list tradelist = [] # generate trade file for dt, row in pf.iterrows(): # extract data from row bar_number = row['bar_number'] h = row['high'] l = row['low'] c = row['close'] end_of_day = row['end_of_day'] # open range breakout if bar_number == 0: # new day traded = False inlong = False inshort = False hh = h ll = l elif bar_number < trigger_first: # set opening range if h > hh: hh = h if l < ll: ll = l else: if not traded and bar_number < trigger_last: # trigger trade if h > hh: # long breakout triggers tradelist.append((dt, [name, Orders.le, quantity, hh])) inlong = True traded = True if l < ll and not traded: # short breakout triggers tradelist.append((dt, [name, Orders.se, -quantity, ll])) inshort = True traded = True # test stop loss if inlong and l < ll: tradelist.append((dt, [name, Orders.lx, -quantity, ll])) inlong = False if inshort and h > hh: tradelist.append((dt, [name, Orders.sx, quantity, hh])) inshort = False # exit any positions at the end of the day if inlong and end_of_day: # long active, so exit long tradelist.append((dt, [name, Orders.lx, -quantity, c])) if inshort and end_of_day: # short active, so exit short tradelist.append((dt, [name, Orders.sx, quantity, c])) return tradelist
def long_short(system, name, space, quantity): r"""Run a long/short system. A long/short system is always in the market. At any given time, either a long position is active, or a short position is active. Parameters ---------- system : alphapy.System The long/short system to run. name : str The symbol to trade. space : alphapy.Space Namespace of instrument prices. quantity : float The amount of the ``name`` to trade, e.g., number of shares Returns ------- tradelist : list List of trade entries and exits. Other Parameters ---------------- Frame.frames : dict All of the data frames containing price data. """ # extract the system parameters longentry = system.longentry shortentry = system.shortentry longexit = system.longexit shortexit = system.shortexit holdperiod = system.holdperiod scale = system.scale # price frame pf = Frame.frames[frame_name(name, space)].df # initialize the trade list tradelist = [] # evaluate the long and short events if longentry: vexec(pf, longentry) if shortentry: vexec(pf, shortentry) if longexit: vexec(pf, longexit) if shortexit: vexec(pf, shortexit) # generate trade file inlong = False inshort = False h = 0 p = 0 q = quantity for dt, row in pf.iterrows(): # evaluate entry and exit conditions lerow = None if longentry: lerow = row[longentry] serow = None if shortentry: serow = row[shortentry] lxrow = None if longexit: lxrow = row[longexit] sxrow = None if shortexit: sxrow = row[shortexit] # get closing price c = row['close'] # process the long and short events if lerow: if p < 0: # short active, so exit short tradelist.append((dt, [name, Orders.sx, -p, c])) inshort = False h = 0 p = 0 if p == 0 or scale: # go long (again) tradelist.append((dt, [name, Orders.le, q, c])) inlong = True p = p + q elif serow: if p > 0: # long active, so exit long tradelist.append((dt, [name, Orders.lx, -p, c])) inlong = False h = 0 p = 0 if p == 0 or scale: # go short (again) tradelist.append((dt, [name, Orders.se, -q, c])) inshort = True p = p - q # check exit conditions if inlong and h > 0 and lxrow: # long active, so exit long tradelist.append((dt, [name, Orders.lx, -p, c])) inlong = False h = 0 p = 0 if inshort and h > 0 and sxrow: # short active, so exit short tradelist.append((dt, [name, Orders.sx, -p, c])) inshort = False h = 0 p = 0 # if a holding period was given, then check for exit if holdperiod > 0 and h >= holdperiod: if inlong: tradelist.append((dt, [name, Orders.lh, -p, c])) inlong = False if inshort: tradelist.append((dt, [name, Orders.sh, -p, c])) inshort = False h = 0 p = 0 # increment the hold counter if inlong or inshort: h += 1 return tradelist
def get_market_data(model, group, lookback_period, resample_data): r"""Get data from an external feed. Parameters ---------- model : alphapy.Model The model object describing the data. group : alphapy.Group The group of symbols. lookback_period : int The number of periods of data to retrieve. Returns ------- n_periods : int The maximum number of periods actually retrieved. """ # Unpack model specifications directory = model.specs['directory'] extension = model.specs['extension'] separator = model.specs['separator'] # Unpack group elements gspace = group.space schema = gspace.schema fractal = gspace.fractal # Determine the feed source if any(substring in fractal for substring in PD_INTRADAY_OFFSETS): # intraday data (date and time) logger.info("Getting Intraday Data [%s] from %s", fractal, schema) intraday_data = True index_column = 'datetime' else: # daily data or higher (date only) logger.info("Getting Daily Data [%s] from %s", fractal, schema) intraday_data = False index_column = 'date' # Get the data from the relevant feed data_dir = SSEP.join([directory, 'data']) pandas_data = any(substring in schema for substring in PD_WEB_DATA_FEEDS) n_periods = 0 for item in group.members: logger.info("Getting %s data for last %d days", item, lookback_period) # Locate the data source if schema == 'data': fname = frame_name(item.lower(), gspace) df = read_frame(data_dir, fname, extension, separator) if not intraday_data: df.set_index(pd.DatetimeIndex(df[index_column]), drop=True, inplace=True) elif schema == 'google' and intraday_data: df = get_google_data(item, lookback_period, fractal) elif pandas_data: df = get_pandas_data(schema, item, lookback_period) else: logger.error("Unsupported Data Source: %s", schema) # Now that we have content, standardize the data if df is not None and not df.empty: logger.info("Rows: %d", len(df)) # standardize column names df = df.rename(columns=lambda x: x.lower().replace(' ', '')) # add intraday columns if necessary if intraday_data: df = enhance_intraday_data(df) # order by increasing date if necessary df = df.sort_index() # resample data if resample_data: df = df.resample(fractal).agg({ 'open': 'first', 'high': 'max', 'low': 'min', 'close': 'last', 'volume': 'sum' }) logger.info("Rows after Resampling at %s: %d", fractal, len(df)) # allocate global Frame newf = Frame(item.lower(), gspace, df) if newf is None: logger.error("Could not allocate Frame for: %s", item) # calculate maximum number of periods df_len = len(df) if df_len > n_periods: n_periods = df_len else: logger.info("No DataFrame for %s", item) # The number of periods actually retrieved return n_periods
def exec_trade(p, name, order, quantity, price, tdate): r"""Execute a trade for a portfolio. Parameters ---------- p : alphapy.Portfolio Portfolio in which to trade. name : str The symbol to trade. order : alphapy.Orders Long or short trade for entry or exit. quantity : int The quantity for the order. price : str The execution price of the trade. tdate : datetime The date and time of the trade. Returns ------- tsize : float The executed trade size. Other Parameters ---------------- Frame.frames : dict Dataframe for the price data. """ # see if the position already exists if name in p.positions: pos = p.positions[name] newpos = False else: pos = Position(p, name, tdate) newpos = True # check the dynamic position sizing variable if not p.posby: tsize = quantity else: if order == Orders.le or order == Orders.se: pf = Frame.frames[frame_name(name, p.space)].df cv = float(pf.ix[tdate][p.posby]) tsize = math.trunc((p.value * p.fixedfrac) / cv) if quantity < 0: tsize = -tsize else: tsize = -pos.quantity # instantiate and allocate the trade newtrade = Trade(name, order, tsize, price, tdate) allocation = allocate_trade(p, pos, newtrade) if allocation != 0: # create a new position if necessary if newpos: p = add_position(p, name, pos) p.npos += 1 # update the portfolio p = update_portfolio(p, pos, newtrade) # if net position is zero, then close the position pflat = pos.quantity == 0 if pflat: p = close_position(p, pos, tdate) p.npos -= 1 else: logger.info("Trade Allocation for %s is 0", name) # return trade size return tsize
def gen_portfolio(model, system, group, tframe, startcap=100000, posby='close'): r"""Create a portfolio from a trades frame. Parameters ---------- model : alphapy.Model The model with specifications. system : str Name of the system. group : alphapy.Group The group of instruments in the portfolio. tframe : pandas.DataFrame The input trade list from running the system. startcap : float Starting capital. posby : str The position sizing column in the price dataframe. Returns ------- p : alphapy.Portfolio The generated portfolio. Raises ------ MemoryError Could not allocate Portfolio. Notes ----- This function also generates the files required for analysis by the *pyfolio* package: * Returns File * Positions File * Transactions File """ logger.info("Creating Portfolio for System %s", system) # Unpack the model data. directory = model.specs['directory'] extension = model.specs['extension'] separator = model.specs['separator'] # Create the portfolio. gname = group.name gspace = group.space gmembers = group.members ff = 1.0 / len(gmembers) p = Portfolio(gname, system, gspace, startcap=startcap, posby=posby, restricted=False, fixedfrac=ff) if not p: raise MemoryError("Could not allocate Portfolio") # Build pyfolio data from the trades frame. start = tframe.index[0] end = tframe.index[-1] trange = np.unique( tframe.index.map(lambda x: x.date().strftime('%Y-%m-%d'))).tolist() drange = date_range(start, end).map(lambda x: x.date().strftime('%Y-%m-%d')) # Initialize return, position, and transaction data. rs = [] pcols = list(gmembers) pcols.extend(['cash']) pf = DataFrame(index=drange, columns=pcols).fillna(0.0) ts = [] # Iterate through the date range, updating the portfolio. for d in drange: # process today's trades if d in trange: trades = tframe.ix[d] if isinstance(trades, Series): trades = DataFrame(trades).transpose() for t in trades.iterrows(): tdate = t[0] row = t[1] tsize = exec_trade(p, row['name'], row['order'], row['quantity'], row['price'], tdate) if tsize != 0: ts.append((d, [tsize, row['price'], row['name']])) else: logger.info("Trade could not be executed for %s", row['name']) # iterate through current positions positions = p.positions pfrow = pf.ix[d] for key in positions: pos = positions[key] if pos.quantity > 0: value = pos.value else: value = -pos.value pfrow[pos.name] = value pfrow['cash'] = p.cash # update the portfolio returns p = valuate_portfolio(p, d) rs.append((d, [p.netreturn])) # Create systems directory path system_dir = SSEP.join([directory, 'systems']) # Create and record the returns frame for this system. logger.info("Recording Returns Frame") rspace = Space(system, 'returns', gspace.fractal) rf = DataFrame.from_items(rs, orient='index', columns=['return']) rfname = frame_name(gname, rspace) write_frame(rf, system_dir, rfname, extension, separator, index=True, index_label='date') del rspace # Record the positions frame for this system. logger.info("Recording Positions Frame") pspace = Space(system, 'positions', gspace.fractal) pfname = frame_name(gname, pspace) write_frame(pf, system_dir, pfname, extension, separator, index=True, index_label='date') del pspace # Create and record the transactions frame for this system. logger.info("Recording Transactions Frame") tspace = Space(system, 'transactions', gspace.fractal) tf = DataFrame.from_items(ts, orient='index', columns=['amount', 'price', 'symbol']) tfname = frame_name(gname, tspace) write_frame(tf, system_dir, tfname, extension, separator, index=True, index_label='date') del tspace # Return the portfolio. return p
def get_market_data(model, group, lookback_period, data_fractal, intraday_data=False): r"""Get data from an external feed. Parameters ---------- model : alphapy.Model The model object describing the data. group : alphapy.Group The group of symbols. lookback_period : int The number of periods of data to retrieve. data_fractal : str Pandas offset alias. intraday_data : bool If True, then get intraday data. Returns ------- n_periods : int The maximum number of periods actually retrieved. """ # Unpack model specifications directory = model.specs['directory'] extension = model.specs['extension'] separator = model.specs['separator'] # Unpack group elements gspace = group.space schema = gspace.schema fractal = gspace.fractal # Determine the feed source if intraday_data: # intraday data (date and time) logger.info("Getting Intraday Data [%s] from %s", data_fractal, schema) index_column = 'datetime' else: # daily data or higher (date only) logger.info("Getting Daily Data [%s] from %s", data_fractal, schema) index_column = 'date' # Get the data from the relevant feed data_dir = SSEP.join([directory, 'data']) pandas_data = any(substring in schema for substring in PD_WEB_DATA_FEEDS) n_periods = 0 resample_data = True if fractal != data_fractal else False df = None to_date = pd.to_datetime('today') from_date = to_date - pd.to_timedelta(lookback_period, unit='d') for item in group.members: logger.info("Getting %s data for last %d days", item, lookback_period) # Locate the data source if schema == 'data': # local intraday or daily dspace = Space(gspace.subject, gspace.schema, data_fractal) fname = frame_name(item.lower(), dspace) df = read_frame(data_dir, fname, extension, separator) elif schema == 'google' and intraday_data: # intraday only df = get_google_data(item, lookback_period, data_fractal) elif pandas_data: # daily only df = get_pandas_data(schema, item, lookback_period) else: logger.error("Unsupported Data Source: %s", schema) # Now that we have content, standardize the data if df is not None and not df.empty: logger.info("%d data points from %s to %s", len(df), from_date, to_date) # convert data to canonical form df = convert_data(df, index_column, intraday_data) # resample data and forward fill any NA values if resample_data: df = df.resample(fractal).agg({'open' : 'first', 'high' : 'max', 'low' : 'min', 'close' : 'last', 'volume' : 'sum'}) df.dropna(axis=0, how='any', inplace=True) logger.info("Rows after Resampling at %s: %d", fractal, len(df)) # add intraday columns if necessary if intraday_data: df = enhance_intraday_data(df) # allocate global Frame newf = Frame(item.lower(), gspace, df) if newf is None: logger.error("Could not allocate Frame for: %s", item) # calculate maximum number of periods df_len = len(df) if df_len > n_periods: n_periods = df_len else: logger.info("No DataFrame for %s", item) # The number of periods actually retrieved return n_periods
def get_market_data(model, market_specs, group, lookback_period, intraday_data=False): r"""Get data from an external feed. Parameters ---------- model : alphapy.Model The model object describing the data. market_specs : dict The specifications for controlling the MarketFlow pipeline. group : alphapy.Group The group of symbols. lookback_period : int The number of periods of data to retrieve. intraday_data : bool If True, then get intraday data. Returns ------- n_periods : int The maximum number of periods actually retrieved. """ # Unpack market specifications data_fractal = market_specs['data_fractal'] subschema = market_specs['subschema'] # Unpack model specifications directory = model.specs['directory'] extension = model.specs['extension'] separator = model.specs['separator'] # Unpack group elements gspace = group.space schema = gspace.schema fractal = gspace.fractal # Determine the feed source if intraday_data: # intraday data (date and time) logger.info("%s Intraday Data [%s] for %d periods", schema, data_fractal, lookback_period) index_column = 'datetime' else: # daily data or higher (date only) logger.info("%s Daily Data [%s] for %d periods", schema, data_fractal, lookback_period) index_column = 'date' # Get the data from the relevant feed data_dir = SSEP.join([directory, 'data']) n_periods = 0 resample_data = True if fractal != data_fractal else False # Date Arithmetic to_date = pd.to_datetime('today') from_date = to_date - pd.to_timedelta(lookback_period, unit='d') to_date = to_date.strftime('%Y-%m-%d') from_date = from_date.strftime('%Y-%m-%d') # Get the data from the specified data feed df = pd.DataFrame() for symbol in group.members: logger.info("Getting %s data from %s to %s", symbol.upper(), from_date, to_date) # Locate the data source if schema == 'data': # local intraday or daily dspace = Space(gspace.subject, gspace.schema, data_fractal) fname = frame_name(symbol.lower(), dspace) df = read_frame(data_dir, fname, extension, separator) elif schema in data_dispatch_table.keys(): df = data_dispatch_table[schema](schema, subschema, symbol, intraday_data, data_fractal, from_date, to_date, lookback_period) else: logger.error("Unsupported Data Source: %s", schema) # Now that we have content, standardize the data if not df.empty: logger.info("Rows: %d [%s]", len(df), data_fractal) # convert data to canonical form df = convert_data(df, index_column, intraday_data) # resample data and forward fill any NA values if resample_data: df = df.resample(fractal).agg({ 'open': 'first', 'high': 'max', 'low': 'min', 'close': 'last', 'volume': 'sum' }) df.dropna(axis=0, how='any', inplace=True) logger.info("Rows after Resampling at %s: %d", fractal, len(df)) # add intraday columns if necessary if intraday_data: df = enhance_intraday_data(df) # allocate global Frame newf = Frame(symbol.lower(), gspace, df) if newf is None: logger.error("Could not allocate Frame for: %s", symbol.upper()) # calculate maximum number of periods df_len = len(df) if df_len > n_periods: n_periods = df_len else: logger.info("No DataFrame for %s", symbol.upper()) # The number of periods actually retrieved return n_periods