def dlm_exogenous_r3(y, s, k, a, t, e, r): """ One way to use dlm :returns: x, s', w """ if not s: s = dict() s['dim'] = dimension(y) s = dlm_set_exog_hyperparams(s=s, r=r) y0, exog = split_exogenous(y=y) s['n_obs'] = 0 s['model'] = quietDlm([], printInfo=False) + trend( s['trend_degree'], s['discount']) + seasonality( s['period'], s['discount']) s['model'] = s['model'] + fixedAutoReg( degree=s['auto_degree'], name='ar', w=1.0) if exog: exog_wrapped = [[None if np.isnan(ex0) else ex0 for ex0 in exog]] s['model'] = s['model'] + dynamic(features=exog_wrapped, discount=0.99, name='exog') # Set's first exog if y is not None: y = wrap(y) assert dimension(y) == s['dim'], 'Cannot change dimension of data sent' s['n_obs'] += 1 y0, exog = split_exogenous(y=y) y0_passed_in = None if np.isnan( y0) else y0 # pydlm uses None for missing values s['model'].append([y0_passed_in]) if exog: exog_wrapped = [[None if np.isnan(ex0) else ex0 for ex0 in exog]] if s['n_obs'] > 1: s['model'].append( data=exog_wrapped, component='exog') # Don't get first exog twice num_obs = len(s['model'].data) if s.get('model') else 0 if num_obs % s['n_fit'] == s['n_fit'] - 1: _, _, s = dlm_exogenous_r3(y=None, s=s, k=k, a=a, t=t, e=10, r=r) s['model'].fitForwardFilter() return _dlm_exog_prediction_helper(s=s, k=k, y=y) if y is None: if dimension(y) == 1: s['model'].tune(maxit=20) # Don't tune if exogenous ... haven't got this to work s['model'].fit() return None, None, s
def _dlm_exog_prediction_helper(s, k: int, y: Y_TYPE): num_obs = len(s['model'].data) if s.get('model') else 0 if num_obs < s['n_burn']: y0, exog = split_exogenous(y) return [y0] * k, [abs(y0)] * k, s else: assert k == 1, 'only k==1 for now' # TODO: Fix to allow for k>1 y0, exog = split_exogenous(y) if exog: exog_passed_in = [None if np.isnan(ex0) else ex0 for ex0 in exog] x_mean, x_var = s['model'].predict( featureDict={'exog': exog_passed_in[0]}) else: x_mean, x_var = s['model'].predict() x = [x_mean[0, 0]] x_std = [math.sqrt(x_var[0, 0])] return x, x_std, s
def _dlm_prediction_helper(s, k: int, y: Y_TYPE): """ Calls down to predictN, if we are passed the warm-up stage """ num_obs = len(s['model'].data) if s.get('model') else 0 if num_obs < s['n_warm']: y0, _ = split_exogenous(y) return [y0] * k, [abs(y0)] * k, s else: x_mean, x_var = s['model'].predictN(N=k) x = list(x_mean) x_std = [math.sqrt(v) for v in x_var] return x, x_std, s
def dlm_univariate_r3(y, s: dict, k: int, a=None, t=None, e=None, r=None): """ Univariate filter - Uses the discounting method of H/W so, doesn't need to be fit as often - Discount factors are periodically tuned - The hyper-parameter controls 'auto_degree', 'trend_degree', 'period' :returns: x, x_std, s """ assert r is not None, 'Requires hyper-parameter (interpreted in dimension 3) ' if not s: s = dict() s = dlm_set_univariate_params(s=s, r=r) s['dim'] = dimension(y) s['n_obs'] = 0 s['model'] = dlm([], printInfo=False) + trend( s['trend_degree'], s['discount']) + seasonality( s['period'], s['discount']) s['model'] = s['model'] + fixedAutoReg( degree=s['auto_degree'], name='ar', w=1.0) if y is not None: s['n_obs'] += 1 assert isinstance(y, float) or len( y) == s['dim'], ' Cannot change dimension of input in flight ' y0, exog = split_exogenous(y=y) y0_passed_in = None if np.isnan( y0) else y0 # pydlm uses None for missing values s['model'].append([y0_passed_in]) num_obs = len(s['model'].data) if s.get('model') else 0 if num_obs % s['n_fit'] == s['n_fit'] - 1: # Perform periodic tuning of discount factors _, _, s = dlm_univariate_r3(y=None, s=s, k=k, a=a, t=t, e=1000, r=r) s['model'].fitForwardFilter() return _dlm_prediction_helper(s=s, k=k, y=y) if y is None and e > 60: s['model'].tune() # Tunes discount factors s['model'].fit() return None, None, s
def flux_auto(y, s, k, a, t, e, r): """ One way to use flux package - Contemporaneous y[1:] variables are used as exogenous 'X' in pmdarima - This only works for k=1 :returns: x, s', w """ if s is None: s = dict() s = flux_hyperparams(s=s, r=r) s = initialize_buffers(s=s, y=y) if y is not None: # Process observation and return prediction assert isinstance(y, float) or len( y) == s['dim'], ' Cannot change dimension of input in flight ' y0, exog = split_exogenous(y=y, dim=s['dim']) s = update_buffers(s=s, a=a, exog=exog, y0=y0) if True: # Always fit prior to prediction none_, s, _ = flux_auto(y=None, s=s, k=k, a=a, t=t, e=e, r=r) # Fit the model assert none_ is None return flux_or_last_value(s=s, k=k, exog=exog, y0=y0) if y is None: if len(s.get('buffer')) < s['n_burn']: s['model'] = None else: data = pd.DataFrame(columns=['y'], data=s.get('buffer')) s['model'] = pf.ARIMA(data=data, ar=s['ar'], ma=s['ma'], target='y', family=s['family']) _ = s['model'].fit("MLE") return None, s, None # Acknowledge that a fit was requested by returning x=None, w=None
def pmd_skater_factory(y:Y_TYPE, s:dict, k:int=1, a:A_TYPE=None, t:T_TYPE=None, e:E_TYPE=None, method: str= 'default', n_warm=50, model_params:dict=None)->(Union[List[float],None], Union[List[float],None], Any): """ Predict using both simultaneously observed and known in advance variables y: Y_TYPE scalar or list where y[1:] are interpreted as contemporaneously observed exogenous variables s: state k: Number of steps ahead to predict a: (optional) scalar or list of variables known k-steps in advance. When calling, provide the known variable k steps ahead, not the contemporaneous one. t: (optional) Time of observation. e: (optional) Maximum computation time (supply e>60 to give hint to do fitting) :returns: x [float] , s', scale [float] Remarks: - Model params cannot be changed after the first invocation. - Allows y=None to be used """ y = wrap(y) a = wrap(a) if not s.get('n_obs'): # Initialize s['n_obs'] = 0 s['model'] = None s['immutable'] = pmd_set_immutable(k=k, y=y, a=a, n_warm=n_warm) s['params'] = pmd_params(method=method) if model_params: s['params'].update(model_params) s['o'] = dict() # Observance else: pmd_check_consistent_usage(y=y,s=s,a=a,k=k) tick(s) if t is not None: pass # Other models might perform an evolution step here. Not applicable to PMDARIMA if y is not None: # Receive observation y[0], possibly exogenous y[1:] and possibly k-in-advance a[:] # Collect from queues the contemporaneous variables s['n_obs']+=1 y_t, z = split_exogenous(y) x_t, s['o'] = observance(y=y,o=s['o'],k=k,a=a) # Update the pmdarima model itself if x_t is not None: if s['model'] is not None: if x_t: s['model'].update([y_t], [x_t]) else: s['model'].update([y_t]) # Predict if s['model'] is None: # Fall back to last value if there is no model calibrated as yet x = [y_t]*k if len(s['o']['x']) > 5 + 2*k: Y = s['o']['y'][k+1:] X = s['o']['x'][k+1:] x_std = [ np.nanstd( [ xi[0]-yk[0] for xi, yk in zip( X, Y[j:] ) ] ) for j in range(1,k+1) ] else: x_std = [1.0]*k # Fallback to dreadful estimate else: # Predict forward, supplying known data if it exists if not a and not z: z_forward = None else: if not a: z_forward = [z]*k else: z_forward = [ list(z) + list(ai) for ai in s['o']['a'] ] # Add known k-steps ahead # This estimate could be improved by predicting z's and attenuating # It is only really a good idea for k=1 x, ntvls = s['model'].predict(n_periods=k, X=z_forward, return_conf_int=True, alpha=s['immutable']['alpha']) x_std = list([ ntvl[1] - ntvl[0] for ntvl in ntvls ]) # Fit tock(s) if pmd_it_is_time_to_fit(s=s, e=e): tick(s) X = s['o'].get('x') or None Y = s['o']['y'] s['model'] = pm.auto_arima(y=Y, X=X, **s['params']) print(s['model']) print(s['model'].arima_res_.params) pprint(tocks(s)) tock(s,'fit') pprint(tocks(s)) if y is not None: return list(x), list(x_std), s else: return None, None, s
def observance(y: [float], o: dict, k: int, a: [float] = None): """ This marshals the k-step ahead vector a and the contemporaneous y[1:] and returns a combined vector of all exogenous variables. It tracks a list of x and corresponding y, by putting a's in a FIFO queue and by caching the previous value of y[1:] :param o: state :param k: Number of steps ahead that a is provided :param y: :param a: :returns: x_t:[float] vector combining y[1:] with previously supplied a's """ yw = wrap(y) aw = wrap(a) if not o: o = { 'a': [None for _ in range(k)], 'z': None, # Stores the previous value of y[1:] 'x': list(), 'y': list() } y_t, z = split_exogenous(yw) # Get the contemporaneous variables from last observation if z: z_t = o.get('z') # The previously revealed exogenous variables o['z'] = z # Store for next time else: z = None z_t = None # Determine the known in advance variable pertaining to the present if aw: a_t = o['a'].pop( 0) # The known in advance variable pertaining to this time step o['a'].append(aw) # Put the k-ahead received a value(s) on the queue else: a = None a_t = None # Combine into exogenous variables ... but only if both arrived if aw and z: x_t = z_t + a_t if (z_t and a_t) else None elif aw and not z: x_t = a_t if a_t else None elif (not aw) and z: x_t = z_t if z_t else None elif (not aw) and not z: x_t = None if (not z) and (not aw): o['y'].append([y_t]) # Special case, no need to wait else: if x_t: o['x'].append(x_t) o['y'].append([y_t]) assert len(o['x']) == len(o['y']), "post-condition" return x_t, o