def get_resp_word_raw_values(self, hndl_Word=None): if hndl_Word == None: if len(self._resp_words) != 1: raise NameError else: hndl_Word = self._resp_words[0] key_ = raw_word_key(hndl_Word) return self.get_values_by_col(key_, filter_nulls=True)
def fdel(self):
log.info('WORDSET: Removing Response Words')
keys = [word_key(w) for w in self._resp_words]
keys += [raw_word_key(w) for w in self._resp_words]
keys += [prd_word_key(w) for w in self._resp_words]
del self._resp_words
for key_ in keys:
self.delete_column(key_, expect_metadata=False)
def set_response_words(self, respWords, refillResp=False):
self._resp_words = respWords
keys = [str(w) for w in self._resp_words]
log.info('RESULTS: Setting Results Response Words: {0}'.format(keys))
# Load Data if Saved
self.load()
for hndl_Word in self._resp_words:
key = raw_word_key(hndl_Word)
self.add_column(hndl_Word.get_raw_dates(), hndl_Word.get_raw_values(), key=key, force=refillResp)
def get_residual_data(self, hndl_Word): raw_key = raw_word_key(hndl_Word) prd_key = prd_word_key(hndl_Word) prd_ = self.get_series_by_col(prd_key, filter_nulls=False) raw_ = self.get_series_by_col(raw_key, filter_nulls=False) residuals = prd_ - raw_ filter_ = ~residuals.isnull() smoothed = savitzky_golay(np.array(prd_[filter_])) return (smoothed, prd_[filter_], raw_[filter_], residuals[filter_])
def fset(self, value): self._resp_words = value for hndl_Word in self._resp_words: # Add Raw Values dates = hndl_Word.get_raw_dates() values = hndl_Word.get_raw_values() raw_key = raw_word_key(hndl_Word) self.add_column(dates, values, key=raw_key, force=False) # Add Transformed Values dates = hndl_Word.get_series_dates() values = hndl_Word.get_series_values() key_ = word_key(hndl_Word) self.add_column(dates, values, key=key_, force=True)
def get_model_filter(self, mode=TRAINING): if mode == PREDICTION_INDEPENDENT: keys = [word_key(w) for w in self._pred_words] elif mode == PREDICTION_DEPENDENT: keys = [word_key(w) for w in self._pred_words] keys += [raw_word_key(w) for w in self._resp_words] elif mode == TRAINING: keys = [word_key(w) for w in self._pred_words] keys += [word_key(w) for w in self._resp_words] else: raise NameError and_ = lambda c1, c2: c1 & c2 return reduce(and_, [~self.col_array[key].isnull() for key in keys])
def plot_values(self, hndl_Word): ''' Plot the raw data, predicted data, and smoothed data (For testing) ''' from util_Testing import plot_data_series from handle_TestSeries import EMF_TestSeries_Handle raw_key = raw_word_key(hndl_Word) prd_key = prd_word_key(hndl_Word) raw = EMF_TestSeries_Handle() (raw.dates, raw.values) = self.get_values_by_col(raw_key, filter_nulls=True) prd = EMF_TestSeries_Handle() (prd.dates, prd.values) = self.get_values_by_col(prd_key, filter_nulls=True) smooth = EMF_TestSeries_Handle() smooth.values = savitzky_golay(np.array(prd.values)) smooth.dates = prd.dates plot_data_series(raw, prd, smooth)
def save_predictions(self, hndl_Model, hndl_Word): ''' Create the model predictions and store them in the array. ''' if hndl_Word.prediction_requires_raw_data(): filter_ = self.get_model_filter(mode=PREDICTION_DEPENDENT) raw_key = raw_word_key(hndl_Word) raw_data = np.array(self.col_array[raw_key][filter_]).reshape(-1) raw_predictions = hndl_Model.model.predict(self.get_word_arrays(mode=PREDICTION_DEPENDENT)[0]) else: filter_ = self.get_model_filter(mode=PREDICTION_INDEPENDENT) raw_data = None raw_predictions = hndl_Model.model.predict(self.get_word_arrays(mode=PREDICTION_INDEPENDENT)[0]) values = hndl_Word.hndl_Trns.reverse_transform_data(raw_data, raw_predictions) dates = self.find_prediction_dates(hndl_Word) prd_key = prd_word_key(hndl_Word) self.add_column(dates, values, key=prd_key, force=True)
def get_prediction_dates(self): keys = [raw_word_key(w) for w in self._resp_words] filter_ = self.col_array[keys[0]].isnull() for i in xrange(1, len(keys)): filter_ = filter_ | self.col_array[keys[1]].isnull() return np.array(self.col_array[filter_].index)
