def __sub__(self, mat): # returns (cls - mat) # tmp = cls - tmp(=mat) tmp = FrovedisBlockcyclicMatrix(mat=mat) # copy if (self.get_dtype() != tmp.get_dtype()): raise TypeError("sub: input matrix types are not same!") # geadd performs B = al*A + be*B, thus tmp = B and tmp = A - tmp (host, port) = FrovedisServer.getServerInstance() rpclib.pgeadd(host, port, self.get(), tmp.get(), False, 1.0, -1.0, self.get_dtype()) excpt = rpclib.check_server_exception() if excpt["status"]: raise RuntimeError(excpt["info"]) return tmp
def load(self, fname): """ NAME: load """ self.release() self.__mid = ModelID.get() (host, port) = FrovedisServer.getServerInstance() rpclib.load_frovedis_model(host, port, self.__mid, self.__mkind, DTYPE.DOUBLE, fname.encode('ascii')) excpt = rpclib.check_server_exception() if excpt["status"]: raise RuntimeError(excpt["info"]) return self
def release(self): """release""" if self.__fdata is not None: (host, port) = FrovedisServer.getServerInstance() rpclib.release_frovedis_dense_matrix(host, port, self.get(), self.__mtype.encode('ascii'), self.get_dtype()) excpt = rpclib.check_server_exception() if excpt["status"]: raise RuntimeError(excpt["info"]) self.__fdata = None self.__num_row = 0 self.__num_col = 0
def to_frovedis_rowmatrix(self): """to_frovedis_rowmatrix""" if self.__mtype == 'R': return self (host, port) = FrovedisServer.getServerInstance() dmat = rpclib.get_frovedis_rowmatrix(host, port, self.get(), self.numRows(), self.numCols(), self.__mtype.encode('ascii'), self.get_dtype()) excpt = rpclib.check_server_exception() if excpt["status"]: raise RuntimeError(excpt["info"]) return FrovedisDenseMatrix(mtype='R', mat=dmat, dtype=self.__dtype)
def fittedvalues(self): """ DESC: fittedvalues getter RETURNS: TYPE: ndarray of shape (n_samples,), returns the fitted values of the model """ if self._fittedvalues is None: (host, port) = FrovedisServer.getServerInstance() ret = rpclib.get_fitted_vector(host, port, self.__mid, \ self.__mkind, self.__mdtype) excpt = rpclib.check_server_exception() if excpt["status"]: raise RuntimeError(excpt["info"]) self._fittedvalues = np.asarray(ret, dtype=np.float64) return self._fittedvalues
def transpose(self): """transpose""" if self.__fdata is not None: (host, port) = FrovedisServer.getServerInstance() dmat = rpclib.transpose_frovedis_dense_matrix( host, port, self.get(), self.__mtype.encode('ascii'), self.get_dtype()) excpt = rpclib.check_server_exception() if excpt["status"]: raise RuntimeError(excpt["info"]) return FrovedisDenseMatrix(mtype=self.__mtype, mat=dmat, dtype=self.__dtype) else: raise ValueError("Empty input matrix.")
def fit(self, data): """ NAME: fit """ if self.minSupport < 0: raise ValueError("Negative minsupport factor!") self.release() self.__mid = ModelID.get() f_df = self.__convert_to_df(data) (host, port) = FrovedisServer.getServerInstance() rpclib.fpgrowth_trainer(host, port, f_df.get(), self.__mid, \ self.minSupport, self.verbose) excpt = rpclib.check_server_exception() if excpt["status"]: raise RuntimeError(excpt["info"]) return self
def copy(self, mat): # cls = mat """copy""" self.release() if self.__dtype is None: self.__dtype = mat.__dtype if mat.__mtype != self.__mtype or mat.__dtype != self.__dtype: raise TypeError("Incompatible types for copy operation") if mat.__fdata is not None: (host, port) = FrovedisServer.getServerInstance() dmat = rpclib.copy_frovedis_dense_matrix( host, port, mat.get(), mat.__mtype.encode('ascii'), mat.get_dtype()) excpt = rpclib.check_server_exception() if excpt["status"]: raise RuntimeError(excpt["info"]) return self.load_dummy(dmat)
def load_binary(self, fname, dtype=None): """load_binary""" self.release() if dtype is None: dtype = self.__dtype else: self.__dtype = dtype (host, port) = FrovedisServer.getServerInstance() if self.__dtype is None: self.__dtype = np.float32 # default type: float dmat = rpclib.load_frovedis_dense_matrix(host, port, fname.encode("ascii"), True, self.__mtype.encode('ascii'), self.get_dtype()) excpt = rpclib.check_server_exception() if excpt["status"]: raise RuntimeError(excpt["info"]) return self.load_dummy(dmat)
def predict(self, start=None, end=None, dynamic=False, **kwargs): """ DESC: Perform in-sample prediction and out-of-sample forecasting PARAMS: start -> TYPE: int, DEFAULT: None, it specifies the staring index after which the values are to be predicted stop -> TYPE: int, DEFAULT: None, it specifies the index till which the values are to be predicted dynamic -> DEFAULT: False, (Unused) **kwargs -> (Unused) NOTE: In case start or stop are negative then predicted values are returned from fitted values as long as indexes are accessible(in range). This is in sync with statsmodel behaviour. RETURNS: TYPE: ndarray of shape (n_samples,), it returns the predicted values """ if start is None: start = 0 elif start < 0: if self._endog_len >= abs(start): start = self._endog_len + start else: raise KeyError("The `start` argument could not be matched " + \ "to a location related to the index of " + \ "the data.") if end is None: end = self._endog_len - 1 elif end < 0: if self._endog_len >= abs(end): end = self._endog_len + end else: raise KeyError("The `end` argument could not be matched to " + \ "a location related to the index of the data.") if end < start: raise ValueError("Prediction must have `end` after `start`!") if dynamic: raise ValueError("Currently, ARIMA.predict() does not support " + \ "dynamic = True!") (host, port) = FrovedisServer.getServerInstance() arima_pred = rpclib.arima_predict(host, port, start, end, \ self.__mid, self.__mdtype) excpt = rpclib.check_server_exception() if excpt["status"]: raise RuntimeError(excpt["info"]) return np.asarray(arima_pred, dtype = np.float64)
def fit(self): """ DESC: Fit (estimate) the parameters of the model RETURNS: TYPE: list of shape (n_samples,), returns the fitted values of the model """ self.reset_metadata() if len(self.order) != 3: raise ValueError("`order` argument must be an iterable with " + \ "three elements.") if self.ar_lag < 1: raise ValueError("Terms in the AR order cannot be less than 1.") if self.diff_order < 0: raise ValueError("Cannot specify negative differencing.") if self.ma_lag < 0: raise ValueError("Terms in the MA order cannot be negative.") if self.seasonal is None: self.seasonal = 0 elif self.seasonal < 0: raise ValueError("The seasonal differencing interval cannot " + \ "be negative, given: " + str(self.seasonal)) if self.auto_arima is True and self.ma_lag <= 0: raise ValueError("Currently, auto_arima cannot start with a " + \ "moving average component having value less " + \ "than 1!") if self.solver == 'sag': self.solver = 'sgd' elif self.solver not in ['lapack', 'lbfgs', 'scalapack']: raise ValueError("Unknown solver: " + self.solver + " for time " + \ "series analysis!") if isinstance(self.endog, FrovedisDvector): self.__mdtype = self.endog.get_dtype() inp_data = self.endog.to_numpy_array() shape = np.shape(inp_data) if np.shape(inp_data)[0] < (self.ar_lag + self.diff_order + \ self.ma_lag + self.seasonal + 2): raise ValueError("Number of samples in input is too less " + \ "for time series analysis!") self._endog_len = shape[0] elif isinstance(self.endog, (FrovedisCRSMatrix, FrovedisDenseMatrix, FrovedisRowmajorMatrix, FrovedisColmajorMatrix)): raise TypeError("endog can only be FrovedisDvector, " + \ "not {}".format(self.endog)) else: shape = np.shape(self.endog) if len(shape) == 1 or (len(shape) == 2 and shape[1] == 1): if np.shape(self.endog)[0] < (self.ar_lag + self.diff_order + \ self.ma_lag + self.seasonal + 2): raise ValueError("Number of samples in input is too " + \ "less for time series analysis!") self.endog = np.ravel(self.endog) self._endog_len = shape[0] self.endog = FrovedisDvector().as_dvec(self.endog) self.__mdtype = self.endog.get_dtype() else: raise ValueError("Frovedis ARIMA models require univariate " + \ "`endog`. Got shape {0}".format(shape)) self.__mkind = M_KIND.ARM (host, port) = FrovedisServer.getServerInstance() rpclib.arima_fit(host, port, self.endog.get(), self.ar_lag, \ self.diff_order, self.ma_lag, self.seasonal, \ self.auto_arima, str_encode(self.solver), \ self.verbose, self.__mid, self.__mdtype) excpt = rpclib.check_server_exception() if excpt["status"]: raise RuntimeError(excpt["info"]) self.isfitted = True return self
def __to_numpy_data_inplace(self, data, is_ndarray=True): """ non-retuning function to overwrite input numpy matrix/ndarray with converted matrix. self.size needs to be matched with data.size self.__dtype needs to be matched with data.dtype """ if self.__fdata is not None: data = np.asmatrix(data) # doesn't copy. it is needed #to get flattened array A1 if data.size != self.size: raise ValueError(\ "input matrix/ndarray size is different than self size!") (host, port) = FrovedisServer.getServerInstance() arr = data.A1 #getting the flatten array from numpy matrix data_size = self.size inp_type = self.get_dtype() out_type = TypeUtil.to_id_dtype(data.dtype) # rpc functions overwrite the arr data in C-level if inp_type == DTYPE.DOUBLE and out_type == DTYPE.INT: rpclib.get_double_rowmajor_array_as_int_array( host, port, self.get(), self.__mtype.encode('ascii'), arr, data_size) elif inp_type == DTYPE.DOUBLE and out_type == DTYPE.LONG: rpclib.get_double_rowmajor_array_as_long_array( host, port, self.get(), self.__mtype.encode('ascii'), arr, data_size) elif inp_type == DTYPE.DOUBLE and out_type == DTYPE.FLOAT: rpclib.get_double_rowmajor_array_as_float_array( host, port, self.get(), self.__mtype.encode('ascii'), arr, data_size) elif inp_type == DTYPE.DOUBLE and out_type == DTYPE.DOUBLE: rpclib.get_double_rowmajor_array_as_double_array( host, port, self.get(), self.__mtype.encode('ascii'), arr, data_size) elif inp_type == DTYPE.FLOAT and out_type == DTYPE.INT: rpclib.get_float_rowmajor_array_as_int_array( host, port, self.get(), self.__mtype.encode('ascii'), arr, data_size) elif inp_type == DTYPE.FLOAT and out_type == DTYPE.LONG: rpclib.get_float_rowmajor_array_as_long_array( host, port, self.get(), self.__mtype.encode('ascii'), arr, data_size) elif inp_type == DTYPE.FLOAT and out_type == DTYPE.FLOAT: rpclib.get_float_rowmajor_array_as_float_array( host, port, self.get(), self.__mtype.encode('ascii'), arr, data_size) elif inp_type == DTYPE.FLOAT and out_type == DTYPE.DOUBLE: rpclib.get_float_rowmajor_array_as_double_array( host, port, self.get(), self.__mtype.encode('ascii'), arr, data_size) elif inp_type == DTYPE.LONG and out_type == DTYPE.INT: rpclib.get_long_rowmajor_array_as_int_array( host, port, self.get(), self.__mtype.encode('ascii'), arr, data_size) elif inp_type == DTYPE.LONG and out_type == DTYPE.LONG: rpclib.get_long_rowmajor_array_as_long_array( host, port, self.get(), self.__mtype.encode('ascii'), arr, data_size) elif inp_type == DTYPE.LONG and out_type == DTYPE.FLOAT: rpclib.get_long_rowmajor_array_as_float_array( host, port, self.get(), self.__mtype.encode('ascii'), arr, data_size) elif inp_type == DTYPE.LONG and out_type == DTYPE.DOUBLE: rpclib.get_long_rowmajor_array_as_double_array( host, port, self.get(), self.__mtype.encode('ascii'), arr, data_size) elif inp_type == DTYPE.INT and out_type == DTYPE.INT: rpclib.get_int_rowmajor_array_as_int_array( host, port, self.get(), self.__mtype.encode('ascii'), arr, data_size) elif inp_type == DTYPE.INT and out_type == DTYPE.LONG: rpclib.get_int_rowmajor_array_as_long_array( host, port, self.get(), self.__mtype.encode('ascii'), arr, data_size) elif inp_type == DTYPE.INT and out_type == DTYPE.FLOAT: rpclib.get_int_rowmajor_array_as_float_array( host, port, self.get(), self.__mtype.encode('ascii'), arr, data_size) elif inp_type == DTYPE.INT and out_type == DTYPE.DOUBLE: rpclib.get_int_rowmajor_array_as_double_array( host, port, self.get(), self.__mtype.encode('ascii'), arr, data_size) else: raise TypeError("to_numpy_matrix/array: \ Supported dtypes are int/long/float/double only!") excpt = rpclib.check_server_exception() if excpt["status"]: raise RuntimeError(excpt["info"]) if is_ndarray: data = np.asarray(data)