def as_h2o_frame(self, dataframe, framename=None): """ Transforms given Spark RDD or DataFrame to H2OFrame. Parameters ---------- dataframe : Spark RDD or DataFrame framename : Optional name for resulting H2OFrame Returns ------- H2OFrame which contains data of original input Spark data structure """ if isinstance(dataframe, DataFrame): return fc._as_h2o_frame_from_dataframe(self, dataframe, framename) elif isinstance(dataframe, RDD): # First check if the type T in RDD[T] is one of the python "primitive" types # String, Boolean, Int and Double (Python Long is converted to java.lang.BigInteger) if _is_of_simple_type(dataframe): first = _get_first(dataframe) if isinstance(first, str): return fc._as_h2o_frame_from_RDD_String(self, dataframe, framename) elif isinstance(first, bool): return fc._as_h2o_frame_from_RDD_Bool(self, dataframe, framename) elif isinstance(dataframe.min(), int) and isinstance(dataframe.max(), int): if dataframe.min() >= self._jvm.Integer.MIN_VALUE and dataframe.max() <= self._jvm.Integer.MAX_VALUE: return fc._as_h2o_frame_from_RDD_Int(self, dataframe, framename) else: return fc._as_h2o_frame_from_RDD_Long(self, dataframe, framename) elif isinstance(first, float): return fc._as_h2o_frame_from_RDD_Float(self, dataframe, framename) elif isinstance(dataframe.max(), long): raise ValueError('Numbers in RDD Too Big') else: return fc._as_h2o_frame_from_complex_type(self, dataframe, framename)
def as_h2o_frame(self, dataframe, framename=None, full_cols=100): """ Transforms given Spark RDD or DataFrame to H2OFrame. Parameters ---------- dataframe : Spark RDD or DataFrame framename : Optional name for resulting H2OFrame full_cols : number of first n columns which are sent to the client together with the data Returns ------- H2OFrame which contains data of original input Spark data structure """ if isinstance(dataframe, DataFrame): return fc._as_h2o_frame_from_dataframe(self, dataframe, framename, full_cols) elif isinstance(dataframe, RDD) and dataframe.isEmpty(): schema = StructType([]) empty = self._spark_session.createDataFrame(self._spark_session.sparkContext.emptyRDD(), schema) return fc._as_h2o_frame_from_dataframe(self, empty, framename, full_cols) elif isinstance(dataframe, RDD): # First check if the type T in RDD[T] is one of the python "primitive" types # String, Boolean, Int and Double (Python Long is converted to java.lang.BigInteger) if _is_of_simple_type(dataframe): first = _get_first(dataframe) # Make this code compatible with python 3.6 and python 2.7 global long if sys.version_info > (3,): long = int if isinstance(first, str): return fc._as_h2o_frame_from_RDD_String(self, dataframe, framename, full_cols) elif isinstance(first, bool): return fc._as_h2o_frame_from_RDD_Bool(self, dataframe, framename, full_cols) elif (isinstance(dataframe.min(), int) and isinstance(dataframe.max(), int)) or (isinstance(dataframe.min(), long) and isinstance(dataframe.max(), long)): if dataframe.min() >= self._jvm.Integer.MIN_VALUE and dataframe.max() <= self._jvm.Integer.MAX_VALUE: return fc._as_h2o_frame_from_RDD_Int(self, dataframe, framename, full_cols) elif dataframe.min() >= self._jvm.Long.MIN_VALUE and dataframe.max() <= self._jvm.Long.MAX_VALUE: return fc._as_h2o_frame_from_RDD_Long(self, dataframe, framename, full_cols) else: raise ValueError('Numbers in RDD Too Big') elif isinstance(first, float): return fc._as_h2o_frame_from_RDD_Float(self, dataframe, framename, full_cols) else: return fc._as_h2o_frame_from_complex_type(self, dataframe, framename, full_cols) else: raise ValueError('The as_h2o_frame method expects Spark DataFrame or RDD as the input only!')
def as_h2o_frame(self, dataframe, framename=None): """ Transforms given Spark RDD or DataFrame to H2OFrame. Parameters ---------- dataframe : Spark RDD or DataFrame framename : Optional name for resulting H2OFrame Returns ------- H2OFrame which contains data of original input Spark data structure """ if isinstance(dataframe, DataFrame): return fc._as_h2o_frame_from_dataframe(self, dataframe, framename) elif isinstance(dataframe, RDD): # First check if the type T in RDD[T] is one of the python "primitive" types # String, Boolean, Int and Double (Python Long is converted to java.lang.BigInteger) if _is_of_simple_type(dataframe): first = _get_first(dataframe) if isinstance(first, str): return fc._as_h2o_frame_from_RDD_String( self, dataframe, framename) elif isinstance(first, bool): return fc._as_h2o_frame_from_RDD_Bool( self, dataframe, framename) elif isinstance(dataframe.min(), int) and isinstance( dataframe.max(), int): if dataframe.min( ) >= self._jvm.Integer.MIN_VALUE and dataframe.max( ) <= self._jvm.Integer.MAX_VALUE: return fc._as_h2o_frame_from_RDD_Int( self, dataframe, framename) else: return fc._as_h2o_frame_from_RDD_Long( self, dataframe, framename) elif isinstance(first, float): return fc._as_h2o_frame_from_RDD_Float( self, dataframe, framename) elif isinstance(dataframe.max(), long): raise ValueError('Numbers in RDD Too Big') else: return fc._as_h2o_frame_from_complex_type( self, dataframe, framename)
def as_h2o_frame(self, dataframe, framename=None, full_cols=100): """ Transforms given Spark RDD or DataFrame to H2OFrame. Parameters ---------- dataframe : Spark RDD or DataFrame framename : Optional name for resulting H2OFrame full_cols : number of first n columns which are sent to the client together with the data Returns ------- H2OFrame which contains data of original input Spark data structure """ if isinstance(dataframe, DataFrame): return fc._as_h2o_frame_from_dataframe(self, dataframe, framename, full_cols) elif isinstance(dataframe, RDD) and dataframe.isEmpty(): schema = StructType([]) empty = self._spark_session.createDataFrame( self._spark_session.sparkContext.emptyRDD(), schema) return fc._as_h2o_frame_from_dataframe(self, empty, framename, full_cols) elif isinstance(dataframe, RDD): # First check if the type T in RDD[T] is one of the python "primitive" types # String, Boolean, Int and Double (Python Long is converted to java.lang.BigInteger) if _is_of_simple_type(dataframe): first = _get_first(dataframe) # Make this code compatible with python 3.6 and python 2.7 global long if sys.version_info > (3, ): long = int if isinstance(first, str): return fc._as_h2o_frame_from_RDD_String( self, dataframe, framename, full_cols) elif isinstance(first, bool): return fc._as_h2o_frame_from_RDD_Bool( self, dataframe, framename, full_cols) elif (isinstance(dataframe.min(), int) and isinstance(dataframe.max(), int)) or ( isinstance(dataframe.min(), long) and isinstance(dataframe.max(), long)): if dataframe.min( ) >= self._jvm.Integer.MIN_VALUE and dataframe.max( ) <= self._jvm.Integer.MAX_VALUE: return fc._as_h2o_frame_from_RDD_Int( self, dataframe, framename, full_cols) elif dataframe.min( ) >= self._jvm.Long.MIN_VALUE and dataframe.max( ) <= self._jvm.Long.MAX_VALUE: return fc._as_h2o_frame_from_RDD_Long( self, dataframe, framename, full_cols) else: raise ValueError('Numbers in RDD Too Big') elif isinstance(first, float): return fc._as_h2o_frame_from_RDD_Float( self, dataframe, framename, full_cols) else: return fc._as_h2o_frame_from_complex_type( self, dataframe, framename, full_cols) else: raise ValueError( 'The as_h2o_frame method expects Spark DataFrame or RDD as the input only!' )