def test_from_pandas(self): pdf = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) internal = InternalFrame.from_pandas(pdf) sdf = internal.spark_frame self.assert_eq(internal.index_map, OrderedDict({SPARK_DEFAULT_INDEX_NAME: None})) self.assert_eq(internal.column_labels, [("a", ), ("b", )]) self.assert_eq(internal.data_spark_column_names, ["a", "b"]) self.assertTrue( internal.spark_column_for(("a", ))._jc.equals(sdf["a"]._jc)) self.assertTrue( internal.spark_column_for(("b", ))._jc.equals(sdf["b"]._jc)) self.assert_eq(internal.to_pandas_frame, pdf) # multi-index pdf.set_index("a", append=True, inplace=True) internal = InternalFrame.from_pandas(pdf) sdf = internal.spark_frame self.assert_eq( internal.index_map, OrderedDict([(SPARK_INDEX_NAME_FORMAT(0), None), (SPARK_INDEX_NAME_FORMAT(1), ("a", ))]), ) self.assert_eq(internal.column_labels, [("b", )]) self.assert_eq(internal.data_spark_column_names, ["b"]) self.assertTrue( internal.spark_column_for(("b", ))._jc.equals(sdf["b"]._jc)) self.assert_eq(internal.to_pandas_frame, pdf) # multi-index columns pdf.columns = pd.MultiIndex.from_tuples([("x", "b")]) internal = InternalFrame.from_pandas(pdf) sdf = internal.spark_frame self.assert_eq( internal.index_map, OrderedDict([(SPARK_INDEX_NAME_FORMAT(0), None), (SPARK_INDEX_NAME_FORMAT(1), ("a", ))]), ) self.assert_eq(internal.column_labels, [("x", "b")]) self.assert_eq(internal.data_spark_column_names, ["(x, b)"]) self.assertTrue( internal.spark_column_for( ("x", "b"))._jc.equals(sdf["(x, b)"]._jc)) self.assert_eq(internal.to_pandas_frame, pdf)
def intersection(self, other) -> "MultiIndex": """ Form the intersection of two Index objects. This returns a new Index with elements common to the index and `other`. Parameters ---------- other : Index or array-like Returns ------- intersection : MultiIndex Examples -------- >>> midx1 = ks.MultiIndex.from_tuples([("a", "x"), ("b", "y"), ("c", "z")]) >>> midx2 = ks.MultiIndex.from_tuples([("c", "z"), ("d", "w")]) >>> midx1.intersection(midx2).sort_values() # doctest: +SKIP MultiIndex([('c', 'z')], ) """ if isinstance(other, Series) or not is_list_like(other): raise TypeError("other must be a MultiIndex or a list of tuples") elif isinstance(other, DataFrame): raise ValueError("Index data must be 1-dimensional") elif isinstance(other, MultiIndex): spark_frame_other = other.to_frame().to_spark() keep_name = self.names == other.names elif isinstance(other, Index): # Always returns an empty MultiIndex if `other` is Index. return self.to_frame().head(0).index # type: ignore elif not all(isinstance(item, tuple) for item in other): raise TypeError("other must be a MultiIndex or a list of tuples") else: other = MultiIndex.from_tuples(list(other)) spark_frame_other = other.to_frame().to_spark() keep_name = True default_name = [ SPARK_INDEX_NAME_FORMAT(i) for i in range(self.nlevels) ] spark_frame_self = self.to_frame(name=default_name).to_spark() spark_frame_intersected = spark_frame_self.intersect(spark_frame_other) if keep_name: index_names = self._internal.index_names else: index_names = None internal = InternalFrame( # TODO: dtypes? spark_frame=spark_frame_intersected, index_spark_columns=[ scol_for(spark_frame_intersected, col) for col in default_name ], index_names=index_names, ) return cast(MultiIndex, DataFrame(internal).index)
def _is_monotonic_decreasing(self): scol = self.spark.column window = Window.orderBy(NATURAL_ORDER_COLUMN_NAME).rowsBetween(-1, -1) prev = F.lag(scol, 1).over(window) cond = F.lit(True) has_not_null = F.lit(True) for field in self.spark.data_type[::-1]: left = scol.getField(field.name) right = prev.getField(field.name) compare = MultiIndex._comparator_for_monotonic_decreasing( field.dataType) # Since pandas 1.1.4, null value is not allowed at any levels of MultiIndex. # Therefore, we should check `has_not_null` over the all levels. has_not_null = has_not_null & left.isNotNull() cond = F.when(left.eqNullSafe(right), cond).otherwise( compare(left, right, spark.Column.__lt__)) cond = has_not_null & (prev.isNull() | cond) cond_name = verify_temp_column_name( self._internal.spark_frame.select( self._internal.index_spark_columns), "__is_monotonic_decreasing_cond__", ) sdf = self._internal.spark_frame.select( self._internal.index_spark_columns + [cond.alias(cond_name)]) internal = InternalFrame( spark_frame=sdf, index_spark_columns=[ scol_for(sdf, col) for col in self._internal.index_spark_column_names ], index_names=self._internal.index_names, index_dtypes=self._internal.index_dtypes, ) return first_series(DataFrame(internal))
def value_counts(self, normalize=False, sort=True, ascending=False, bins=None, dropna=True): """ Return a Series containing counts of unique values. The resulting object will be in descending order so that the first element is the most frequently-occurring element. Excludes NA values by default. Parameters ---------- normalize : boolean, default False If True then the object returned will contain the relative frequencies of the unique values. sort : boolean, default True Sort by values. ascending : boolean, default False Sort in ascending order. bins : Not Yet Supported dropna : boolean, default True Don't include counts of NaN. Returns ------- counts : Series See Also -------- Series.count: Number of non-NA elements in a Series. Examples -------- For Series >>> df = ks.DataFrame({'x':[0, 0, 1, 1, 1, np.nan]}) >>> df.x.value_counts() # doctest: +NORMALIZE_WHITESPACE 1.0 3 0.0 2 Name: x, dtype: int64 With `normalize` set to `True`, returns the relative frequency by dividing all values by the sum of values. >>> df.x.value_counts(normalize=True) # doctest: +NORMALIZE_WHITESPACE 1.0 0.6 0.0 0.4 Name: x, dtype: float64 **dropna** With `dropna` set to `False` we can also see NaN index values. >>> df.x.value_counts(dropna=False) # doctest: +NORMALIZE_WHITESPACE 1.0 3 0.0 2 NaN 1 Name: x, dtype: int64 For Index >>> idx = ks.Index([3, 1, 2, 3, 4, np.nan]) >>> idx Float64Index([3.0, 1.0, 2.0, 3.0, 4.0, nan], dtype='float64') >>> idx.value_counts().sort_index() 1.0 1 2.0 1 3.0 2 4.0 1 dtype: int64 **sort** With `sort` set to `False`, the result wouldn't be sorted by number of count. >>> idx.value_counts(sort=True).sort_index() 1.0 1 2.0 1 3.0 2 4.0 1 dtype: int64 **normalize** With `normalize` set to `True`, returns the relative frequency by dividing all values by the sum of values. >>> idx.value_counts(normalize=True).sort_index() 1.0 0.2 2.0 0.2 3.0 0.4 4.0 0.2 dtype: float64 **dropna** With `dropna` set to `False` we can also see NaN index values. >>> idx.value_counts(dropna=False).sort_index() # doctest: +SKIP 1.0 1 2.0 1 3.0 2 4.0 1 NaN 1 dtype: int64 For MultiIndex. >>> midx = pd.MultiIndex([['lama', 'cow', 'falcon'], ... ['speed', 'weight', 'length']], ... [[0, 0, 0, 1, 1, 1, 2, 2, 2], ... [1, 1, 1, 1, 1, 2, 1, 2, 2]]) >>> s = ks.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3], index=midx) >>> s.index # doctest: +SKIP MultiIndex([( 'lama', 'weight'), ( 'lama', 'weight'), ( 'lama', 'weight'), ( 'cow', 'weight'), ( 'cow', 'weight'), ( 'cow', 'length'), ('falcon', 'weight'), ('falcon', 'length'), ('falcon', 'length')], ) >>> s.index.value_counts().sort_index() (cow, length) 1 (cow, weight) 2 (falcon, length) 2 (falcon, weight) 1 (lama, weight) 3 dtype: int64 >>> s.index.value_counts(normalize=True).sort_index() (cow, length) 0.111111 (cow, weight) 0.222222 (falcon, length) 0.222222 (falcon, weight) 0.111111 (lama, weight) 0.333333 dtype: float64 If Index has name, keep the name up. >>> idx = ks.Index([0, 0, 0, 1, 1, 2, 3], name='koalas') >>> idx.value_counts().sort_index() 0 3 1 2 2 1 3 1 Name: koalas, dtype: int64 """ from databricks.koalas.series import first_series if bins is not None: raise NotImplementedError( "value_counts currently does not support bins") if dropna: sdf_dropna = self._internal.spark_frame.select( self.spark.column).dropna() else: sdf_dropna = self._internal.spark_frame.select(self.spark.column) index_name = SPARK_DEFAULT_INDEX_NAME column_name = self._internal.data_spark_column_names[0] sdf = sdf_dropna.groupby( scol_for(sdf_dropna, column_name).alias(index_name)).count() if sort: if ascending: sdf = sdf.orderBy(F.col("count")) else: sdf = sdf.orderBy(F.col("count").desc()) if normalize: sum = sdf_dropna.count() sdf = sdf.withColumn("count", F.col("count") / F.lit(sum)) internal = InternalFrame( spark_frame=sdf, index_map=OrderedDict({index_name: None}), column_labels=self._internal.column_labels, data_spark_columns=[scol_for(sdf, "count")], column_label_names=self._internal.column_label_names, ) return first_series(DataFrame(internal))
def combine_frames(this, *args, how="full", preserve_order_column=False): """ This method combines `this` DataFrame with a different `that` DataFrame or Series from a different DataFrame. It returns a DataFrame that has prefix `this_` and `that_` to distinct the columns names from both DataFrames It internally performs a join operation which can be expensive in general. So, if `compute.ops_on_diff_frames` option is False, this method throws an exception. """ from databricks.koalas.config import get_option from databricks.koalas.frame import DataFrame from databricks.koalas.internal import ( InternalFrame, HIDDEN_COLUMNS, NATURAL_ORDER_COLUMN_NAME, SPARK_INDEX_NAME_FORMAT, ) from databricks.koalas.series import Series if all(isinstance(arg, Series) for arg in args): assert all( same_anchor(arg, args[0]) for arg in args ), "Currently only one different DataFrame (from given Series) is supported" assert not same_anchor( this, args[0]), "We don't need to combine. All series is in this." that = args[0]._kdf[list(args)] elif len(args) == 1 and isinstance(args[0], DataFrame): assert isinstance(args[0], DataFrame) assert not same_anchor( this, args[0]), "We don't need to combine. `this` and `that` are same." that = args[0] else: raise AssertionError("args should be single DataFrame or " "single/multiple Series") if get_option("compute.ops_on_diff_frames"): def resolve(internal, side): rename = lambda col: "__{}_{}".format(side, col) internal = internal.resolved_copy sdf = internal.spark_frame sdf = internal.spark_frame.select([ scol_for(sdf, col).alias(rename(col)) for col in sdf.columns if col not in HIDDEN_COLUMNS ] + list(HIDDEN_COLUMNS)) return internal.copy( spark_frame=sdf, index_spark_columns=[ scol_for(sdf, rename(col)) for col in internal.index_spark_column_names ], data_spark_columns=[ scol_for(sdf, rename(col)) for col in internal.data_spark_column_names ], ) this_internal = resolve(this._internal, "this") that_internal = resolve(that._internal, "that") this_index_map = list( zip(this_internal.index_spark_column_names, this_internal.index_names)) that_index_map = list( zip(that_internal.index_spark_column_names, that_internal.index_names)) assert len(this_index_map) == len(that_index_map) join_scols = [] merged_index_scols = [] # Note that the order of each element in index_map is guaranteed according to the index # level. this_and_that_index_map = list(zip(this_index_map, that_index_map)) this_sdf = this_internal.spark_frame.alias("this") that_sdf = that_internal.spark_frame.alias("that") # If the same named index is found, that's used. index_column_names = [] for i, ((this_column, this_name), (that_column, that_name)) in enumerate(this_and_that_index_map): if this_name == that_name: # We should merge the Spark columns into one # to mimic pandas' behavior. this_scol = scol_for(this_sdf, this_column) that_scol = scol_for(that_sdf, that_column) join_scol = this_scol == that_scol join_scols.append(join_scol) column_name = SPARK_INDEX_NAME_FORMAT(i) index_column_names.append(column_name) merged_index_scols.append( F.when(this_scol.isNotNull(), this_scol).otherwise(that_scol).alias(column_name)) else: raise ValueError( "Index names must be exactly matched currently.") assert len( join_scols) > 0, "cannot join with no overlapping index names" joined_df = this_sdf.join(that_sdf, on=join_scols, how=how) if preserve_order_column: order_column = [scol_for(this_sdf, NATURAL_ORDER_COLUMN_NAME)] else: order_column = [] joined_df = joined_df.select(merged_index_scols + [ scol_for(this_sdf, this_internal.spark_column_name_for(label)) for label in this_internal.column_labels ] + [ scol_for(that_sdf, that_internal.spark_column_name_for(label)) for label in that_internal.column_labels ] + order_column) index_columns = set(index_column_names) new_data_columns = [ col for col in joined_df.columns if col not in index_columns and col != NATURAL_ORDER_COLUMN_NAME ] level = max(this_internal.column_labels_level, that_internal.column_labels_level) def fill_label(label): if label is None: return ([""] * (level - 1)) + [None] else: return ([""] * (level - len(label))) + list(label) column_labels = [ tuple(["this"] + fill_label(label)) for label in this_internal.column_labels ] + [ tuple(["that"] + fill_label(label)) for label in that_internal.column_labels ] column_label_names = ([None] * (1 + level - this_internal.column_labels_level) ) + this_internal.column_label_names return DataFrame( InternalFrame( spark_frame=joined_df, index_spark_columns=[ scol_for(joined_df, col) for col in index_column_names ], index_names=this_internal.index_names, column_labels=column_labels, data_spark_columns=[ scol_for(joined_df, col) for col in new_data_columns ], column_label_names=column_label_names, )) else: raise ValueError( "Cannot combine the series or dataframe because it comes from a different dataframe. " "In order to allow this operation, enable 'compute.ops_on_diff_frames' option." )
def test_from_pandas(self): pdf = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) internal = InternalFrame.from_pandas(pdf) sdf = internal.spark_frame self.assert_eq(internal.index_spark_column_names, [SPARK_DEFAULT_INDEX_NAME]) self.assert_eq(internal.index_names, [None]) self.assert_eq(internal.column_labels, [("a",), ("b",)]) self.assert_eq(internal.data_spark_column_names, ["a", "b"]) self.assertTrue(internal.spark_column_for(("a",))._jc.equals(sdf["a"]._jc)) self.assertTrue(internal.spark_column_for(("b",))._jc.equals(sdf["b"]._jc)) self.assert_eq(internal.to_pandas_frame, pdf) # non-string column name pdf1 = pd.DataFrame({0: [1, 2, 3], 1: [4, 5, 6]}) internal = InternalFrame.from_pandas(pdf1) sdf = internal.spark_frame self.assert_eq(internal.index_spark_column_names, [SPARK_DEFAULT_INDEX_NAME]) self.assert_eq(internal.index_names, [None]) self.assert_eq(internal.column_labels, [(0,), (1,)]) self.assert_eq(internal.data_spark_column_names, ["0", "1"]) self.assertTrue(internal.spark_column_for((0,))._jc.equals(sdf["0"]._jc)) self.assertTrue(internal.spark_column_for((1,))._jc.equals(sdf["1"]._jc)) self.assert_eq(internal.to_pandas_frame, pdf1) # multi-index pdf.set_index("a", append=True, inplace=True) internal = InternalFrame.from_pandas(pdf) sdf = internal.spark_frame self.assert_eq( internal.index_spark_column_names, [SPARK_INDEX_NAME_FORMAT(0), SPARK_INDEX_NAME_FORMAT(1)], ) self.assert_eq(internal.index_names, [None, ("a",)]) self.assert_eq(internal.column_labels, [("b",)]) self.assert_eq(internal.data_spark_column_names, ["b"]) self.assertTrue(internal.spark_column_for(("b",))._jc.equals(sdf["b"]._jc)) self.assert_eq(internal.to_pandas_frame, pdf) # multi-index columns pdf.columns = pd.MultiIndex.from_tuples([("x", "b")]) internal = InternalFrame.from_pandas(pdf) sdf = internal.spark_frame self.assert_eq( internal.index_spark_column_names, [SPARK_INDEX_NAME_FORMAT(0), SPARK_INDEX_NAME_FORMAT(1)], ) self.assert_eq(internal.index_names, [None, ("a",)]) self.assert_eq(internal.column_labels, [("x", "b")]) self.assert_eq(internal.data_spark_column_names, ["(x, b)"]) self.assertTrue(internal.spark_column_for(("x", "b"))._jc.equals(sdf["(x, b)"]._jc)) self.assert_eq(internal.to_pandas_frame, pdf)
def attach_id_column(self, id_type: str, column: Union[str, Tuple[str, ...]]) -> "DataFrame": """ Attach a column to be used as identifier of rows similar to the default index. See also `Default Index type <https://koalas.readthedocs.io/en/latest/user_guide/options.html#default-index-type>`_. Parameters ---------- id_type : string The id type. - 'sequence' : a sequence that increases one by one. .. note:: this uses Spark's Window without specifying partition specification. This leads to move all data into single partition in single machine and could cause serious performance degradation. Avoid this method against very large dataset. - 'distributed-sequence' : a sequence that increases one by one, by group-by and group-map approach in a distributed manner. - 'distributed' : a monotonically increasing sequence simply by using PySpark’s monotonically_increasing_id function in a fully distributed manner. column : string or tuple of string The column name. Returns ------- DataFrame The DataFrame attached the column. Examples -------- >>> df = ks.DataFrame({"x": ['a', 'b', 'c']}) >>> df.koalas.attach_id_column(id_type="sequence", column="id") x id 0 a 0 1 b 1 2 c 2 >>> df.koalas.attach_id_column(id_type="distributed-sequence", column="id").sort_index() x id 0 a 0 1 b 1 2 c 2 >>> df.koalas.attach_id_column(id_type="distributed", column="id") ... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE x id 0 a ... 1 b ... 2 c ... For multi-index columns: >>> df = ks.DataFrame({("x", "y"): ['a', 'b', 'c']}) >>> df.koalas.attach_id_column(id_type="sequence", column=("id-x", "id-y")) x id-x y id-y 0 a 0 1 b 1 2 c 2 """ from databricks.koalas.frame import DataFrame if id_type == "sequence": attach_func = InternalFrame.attach_sequence_column elif id_type == "distributed-sequence": attach_func = InternalFrame.attach_distributed_sequence_column elif id_type == "distributed": attach_func = InternalFrame.attach_distributed_column else: raise ValueError( "id_type should be one of 'sequence', 'distributed-sequence' and 'distributed'" ) if isinstance(column, str): column = (column, ) else: assert isinstance(column, tuple), type(column) internal = self._kdf._internal if len(column) != internal.column_labels_level: raise ValueError( "The given column `{}` must be the same length as the existing columns." .format(column)) elif column in internal.column_labels: raise ValueError("The given column `{}` already exists.".format( name_like_string(column))) # Make sure the underlying Spark column names are the form of # `name_like_string(column_label)`. sdf = internal.spark_frame.select([ scol.alias(SPARK_INDEX_NAME_FORMAT(i)) for i, scol in enumerate(internal.index_spark_columns) ] + [ scol.alias(name_like_string(label)) for scol, label in zip( internal.data_spark_columns, internal.column_labels) ]) sdf = attach_func(sdf, name_like_string(column)) return DataFrame( InternalFrame( spark_frame=sdf, index_map=OrderedDict([ (SPARK_INDEX_NAME_FORMAT(i), name) for i, name in enumerate(internal.index_names) ]), column_labels=internal.column_labels + [column], data_spark_columns=([ scol_for(sdf, name_like_string(label)) for label in internal.column_labels ] + [scol_for(sdf, name_like_string(column))]), column_label_names=internal.column_label_names, ).resolved_copy)
def apply_batch(self, func, args=(), **kwds): """ Apply a function that takes pandas DataFrame and outputs pandas DataFrame. The pandas DataFrame given to the function is of a batch used internally. See also `Transform and apply a function <https://koalas.readthedocs.io/en/latest/user_guide/transform_apply.html>`_. .. note:: the `func` is unable to access to the whole input frame. Koalas internally splits the input series into multiple batches and calls `func` with each batch multiple times. Therefore, operations such as global aggregations are impossible. See the example below. >>> # This case does not return the length of whole frame but of the batch internally ... # used. ... def length(pdf) -> ks.DataFrame[int]: ... return pd.DataFrame([len(pdf)]) ... >>> df = ks.DataFrame({'A': range(1000)}) >>> df.koalas.apply_batch(length) # doctest: +SKIP c0 0 83 1 83 2 83 ... 10 83 11 83 .. note:: this API executes the function once to infer the type which is potentially expensive, for instance, when the dataset is created after aggregations or sorting. To avoid this, specify return type in ``func``, for instance, as below: >>> def plus_one(x) -> ks.DataFrame[float, float]: ... return x + 1 If the return type is specified, the output column names become `c0, c1, c2 ... cn`. These names are positionally mapped to the returned DataFrame in ``func``. To specify the column names, you can assign them in a pandas friendly style as below: >>> def plus_one(x) -> ks.DataFrame["a": float, "b": float]: ... return x + 1 >>> pdf = pd.DataFrame({'a': [1, 2, 3], 'b': [3, 4, 5]}) >>> def plus_one(x) -> ks.DataFrame[zip(pdf.dtypes, pdf.columns)]: ... return x + 1 Parameters ---------- func : function Function to apply to each pandas frame. args : tuple Positional arguments to pass to `func` in addition to the array/series. **kwds Additional keyword arguments to pass as keywords arguments to `func`. Returns ------- DataFrame See Also -------- DataFrame.apply: For row/columnwise operations. DataFrame.applymap: For elementwise operations. DataFrame.aggregate: Only perform aggregating type operations. DataFrame.transform: Only perform transforming type operations. Series.koalas.transform_batch: transform the search as each pandas chunks. Examples -------- >>> df = ks.DataFrame([(1, 2), (3, 4), (5, 6)], columns=['A', 'B']) >>> df A B 0 1 2 1 3 4 2 5 6 >>> def query_func(pdf) -> ks.DataFrame[int, int]: ... return pdf.query('A == 1') >>> df.koalas.apply_batch(query_func) c0 c1 0 1 2 >>> def query_func(pdf) -> ks.DataFrame["A": int, "B": int]: ... return pdf.query('A == 1') >>> df.koalas.apply_batch(query_func) A B 0 1 2 You can also omit the type hints so Koalas infers the return schema as below: >>> df.koalas.apply_batch(lambda pdf: pdf.query('A == 1')) A B 0 1 2 You can also specify extra arguments. >>> def calculation(pdf, y, z) -> ks.DataFrame[int, int]: ... return pdf ** y + z >>> df.koalas.apply_batch(calculation, args=(10,), z=20) c0 c1 0 21 1044 1 59069 1048596 2 9765645 60466196 You can also use ``np.ufunc`` and built-in functions as input. >>> df.koalas.apply_batch(np.add, args=(10,)) A B 0 11 12 1 13 14 2 15 16 >>> (df * -1).koalas.apply_batch(abs) A B 0 1 2 1 3 4 2 5 6 """ # TODO: codes here partially duplicate `DataFrame.apply`. Can we deduplicate? from databricks.koalas.groupby import GroupBy from databricks.koalas.frame import DataFrame from databricks import koalas as ks if not isinstance(func, types.FunctionType): assert callable( func), "the first argument should be a callable function." f = func func = lambda *args, **kwargs: f(*args, **kwargs) spec = inspect.getfullargspec(func) return_sig = spec.annotations.get("return", None) should_infer_schema = return_sig is None should_use_map_in_pandas = LooseVersion(pyspark.__version__) >= "3.0" original_func = func func = lambda o: original_func(o, *args, **kwds) self_applied = DataFrame(self._kdf._internal.resolved_copy) if should_infer_schema: # Here we execute with the first 1000 to get the return type. # If the records were less than 1000, it uses pandas API directly for a shortcut. limit = ks.get_option("compute.shortcut_limit") pdf = self_applied.head(limit + 1)._to_internal_pandas() applied = func(pdf) if not isinstance(applied, pd.DataFrame): raise ValueError( "The given function should return a frame; however, " "the return type was %s." % type(applied)) kdf = ks.DataFrame(applied) if len(pdf) <= limit: return kdf return_schema = kdf._internal.to_internal_spark_frame.schema if should_use_map_in_pandas: output_func = GroupBy._make_pandas_df_builder_func( self_applied, func, return_schema, retain_index=True) sdf = self_applied._internal.to_internal_spark_frame.mapInPandas( lambda iterator: map(output_func, iterator), schema=return_schema) else: sdf = GroupBy._spark_group_map_apply( self_applied, func, (F.spark_partition_id(), ), return_schema, retain_index=True) # If schema is inferred, we can restore indexes too. internal = kdf._internal.with_new_sdf(sdf) else: return_type = infer_return_type(original_func) return_schema = return_type.tpe is_return_dataframe = isinstance(return_type, DataFrameType) if not is_return_dataframe: raise TypeError( "The given function should specify a frame as its type " "hints; however, the return type was %s." % return_sig) if should_use_map_in_pandas: output_func = GroupBy._make_pandas_df_builder_func( self_applied, func, return_schema, retain_index=False) sdf = self_applied._internal.to_internal_spark_frame.mapInPandas( lambda iterator: map(output_func, iterator), schema=return_schema) else: sdf = GroupBy._spark_group_map_apply( self_applied, func, (F.spark_partition_id(), ), return_schema, retain_index=False) # Otherwise, it loses index. internal = InternalFrame(spark_frame=sdf, index_map=None) return DataFrame(internal)
def drop(self, codes, level=None) -> "MultiIndex": """ Make new MultiIndex with passed list of labels deleted Parameters ---------- codes : array-like Must be a list of tuples level : int or level name, default None Returns ------- dropped : MultiIndex Examples -------- >>> index = ks.MultiIndex.from_tuples([('a', 'x'), ('b', 'y'), ('c', 'z')]) >>> index # doctest: +SKIP MultiIndex([('a', 'x'), ('b', 'y'), ('c', 'z')], ) >>> index.drop(['a']) # doctest: +SKIP MultiIndex([('b', 'y'), ('c', 'z')], ) >>> index.drop(['x', 'y'], level=1) # doctest: +SKIP MultiIndex([('c', 'z')], ) """ internal = self._internal.resolved_copy sdf = internal.spark_frame index_scols = internal.index_spark_columns if level is None: scol = index_scols[0] elif isinstance(level, int): scol = index_scols[level] else: scol = None for index_spark_column, index_name in zip( internal.index_spark_columns, internal.index_names): if not isinstance(level, tuple): level = (level, ) if level == index_name: if scol is not None: raise ValueError( "The name {} occurs multiple times, use a level number" .format(name_like_string(level))) scol = index_spark_column if scol is None: raise KeyError("Level {} not found".format( name_like_string(level))) sdf = sdf[~scol.isin(codes)] internal = InternalFrame( spark_frame=sdf, index_spark_columns=[ scol_for(sdf, col) for col in internal.index_spark_column_names ], index_names=internal.index_names, index_dtypes=internal.index_dtypes, column_labels=[], data_spark_columns=[], data_dtypes=[], ) return cast(MultiIndex, DataFrame(internal).index)
def symmetric_difference(self, other, result_name=None, sort=None) -> "MultiIndex": """ Compute the symmetric difference of two MultiIndex objects. Parameters ---------- other : Index or array-like result_name : list sort : True or None, default None Whether to sort the resulting index. * True : Attempt to sort the result. * None : Do not sort the result. Returns ------- symmetric_difference : MiltiIndex Notes ----- ``symmetric_difference`` contains elements that appear in either ``idx1`` or ``idx2`` but not both. Equivalent to the Index created by ``idx1.difference(idx2) | idx2.difference(idx1)`` with duplicates dropped. Examples -------- >>> midx1 = pd.MultiIndex([['lama', 'cow', 'falcon'], ... ['speed', 'weight', 'length']], ... [[0, 0, 0, 1, 1, 1, 2, 2, 2], ... [0, 0, 0, 0, 1, 2, 0, 1, 2]]) >>> midx2 = pd.MultiIndex([['koalas', 'cow', 'falcon'], ... ['speed', 'weight', 'length']], ... [[0, 0, 0, 1, 1, 1, 2, 2, 2], ... [0, 0, 0, 0, 1, 2, 0, 1, 2]]) >>> s1 = ks.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3], ... index=midx1) >>> s2 = ks.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3], ... index=midx2) >>> s1.index.symmetric_difference(s2.index) # doctest: +SKIP MultiIndex([('koalas', 'speed'), ( 'lama', 'speed')], ) You can set names of result Index. >>> s1.index.symmetric_difference(s2.index, result_name=['a', 'b']) # doctest: +SKIP MultiIndex([('koalas', 'speed'), ( 'lama', 'speed')], names=['a', 'b']) You can set sort to `True`, if you want to sort the resulting index. >>> s1.index.symmetric_difference(s2.index, sort=True) # doctest: +SKIP MultiIndex([('koalas', 'speed'), ( 'lama', 'speed')], ) You can also use the ``^`` operator: >>> s1.index ^ s2.index # doctest: +SKIP MultiIndex([('koalas', 'speed'), ( 'lama', 'speed')], ) """ if type(self) != type(other): raise NotImplementedError( "Doesn't support symmetric_difference between Index & MultiIndex for now" ) sdf_self = self._kdf._internal.spark_frame.select( self._internal.index_spark_columns) sdf_other = other._kdf._internal.spark_frame.select( other._internal.index_spark_columns) sdf_symdiff = sdf_self.union(sdf_other).subtract( sdf_self.intersect(sdf_other)) if sort: sdf_symdiff = sdf_symdiff.sort(self._internal.index_spark_columns) internal = InternalFrame( # TODO: dtypes? spark_frame=sdf_symdiff, index_spark_columns=[ scol_for(sdf_symdiff, col) for col in self._internal.index_spark_column_names ], index_names=self._internal.index_names, ) result = cast(MultiIndex, DataFrame(internal).index) if result_name: result.names = result_name return result
def from_frame(df, names=None) -> "MultiIndex": """ Make a MultiIndex from a DataFrame. Parameters ---------- df : DataFrame DataFrame to be converted to MultiIndex. names : list-like, optional If no names are provided, use the column names, or tuple of column names if the columns is a MultiIndex. If a sequence, overwrite names with the given sequence. Returns ------- MultiIndex The MultiIndex representation of the given DataFrame. See Also -------- MultiIndex.from_arrays : Convert list of arrays to MultiIndex. MultiIndex.from_tuples : Convert list of tuples to MultiIndex. MultiIndex.from_product : Make a MultiIndex from cartesian product of iterables. Examples -------- >>> df = ks.DataFrame([['HI', 'Temp'], ['HI', 'Precip'], ... ['NJ', 'Temp'], ['NJ', 'Precip']], ... columns=['a', 'b']) >>> df # doctest: +SKIP a b 0 HI Temp 1 HI Precip 2 NJ Temp 3 NJ Precip >>> ks.MultiIndex.from_frame(df) # doctest: +SKIP MultiIndex([('HI', 'Temp'), ('HI', 'Precip'), ('NJ', 'Temp'), ('NJ', 'Precip')], names=['a', 'b']) Using explicit names, instead of the column names >>> ks.MultiIndex.from_frame(df, names=['state', 'observation']) # doctest: +SKIP MultiIndex([('HI', 'Temp'), ('HI', 'Precip'), ('NJ', 'Temp'), ('NJ', 'Precip')], names=['state', 'observation']) """ if not isinstance(df, DataFrame): raise TypeError("Input must be a DataFrame") sdf = df.to_spark() if names is None: names = df._internal.column_labels elif not is_list_like(names): raise ValueError("Names should be list-like for a MultiIndex") else: names = [ name if is_name_like_tuple(name) else (name, ) for name in names ] internal = InternalFrame( spark_frame=sdf, index_spark_columns=[scol_for(sdf, col) for col in sdf.columns], index_names=names, ) return cast(MultiIndex, DataFrame(internal).index)
def insert(self, loc: int, item) -> Index: """ Make new MultiIndex inserting new item at location. Follows Python list.append semantics for negative values. Parameters ---------- loc : int item : object Returns ------- new_index : MultiIndex Examples -------- >>> kmidx = ks.MultiIndex.from_tuples([("a", "x"), ("b", "y"), ("c", "z")]) >>> kmidx.insert(3, ("h", "j")) # doctest: +SKIP MultiIndex([('a', 'x'), ('b', 'y'), ('c', 'z'), ('h', 'j')], ) For negative values >>> kmidx.insert(-2, ("h", "j")) # doctest: +SKIP MultiIndex([('a', 'x'), ('h', 'j'), ('b', 'y'), ('c', 'z')], ) """ length = len(self) if loc < 0: loc = loc + length if loc < 0: raise IndexError( "index {} is out of bounds for axis 0 with size {}".format( (loc - length), length)) else: if loc > length: raise IndexError( "index {} is out of bounds for axis 0 with size {}".format( loc, length)) index_name = self._internal.index_spark_column_names sdf_before = self.to_frame(name=index_name)[:loc].to_spark() sdf_middle = Index([item]).to_frame(name=index_name).to_spark() sdf_after = self.to_frame(name=index_name)[loc:].to_spark() sdf = sdf_before.union(sdf_middle).union(sdf_after) internal = InternalFrame( # TODO: dtypes? spark_frame=sdf, index_spark_columns=[ scol_for(sdf, col) for col in self._internal.index_spark_column_names ], index_names=self._internal.index_names, ) return DataFrame(internal).index
def transform_batch(self, func, *args, **kwargs) -> Union["DataFrame", "Series"]: """ Transform chunks with a function that takes pandas DataFrame and outputs pandas DataFrame. The pandas DataFrame given to the function is of a batch used internally. The length of each input and output should be the same. See also `Transform and apply a function <https://koalas.readthedocs.io/en/latest/user_guide/transform_apply.html>`_. .. note:: the `func` is unable to access to the whole input frame. Koalas internally splits the input series into multiple batches and calls `func` with each batch multiple times. Therefore, operations such as global aggregations are impossible. See the example below. >>> # This case does not return the length of whole frame but of the batch internally ... # used. ... def length(pdf) -> ks.DataFrame[int]: ... return pd.DataFrame([len(pdf)] * len(pdf)) ... >>> df = ks.DataFrame({'A': range(1000)}) >>> df.koalas.transform_batch(length) # doctest: +SKIP c0 0 83 1 83 2 83 ... .. note:: this API executes the function once to infer the type which is potentially expensive, for instance, when the dataset is created after aggregations or sorting. To avoid this, specify return type in ``func``, for instance, as below: >>> def plus_one(x) -> ks.DataFrame[float, float]: ... return x + 1 If the return type is specified, the output column names become `c0, c1, c2 ... cn`. These names are positionally mapped to the returned DataFrame in ``func``. To specify the column names, you can assign them in a pandas friendly style as below: >>> def plus_one(x) -> ks.DataFrame['a': float, 'b': float]: ... return x + 1 >>> pdf = pd.DataFrame({'a': [1, 2, 3], 'b': [3, 4, 5]}) >>> def plus_one(x) -> ks.DataFrame[zip(pdf.dtypes, pdf.columns)]: ... return x + 1 When the given function returns DataFrame and has the return type annotated, the original index of the DataFrame will be lost and then a default index will be attached to the result. Please be careful about configuring the default index. See also `Default Index Type <https://koalas.readthedocs.io/en/latest/user_guide/options.html#default-index-type>`_. Parameters ---------- func : function Function to transform each pandas frame. *args Positional arguments to pass to func. **kwargs Keyword arguments to pass to func. Returns ------- DataFrame or Series See Also -------- DataFrame.koalas.apply_batch: For row/columnwise operations. Series.koalas.transform_batch: transform the search as each pandas chunks. Examples -------- >>> df = ks.DataFrame([(1, 2), (3, 4), (5, 6)], columns=['A', 'B']) >>> df A B 0 1 2 1 3 4 2 5 6 >>> def plus_one_func(pdf) -> ks.DataFrame[int, int]: ... return pdf + 1 >>> df.koalas.transform_batch(plus_one_func) c0 c1 0 2 3 1 4 5 2 6 7 >>> def plus_one_func(pdf) -> ks.DataFrame['A': int, 'B': int]: ... return pdf + 1 >>> df.koalas.transform_batch(plus_one_func) A B 0 2 3 1 4 5 2 6 7 >>> def plus_one_func(pdf) -> ks.Series[int]: ... return pdf.B + 1 >>> df.koalas.transform_batch(plus_one_func) 0 3 1 5 2 7 dtype: int64 You can also omit the type hints so Koalas infers the return schema as below: >>> df.koalas.transform_batch(lambda pdf: pdf + 1) A B 0 2 3 1 4 5 2 6 7 >>> (df * -1).koalas.transform_batch(abs) A B 0 1 2 1 3 4 2 5 6 Note that you should not transform the index. The index information will not change. >>> df.koalas.transform_batch(lambda pdf: pdf.B + 1) 0 3 1 5 2 7 Name: B, dtype: int64 You can also specify extra arguments as below. >>> df.koalas.transform_batch(lambda pdf, a, b, c: pdf.B + a + b + c, 1, 2, c=3) 0 8 1 10 2 12 Name: B, dtype: int64 """ from databricks.koalas.groupby import GroupBy from databricks.koalas.frame import DataFrame from databricks.koalas.series import first_series from databricks import koalas as ks assert callable(func), "the first argument should be a callable function." spec = inspect.getfullargspec(func) return_sig = spec.annotations.get("return", None) should_infer_schema = return_sig is None original_func = func func = lambda o: original_func(o, *args, **kwargs) names = self._kdf._internal.to_internal_spark_frame.schema.names should_by_pass = LooseVersion(pyspark.__version__) >= "3.0" def pandas_concat(series): # The input can only be a DataFrame for struct from Spark 3.0. # This works around to make the input as a frame. See SPARK-27240 pdf = pd.concat(series, axis=1) pdf.columns = names return pdf def apply_func(pdf): return func(pdf).to_frame() def pandas_extract(pdf, name): # This is for output to work around a DataFrame for struct # from Spark 3.0. See SPARK-23836 return pdf[name] def pandas_series_func(f, by_pass): ff = f if by_pass: return lambda *series: first_series(ff(*series)) else: return lambda *series: first_series(ff(pandas_concat(series))) def pandas_frame_func(f, field_name): ff = f return lambda *series: pandas_extract(ff(pandas_concat(series)), field_name) if should_infer_schema: # Here we execute with the first 1000 to get the return type. # If the records were less than 1000, it uses pandas API directly for a shortcut. limit = ks.get_option("compute.shortcut_limit") pdf = self._kdf.head(limit + 1)._to_internal_pandas() transformed = func(pdf) if not isinstance(transformed, (pd.DataFrame, pd.Series)): raise ValueError( "The given function should return a frame; however, " "the return type was %s." % type(transformed) ) if len(transformed) != len(pdf): raise ValueError("transform_batch cannot produce aggregated results") kdf_or_kser = ks.from_pandas(transformed) if isinstance(kdf_or_kser, ks.Series): kser = cast(ks.Series, kdf_or_kser) spark_return_type = force_decimal_precision_scale( as_nullable_spark_type(kser.spark.data_type) ) return_schema = StructType( [StructField(SPARK_DEFAULT_SERIES_NAME, spark_return_type)] ) output_func = GroupBy._make_pandas_df_builder_func( self._kdf, apply_func, return_schema, retain_index=False ) pudf = pandas_udf( pandas_series_func(output_func, should_by_pass), returnType=spark_return_type, functionType=PandasUDFType.SCALAR, ) columns = self._kdf._internal.spark_columns # TODO: Index will be lost in this case. internal = self._kdf._internal.copy( column_labels=kser._internal.column_labels, data_spark_columns=[ (pudf(F.struct(*columns)) if should_by_pass else pudf(*columns)).alias( kser._internal.data_spark_column_names[0] ) ], data_dtypes=kser._internal.data_dtypes, column_label_names=kser._internal.column_label_names, ) return first_series(DataFrame(internal)) else: kdf = cast(DataFrame, kdf_or_kser) if len(pdf) <= limit: # only do the short cut when it returns a frame to avoid # operations on different dataframes in case of series. return kdf # Force nullability. return_schema = force_decimal_precision_scale( as_nullable_spark_type(kdf._internal.to_internal_spark_frame.schema) ) self_applied = DataFrame(self._kdf._internal.resolved_copy) # type: DataFrame output_func = GroupBy._make_pandas_df_builder_func( self_applied, func, return_schema, retain_index=True ) columns = self_applied._internal.spark_columns if should_by_pass: pudf = pandas_udf( output_func, returnType=return_schema, functionType=PandasUDFType.SCALAR ) temp_struct_column = verify_temp_column_name( self_applied._internal.spark_frame, "__temp_struct__" ) applied = pudf(F.struct(*columns)).alias(temp_struct_column) sdf = self_applied._internal.spark_frame.select(applied) sdf = sdf.selectExpr("%s.*" % temp_struct_column) else: applied = [] for field in return_schema.fields: applied.append( pandas_udf( pandas_frame_func(output_func, field.name), returnType=field.dataType, functionType=PandasUDFType.SCALAR, )(*columns).alias(field.name) ) sdf = self_applied._internal.spark_frame.select(*applied) return DataFrame(kdf._internal.with_new_sdf(sdf)) else: return_type = infer_return_type(original_func) is_return_series = isinstance(return_type, SeriesType) is_return_dataframe = isinstance(return_type, DataFrameType) if not is_return_dataframe and not is_return_series: raise TypeError( "The given function should specify a frame or series as its type " "hints; however, the return type was %s." % return_sig ) if is_return_series: spark_return_type = force_decimal_precision_scale( as_nullable_spark_type(cast(SeriesType, return_type).spark_type) ) return_schema = StructType( [StructField(SPARK_DEFAULT_SERIES_NAME, spark_return_type)] ) output_func = GroupBy._make_pandas_df_builder_func( self._kdf, apply_func, return_schema, retain_index=False ) pudf = pandas_udf( pandas_series_func(output_func, should_by_pass), returnType=spark_return_type, functionType=PandasUDFType.SCALAR, ) columns = self._kdf._internal.spark_columns internal = self._kdf._internal.copy( column_labels=[None], data_spark_columns=[ (pudf(F.struct(*columns)) if should_by_pass else pudf(*columns)).alias( SPARK_DEFAULT_SERIES_NAME ) ], data_dtypes=[cast(SeriesType, return_type).dtype], column_label_names=None, ) return first_series(DataFrame(internal)) else: return_schema = cast(DataFrameType, return_type).spark_type self_applied = DataFrame(self._kdf._internal.resolved_copy) output_func = GroupBy._make_pandas_df_builder_func( self_applied, func, return_schema, retain_index=False ) columns = self_applied._internal.spark_columns if should_by_pass: pudf = pandas_udf( output_func, returnType=return_schema, functionType=PandasUDFType.SCALAR ) temp_struct_column = verify_temp_column_name( self_applied._internal.spark_frame, "__temp_struct__" ) applied = pudf(F.struct(*columns)).alias(temp_struct_column) sdf = self_applied._internal.spark_frame.select(applied) sdf = sdf.selectExpr("%s.*" % temp_struct_column) else: applied = [] for field in return_schema.fields: applied.append( pandas_udf( pandas_frame_func(output_func, field.name), returnType=field.dataType, functionType=PandasUDFType.SCALAR, )(*columns).alias(field.name) ) sdf = self_applied._internal.spark_frame.select(*applied) internal = InternalFrame( spark_frame=sdf, index_spark_columns=None, data_dtypes=cast(DataFrameType, return_type).dtypes, ) return DataFrame(internal)