def sub(self, left: IndexOpsLike, right: Any) -> SeriesOrIndex: _sanitize_list_like(right) # Note that timestamp subtraction casts arguments to integer. This is to mimic pandas's # behaviors. pandas returns 'timedelta64[ns]' from 'datetime64[ns]'s subtraction. msg = ( "Note that there is a behavior difference of timestamp subtraction. " "The timestamp subtraction returns an integer in seconds, " "whereas pandas returns 'timedelta64[ns]'.") if isinstance(right, IndexOpsMixin) and isinstance( right.spark.data_type, (TimestampType, TimestampNTZType)): warnings.warn(msg, UserWarning) return left.astype("long") - right.astype("long") elif isinstance(right, datetime.datetime): warnings.warn(msg, UserWarning) return cast( SeriesOrIndex, left._with_new_scol( left.astype("long").spark.column - self._cast_spark_column_timestamp_to_long(SF.lit(right)), field=left._internal.data_fields[0].copy( dtype=np.dtype("int64"), spark_type=LongType()), ), ) else: raise TypeError( "Datetime subtraction can only be applied to datetime series.")
def astype(self, index_ops: IndexOpsLike, dtype: Union[str, type, Dtype]) -> IndexOpsLike: dtype, spark_type = pandas_on_spark_type(dtype) if isinstance(dtype, CategoricalDtype): return _as_categorical_type(index_ops, dtype, spark_type) elif isinstance(spark_type, BooleanType): # Cannot cast binary to boolean in Spark. # We should cast binary to str first, and cast it to boolean return index_ops.astype(str).astype(bool) elif isinstance(spark_type, StringType): return _as_string_type(index_ops, dtype) else: return _as_other_type(index_ops, dtype, spark_type)
def sub(self, left: IndexOpsLike, right: Any) -> SeriesOrIndex: # Note that timestamp subtraction casts arguments to integer. This is to mimic pandas's # behaviors. pandas returns 'timedelta64[ns]' from 'datetime64[ns]'s subtraction. msg = ( "Note that there is a behavior difference of timestamp subtraction. " "The timestamp subtraction returns an integer in seconds, " "whereas pandas returns 'timedelta64[ns]'.") if isinstance(right, IndexOpsMixin) and isinstance( right.spark.data_type, TimestampType): warnings.warn(msg, UserWarning) return left.astype("long") - right.astype("long") elif isinstance(right, datetime.datetime): warnings.warn(msg, UserWarning) return cast( SeriesOrIndex, left.spark.transform(lambda scol: scol.astype("long") - SF.lit( right).cast(as_spark_type("long"))), ) else: raise TypeError( "datetime subtraction can only be applied to datetime series.")