# # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # """This module provides some common test case bases for SparklingPandasTestCases""" from sparklingpandas.utils import add_pyspark_path import pandas add_pyspark_path() from pyspark import SparkConf from sparklingpandas.pcontext import PSparkContext import unittest2 import sys from pandas.util.testing import assert_frame_equal import logging class SparklingPandasTestCase(unittest2.TestCase): """Basic SparklingPandasTestCase, inherit from this class to get a PSparkContext as spark_ctx.""" def setUp(self): """Setup the basic panda spark test case. This right now just creates a PSparkContext.""" logging.info("Setting up spark context")
# this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from sparklingpandas.utils import add_pyspark_path add_pyspark_path() import pandas class PStats: """A object to wrap the stats/aggregation values""" def __init__(self, dataframe): self._df = dataframe.collect() def __getitem__(self, key): return PStatsOnColumn(self._df, key) class PStatsOnColumn: def __init__(self, df, key): self._df = df