#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""This module provides some common test case bases for
SparklingPandasTestCases"""
from sparklingpandas.utils import add_pyspark_path
import pandas
add_pyspark_path()
from pyspark import SparkConf
from sparklingpandas.pcontext import PSparkContext
import unittest2
import sys
from pandas.util.testing import assert_frame_equal
import logging
class SparklingPandasTestCase(unittest2.TestCase):
"""Basic SparklingPandasTestCase, inherit from this class to get a
PSparkContext as spark_ctx."""
def setUp(self):
"""Setup the basic panda spark test case. This right now just creates a
PSparkContext."""
logging.info("Setting up spark context")
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from sparklingpandas.utils import add_pyspark_path
add_pyspark_path()
import pandas
class PStats:
"""A object to wrap the stats/aggregation values"""
def __init__(self, dataframe):
self._df = dataframe.collect()
def __getitem__(self, key):
return PStatsOnColumn(self._df, key)
class PStatsOnColumn:
def __init__(self, df, key):
self._df = df