def spark_session(): sql = import_or_none('pyspark.sql') if sql: spark = sql.SparkSession.builder \ .master('local[2]') \ .config("spark.driver.extraJavaOptions", "-Dio.netty.tryReflectionSetAccessible=True") \ .config("spark.sql.shuffle.partitions", "2") \ .getOrCreate() return spark
def spark_session(): sql = import_or_none("pyspark.sql") if sql: spark = ( sql.SparkSession.builder.master("local[2]") .config( "spark.driver.extraJavaOptions", "-Dio.netty.tryReflectionSetAccessible=True", ) .config("spark.sql.shuffle.partitions", "2") .config("spark.driver.bindAddress", "127.0.0.1") .getOrCreate() ) return spark
from datetime import datetime import dask.dataframe as dd import numpy as np import pandas as pd import pytest from woodwork.exceptions import TypeConversionError from woodwork.logical_types import (Boolean, Categorical, Datetime, Double, Integer, NaturalLanguage) from featuretools.entityset.entityset import LTI_COLUMN_NAME, EntitySet from featuretools.tests.testing_utils import to_pandas from featuretools.utils.gen_utils import Library, import_or_none ks = import_or_none('databricks.koalas') def test_empty_es(): es = EntitySet('es') assert es.id == 'es' assert es.dataframe_dict == {} assert es.relationships == [] assert es.time_type is None @pytest.fixture def pd_df(): return pd.DataFrame({ 'id': [0, 1, 2], 'category': ['a', 'b', 'c'] }).astype({'category': 'category'})
from datetime import datetime import pandas as pd import pytest from dask import dataframe as dd from woodwork.logical_types import Categorical, Datetime, Integer from featuretools.entityset.entityset import LTI_COLUMN_NAME from featuretools.tests.testing_utils import to_pandas from featuretools.utils.gen_utils import Library, import_or_none ps = import_or_none("pyspark.pandas") @pytest.fixture def values_es(es): es.normalize_dataframe( "log", "values", "value", make_time_index=True, new_dataframe_time_index="value_time", ) return es @pytest.fixture def true_values_lti(): true_values_lti = pd.Series( [ datetime(2011, 4, 10, 10, 41, 0),
def test_import_or_none(): math = import_or_none("math") assert math.ceil(0.1) == 1 bad_lib = import_or_none("_featuretools") assert bad_lib is None
def test_import_or_none(): math = import_or_none('math') assert math.ceil(0.1) == 1 bad_lib = import_or_none('_featuretools') assert bad_lib is None