def _register_engines() -> None:
register_execution_engine(
"spark",
lambda conf, **kwargs: SparkExecutionEngine(conf=conf),
on_dup="ignore",
)
register_execution_engine(
SparkSession,
lambda session, conf, **kwargs: SparkExecutionEngine(session, conf=conf),
on_dup="ignore",
)
def make_engine(self):
session = SparkSession.builder.getOrCreate()
e = SparkExecutionEngine(
session, {"test": True, "fugue.spark.use_pandas_udf": True}
)
assert e.conf.get_or_throw("fugue.spark.use_pandas_udf", bool)
return e
def make_engine(self):
session = SparkSession.builder.getOrCreate()
e = SparkExecutionEngine(
session,
{
"test": True,
"fugue.rpc.server": "fugue.rpc.flask.FlaskRPCServer",
"fugue.rpc.flask_server.host": "127.0.0.1",
"fugue.rpc.flask_server.port": "1234",
"fugue.rpc.flask_server.timeout": "2 sec",
"spark.sql.shuffle.partitions": "10",
},
)
return e
def make_engine(self):
session = SparkSession.builder.getOrCreate()
e = SparkExecutionEngine(
session,
{
"test": True,
"fugue.spark.use_pandas_udf": True,
"fugue.rpc.server": "fugue.rpc.flask.FlaskRPCServer",
"fugue.rpc.flask_server.host": "127.0.0.1",
"fugue.rpc.flask_server.port": "1234",
"fugue.rpc.flask_server.timeout": "2 sec",
"spark.sql.shuffle.partitions": "10",
},
)
assert e.conf.get_or_throw("fugue.spark.use_pandas_udf", bool)
return e
def make_engine(self):
session = SparkSession.builder.getOrCreate()
e = SparkExecutionEngine(session, dict(test=True))
return e