def test_jar_not_found():
spark_solid = create_spark_solid("spark_solid", main_class="something")
# guid guaranteed to not exist
run_config = yaml.safe_load(CONFIG_FILE.format(path=str(uuid.uuid4())))
result = execute_solid(spark_solid,
run_config=run_config,
raise_on_error=False,
mode_def=MODE_DEF)
assert result.failure_data
assert ("does not exist. A valid jar must be built before running this op."
in result.failure_data.error.cause.message)
def test_jar_not_found():
spark_solid = create_spark_solid('spark_solid', main_class='something')
# guid guaranteed to not exist
environment_dict = yaml.safe_load(
CONFIG_FILE.format(path=str(uuid.uuid4())))
result = execute_solid(spark_solid,
environment_dict=environment_dict,
raise_on_error=False,
mode_def=MODE_DEF)
assert result.failure_data
assert (
'does not exist. A valid jar must be built before running this solid.'
in result.failure_data.error.message)
def event_ingest_pipeline():
event_ingest = create_spark_solid(
name='event_ingest',
main_class='io.dagster.events.EventPipeline',
description='Ingest events from JSON to Parquet',
)
@solid(input_defs=[InputDefinition('start', Nothing)], required_resource_keys={'snowflake'})
def snowflake_load(context):
# TODO: express dependency of this solid on event_ingest
context.resources.snowflake.load_table_from_local_parquet(
src='file:///tmp/dagster/events/data/output/2019/01/01/*.parquet', table='events'
)
snowflake_load(event_ingest(start=gunzipper(gzip_file=download_from_s3_to_file())))
def test_no_spark_home():
if "SPARK_HOME" in os.environ:
del os.environ["SPARK_HOME"]
spark_solid = create_spark_solid("spark_solid", main_class="something")
run_config = yaml.safe_load(
NO_SPARK_HOME_CONFIG_FILE.format(path=file_relative_path(__file__, "."))
)
result = execute_solid(
spark_solid, run_config=run_config, raise_on_error=False, mode_def=MODE_DEF
)
assert result.failure_data
assert (
"No spark home set. You must either pass spark_home in config or set "
"$SPARK_HOME in your environment (got None)." in result.failure_data.error.cause.message
)
def test_no_spark_home():
if 'SPARK_HOME' in os.environ:
del os.environ['SPARK_HOME']
spark_solid = create_spark_solid('spark_solid', main_class='something')
run_config = yaml.safe_load(
NO_SPARK_HOME_CONFIG_FILE.format(
path=file_relative_path(__file__, '.')))
result = execute_solid(spark_solid,
run_config=run_config,
raise_on_error=False,
mode_def=MODE_DEF)
assert result.failure_data
assert (
'No spark home set. You must either pass spark_home in config or set '
'$SPARK_HOME in your environment (got None).'
in result.failure_data.error.message)
def pipe():
for solid_name in ["first_pi", "second_pi", "third_pi"]:
create_spark_solid(
solid_name, main_class="org.apache.spark.examples.SparkPi")()
def pipe():
for solid_name in ['first_pi', 'second_pi', 'third_pi']:
create_spark_solid(
solid_name, main_class='org.apache.spark.examples.SparkPi')()
