Python get_spark_config Examples

Example #1

File: pyspark_step_launcher.py Project: punneng/dagster

from dagster.core.events import log_step_event
from dagster.core.execution.plan.external_step import (
    PICKLED_EVENTS_FILE_NAME,
    PICKLED_STEP_RUN_REF_FILE_NAME,
    step_context_to_step_run_ref,
)

# On EMR, Spark is installed here
EMR_SPARK_HOME = '/usr/lib/spark/'

CODE_ZIP_NAME = 'code.zip'


@resource({
    'spark_config':
    get_spark_config(),
    'cluster_id':
    Field(StringSource,
          description='Name of the job flow (cluster) on which to execute'),
    'region_name':
    Field(StringSource),
    'action_on_failure':
    Field(str, is_required=False, default_value='CANCEL_AND_WAIT'),
    'staging_bucket':
    Field(
        StringSource,
        is_required=True,
        description=
        'S3 bucket to use for passing files between the plan process and EMR '
        'process.',
    ),

Example #2

File: pyspark_step_launcher.py Project: markjm610/dagster

from dagster.core.events import log_step_event
from dagster.core.execution.plan.external_step import (
    PICKLED_EVENTS_FILE_NAME,
    PICKLED_STEP_RUN_REF_FILE_NAME,
    step_context_to_step_run_ref,
)

# On EMR, Spark is installed here
EMR_SPARK_HOME = "/usr/lib/spark/"

CODE_ZIP_NAME = "code.zip"


@resource(
    {
        "spark_config": get_spark_config(),
        "cluster_id": Field(
            StringSource, description="Name of the job flow (cluster) on which to execute."
        ),
        "region_name": Field(StringSource, description="The AWS region that the cluster is in."),
        "action_on_failure": Field(
            str,
            is_required=False,
            default_value="CANCEL_AND_WAIT",
            description="The EMR action to take when the cluster step fails: "
            "https://docs.aws.amazon.com/emr/latest/APIReference/API_StepConfig.html",
        ),
        "staging_bucket": Field(
            StringSource,
            is_required=True,
            description="S3 bucket to use for passing files between the plan process and EMR "