def _run_step_on_spark(self, step, step_num):
"""Set up a fake working directory and environment, and call the Spark
method."""
# this is kind of a Spark-specific mash-up of _run_streaming_step()
# (in sim.py) and _invoke_task_func(), above
# don't create the output dir for the step; that's Spark's job
# breaking the Spark step down into tasks is pyspark's job, so
# we just have a single dummy task
self.fs.mkdir(self._task_dir('spark', step_num, task_num=0))
# could potentially parse this for cause of error
stderr_path = self._task_stderr_path('spark', step_num, task_num=0)
stdout_path = self._task_output_path('spark', step_num, task_num=0)
self._create_dist_cache_dir(step_num)
wd = self._setup_working_dir('spark', step_num, task_num=0)
# use abspath() on input URIs before changing working dir
task_args = self._spark_script_args(step_num)
with open(stdout_path, 'wb') as stdout, \
open(stderr_path, 'wb') as stderr:
with save_current_environment(), save_cwd(), save_sys_path(), \
save_sys_std():
os.environ.update(_fix_env(self._opts['cmdenv']))
os.chdir(wd)
sys.path = [os.getcwd()] + sys.path
# pretend we redirected stdout and stderr
sys.stdout, sys.stderr = stdout, stderr
task = self._mrjob_cls(task_args)
task.execute()
def _run_step_on_spark(self, step, step_num):
"""Set up a fake working directory and environment, and call the Spark
method."""
# this is kind of a Spark-specific mash-up of _run_streaming_step()
# (in sim.py) and _invoke_task_func(), above
# don't create the output dir for the step; that's Spark's job
# breaking the Spark step down into tasks is pyspark's job, so
# we just have a single dummy task
self.fs.mkdir(self._task_dir('spark', step_num, task_num=0))
# could potentially parse this for cause of error
stderr_path = self._task_stderr_path('spark', step_num, task_num=0)
stdout_path = self._task_output_path('spark', step_num, task_num=0)
self._create_dist_cache_dir(step_num)
wd = self._setup_working_dir('spark', step_num, task_num=0)
# use abspath() on input URIs before changing working dir
task_args = self._spark_script_args(step_num)
with open(stdout_path, 'wb') as stdout, \
open(stderr_path, 'wb') as stderr:
with save_current_environment(), save_cwd(), save_sys_path(), \
save_sys_std():
os.environ.update(_fix_env(self._opts['cmdenv']))
os.chdir(wd)
sys.path = [os.getcwd()] + sys.path
# pretend we redirected stdout and stderr
sys.stdout, sys.stderr = stdout, stderr
task = self._mrjob_cls(task_args)
task.execute()
def invoke_task(stdin, stdout, stderr, wd, env):
with save_current_environment(), save_cwd(), save_sys_path(), \
save_sys_std():
# pretend we're running the script in the working dir
os.environ.update(env)
os.chdir(wd)
sys.path = [os.getcwd()] + sys.path
# pretend we've redirected stdin/stdout/stderr
sys.stdin = stdin
sys.stdout = stdout
sys.stderr = stderr
input_uri = None
try:
args = self._args_for_task(step_num, task_type)
if manifest:
# read input path from stdin, add to args
line = stdin.readline().decode('utf_8')
input_uri = line.split('\t')[-1].rstrip()
# input_uri is an absolute path, can serve
# as path and uri both
args = list(args) + [input_uri, input_uri]
task = self._mrjob_cls(args)
task.execute()
except:
# so users can figure out where the exception came from;
# see _log_cause_of_error(). we can't wrap the exception
# because then we lose the stacktrace (which is the whole
# point of the inline runner)
if input_uri: # from manifest
self._error_while_reading_from = input_uri
else:
self._error_while_reading_from = self._task_input_path(
task_type, step_num, task_num)
raise
def invoke_task(stdin, stdout, stderr, wd, env):
with save_current_environment(), save_cwd(), save_sys_path(), \
save_sys_std():
# pretend we're running the script in the working dir
os.environ.update(env)
os.chdir(wd)
sys.path = [os.getcwd()] + sys.path
# pretend we've redirected stdin/stdout/stderr
sys.stdin = stdin
sys.stdout = stdout
sys.stderr = stderr
input_uri = None
try:
args = self._args_for_task(step_num, task_type)
if manifest:
# read input path from stdin, add to args
line = stdin.readline().decode('utf_8')
input_uri = line.split('\t')[-1].rstrip()
# input_uri is an absolute path, can serve
# as path and uri both
args = list(args) + [input_uri, input_uri]
task = self._mrjob_cls(args)
task.execute()
except:
# so users can figure out where the exception came from;
# see _log_cause_of_error(). we can't wrap the exception
# because then we lose the stacktrace (which is the whole
# point of the inline runner)
if input_uri: # from manifest
self._error_while_reading_from = input_uri
else:
self._error_while_reading_from = self._task_input_path(
task_type, step_num, task_num)
raise