def _run_step_on_spark(self, step, step_num):
if self._opts['upload_archives']:
log.warning('Spark master %r will probably ignore archives' %
self._spark_master())
step_args = self._args_for_spark_step(step_num)
env = dict(os.environ)
env.update(self._spark_cmdenv(step_num))
log.debug('> %s' % cmd_line(step_args))
log.debug(' with environment: %r' % sorted(env.items()))
def _log_line(line):
log.info(' %s' % to_unicode(line).strip('\r\n'))
# try to use a PTY if it's available
try:
pid, master_fd = pty.fork()
except (AttributeError, OSError):
# no PTYs, just use Popen
# user won't get much feedback for a while, so tell them
# spark-submit is running
log.debug('No PTY available, using Popen() to invoke spark-submit')
step_proc = Popen(step_args, stdout=PIPE, stderr=PIPE, env=env)
for line in step_proc.stderr:
for record in _parse_hadoop_log4j_records(
_yield_lines_from_pty_or_pipe(step_proc.stderr)):
_log_log4j_record(record)
# there shouldn't be much output on STDOUT
for line in step_proc.stdout:
_log_line_from_driver(line)
step_proc.stdout.close()
step_proc.stderr.close()
returncode = step_proc.wait()
else:
# we have PTYs
if pid == 0: # we are the child process
os.execvpe(step_args[0], step_args, env)
else:
log.debug('Invoking spark-submit via PTY')
with os.fdopen(master_fd, 'rb') as master:
for record in _parse_hadoop_log4j_records(
_yield_lines_from_pty_or_pipe(master)):
_log_log4j_record(record)
_, returncode = os.waitpid(pid, 0)
if returncode:
reason = str(CalledProcessError(returncode, step_args))
raise StepFailedException(reason=reason,
step_num=step_num,
num_steps=self._num_steps())
def _log_record_from_hadoop(record):
"""Log log4j record parsed from hadoop stderr."""
if not _is_counter_log4j_record(record): # counters are printed separately
_log_log4j_record(record)
def _log_record_from_hadoop(record):
"""Log log4j record parsed from hadoop stderr."""
if not _is_counter_log4j_record(record): # counters are printed separately
_log_log4j_record(record)