def finish_job(self, job_state):
stderr = stdout = ''
job_wrapper = job_state.job_wrapper
try:
client = self.get_client_from_state(job_state)
run_results = client.full_status()
remote_metadata_directory = run_results.get(
"metadata_directory", None)
stdout = run_results.get('stdout', '')
stderr = run_results.get('stderr', '')
exit_code = run_results.get('returncode', None)
pulsar_outputs = PulsarOutputs.from_status_response(run_results)
# Use Pulsar client code to transfer/copy files back
# and cleanup job if needed.
completed_normally = \
job_wrapper.get_state() not in [model.Job.states.ERROR, model.Job.states.DELETED]
cleanup_job = job_wrapper.cleanup_job
client_outputs = self.__client_outputs(client, job_wrapper)
finish_args = dict(client=client,
job_completed_normally=completed_normally,
cleanup_job=cleanup_job,
client_outputs=client_outputs,
pulsar_outputs=pulsar_outputs)
failed = pulsar_finish_job(**finish_args)
if failed:
job_wrapper.fail(
"Failed to find or download one or more job outputs from remote server.",
exception=True)
except Exception:
self.fail_job(job_state,
message=GENERIC_REMOTE_ERROR,
exception=True)
log.exception("failure finishing job %d", job_wrapper.job_id)
return
if not PulsarJobRunner.__remote_metadata(client):
self._handle_metadata_externally(job_wrapper,
resolve_requirements=True)
# Finish the job
try:
job_metrics_directory = os.path.join(job_wrapper.working_directory,
"metadata")
# Following check is a hack for jobs started during 19.01 or earlier release
# and finishing with a 19.05 code base. Eliminate the hack in 19.09 or later
# along with hacks for legacy metadata compute strategy.
if not os.path.exists(job_metrics_directory) or not any([
"__instrument" in f
for f in os.listdir(job_metrics_directory)
]):
job_metrics_directory = job_wrapper.working_directory
job_wrapper.finish(
stdout,
stderr,
exit_code,
remote_metadata_directory=remote_metadata_directory,
job_metrics_directory=job_metrics_directory,
)
except Exception:
log.exception("Job wrapper finish method failed")
job_wrapper.fail("Unable to finish job", exception=True)
def finish_job(self, job_state):
stderr = stdout = ''
job_wrapper = job_state.job_wrapper
try:
client = self.get_client_from_state(job_state)
run_results = client.full_status()
remote_working_directory = run_results.get("working_directory",
None)
remote_metadata_directory = run_results.get(
"metadata_directory", None)
stdout = run_results.get('stdout', '')
stderr = run_results.get('stderr', '')
exit_code = run_results.get('returncode', None)
pulsar_outputs = PulsarOutputs.from_status_response(run_results)
# Use Pulsar client code to transfer/copy files back
# and cleanup job if needed.
completed_normally = \
job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ]
cleanup_job = job_wrapper.cleanup_job
client_outputs = self.__client_outputs(client, job_wrapper)
finish_args = dict(client=client,
job_completed_normally=completed_normally,
cleanup_job=cleanup_job,
client_outputs=client_outputs,
pulsar_outputs=pulsar_outputs)
failed = pulsar_finish_job(**finish_args)
if failed:
job_wrapper.fail(
"Failed to find or download one or more job outputs from remote server.",
exception=True)
except Exception:
message = GENERIC_REMOTE_ERROR
job_wrapper.fail(message, exception=True)
log.exception("failure finishing job %d" % job_wrapper.job_id)
return
if not PulsarJobRunner.__remote_metadata(client):
self._handle_metadata_externally(job_wrapper,
resolve_requirements=True)
# Finish the job
try:
job_wrapper.finish(
stdout,
stderr,
exit_code,
remote_working_directory=remote_working_directory,
remote_metadata_directory=remote_metadata_directory,
)
except Exception:
log.exception("Job wrapper finish method failed")
job_wrapper.fail("Unable to finish job", exception=True)
def finish_job( self, job_state ):
stderr = stdout = ''
job_wrapper = job_state.job_wrapper
try:
client = self.get_client_from_state(job_state)
run_results = client.full_status()
remote_working_directory = run_results.get("working_directory", None)
remote_metadata_directory = run_results.get("metadata_directory", None)
stdout = run_results.get('stdout', '')
stderr = run_results.get('stderr', '')
exit_code = run_results.get('returncode', None)
pulsar_outputs = PulsarOutputs.from_status_response(run_results)
# Use Pulsar client code to transfer/copy files back
# and cleanup job if needed.
completed_normally = \
job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ]
cleanup_job = job_wrapper.cleanup_job
client_outputs = self.__client_outputs(client, job_wrapper)
finish_args = dict( client=client,
job_completed_normally=completed_normally,
cleanup_job=cleanup_job,
client_outputs=client_outputs,
pulsar_outputs=pulsar_outputs )
failed = pulsar_finish_job( **finish_args )
if failed:
job_wrapper.fail("Failed to find or download one or more job outputs from remote server.", exception=True)
except Exception:
message = GENERIC_REMOTE_ERROR
job_wrapper.fail( message, exception=True )
log.exception("failure finishing job %d" % job_wrapper.job_id)
return
if not PulsarJobRunner.__remote_metadata( client ):
self._handle_metadata_externally( job_wrapper, resolve_requirements=True )
# Finish the job
try:
job_wrapper.finish(
stdout,
stderr,
exit_code,
remote_working_directory=remote_working_directory,
remote_metadata_directory=remote_metadata_directory,
)
except Exception:
log.exception("Job wrapper finish method failed")
job_wrapper.fail("Unable to finish job", exception=True)
