def job_handler(event,
job_i,
run_dir,
extra_context=None,
delete_taskdir=True):
"""
Run a deserialized job in run_dir
Just for debugging
"""
debug_pid = open(
"/tmp/pywren.scripts.standalone.{}.{}.log".format(
os.getpid(), time.time()), 'w')
call_id = event['call_id']
callset_id = event['callset_id']
logger.info("jobhandler_thread callset_id={} call_id={}".format(
callset_id, call_id))
original_dir = os.getcwd()
task_run_dir = os.path.join(run_dir, str(job_i))
shutil.rmtree(task_run_dir, True) # delete old modules
os.makedirs(task_run_dir)
copy_runtime(task_run_dir)
context = {'jobnum': job_i}
if extra_context is not None:
context.update(extra_context)
os.chdir(task_run_dir)
try:
debug_pid.write("invoking generic_handler\n")
logger.debug(
"jobhandler_thread callset_id={} call_id={} invoking".format(
callset_id, call_id))
wrenhandler.generic_handler(event, context)
except Exception as e:
logger.warning("jobhandler_thread callset_id={} "
"call_id={} exception={}".format(
callset_id, call_id, str(e)))
finally:
debug_pid.write("generic handler finally\n")
if delete_taskdir:
shutil.rmtree(task_run_dir)
os.chdir(original_dir)
debug_pid.write("done and returning\n")
logger.debug("jobhandler_thread callset_id={} call_id={} returning".format(
callset_id, call_id))
def job_handler(job,
job_i,
run_dir,
aws_region,
server_name,
log_stream_prefix,
extra_context=None,
delete_taskdir=True):
"""
Run a deserialized job in run_dir
Just for debugging
"""
session = boto3.session.Session(region_name=aws_region)
# we do this here instead of in the global context
# because of how multiprocessing works
handler = watchtower.CloudWatchLogHandler(send_interval=20,
log_group="pywren.standalone",
stream_name=log_stream_prefix +
"-{logger_name}",
boto3_session=session,
max_batch_count=10)
log_format_str = '{} %(asctime)s - %(name)s - %(levelname)s - %(message)s'.format(
server_name)
formatter = logging.Formatter(log_format_str, "%Y-%m-%d %H:%M:%S")
handler.setFormatter(formatter)
wren_log = pywren.wrenhandler.logger # logging.getLogger('pywren.wrenhandler')
wren_log.setLevel(logging.DEBUG)
wren_log.propagate = 0
wren_log.addHandler(handler)
original_dir = os.getcwd()
task_run_dir = os.path.join(run_dir, str(job_i))
shutil.rmtree(task_run_dir, True) # delete old modules
os.makedirs(task_run_dir)
copy_runtime(task_run_dir)
context = {'jobnum': job_i}
if extra_context is not None:
context.update(extra_context)
os.chdir(task_run_dir)
try:
wrenhandler.generic_handler(job, context)
finally:
if delete_taskdir:
shutil.rmtree(task_run_dir)
os.chdir(original_dir)
handler.flush()
