def run_job_local(work_queue):
"""
Localhost jobs manager process for consume mode
"""
global localhos_handler
global last_job_key
pull_runtime = standalone_config.get('pull_runtime', False)
def wait_job_completed(job_key):
done = os.path.join(JOBS_DIR, job_key+'.done')
while True:
if os.path.isfile(done):
break
time.sleep(1)
try:
localhos_handler = LocalhostHandler({'pull_runtime': pull_runtime})
while True:
job_payload = work_queue.get()
job_key = job_payload['job_key']
last_job_key = job_key
job_payload['config']['lithops']['backend'] = 'localhost'
localhos_handler.invoke(job_payload)
wait_job_completed(job_key)
except Exception as e:
logger.error(e)
def run_worker(master_ip, job_key):
"""
Run a job
"""
global BUDGET_KEEPER
while True:
url = 'http://{}:{}/get-task/{}'.format(master_ip, STANDALONE_SERVICE_PORT, job_key)
logger.info('Getting task from {}'.format(url))
resp = requests.get(url)
if resp.status_code != 200:
logger.info('All tasks completed'.format(url))
return
job_payload = resp.json()
logger.info(job_payload)
logger.info('Got tasks {}'.format(', '.join(job_payload['call_ids'])))
try:
runtime = job_payload['runtime_name']
verify_runtime_name(runtime)
except Exception:
return
BUDGET_KEEPER.last_usage_time = time.time()
BUDGET_KEEPER.update_config(job_payload['config']['standalone'])
BUDGET_KEEPER.jobs[job_payload['job_key']] = 'running'
pull_runtime = STANDALONE_CONFIG.get('pull_runtime', False)
localhost_handler = LocalhostHandler({'runtime': runtime, 'pull_runtime': pull_runtime})
localhost_handler.invoke(job_payload)
wait_job_completed(job_key)
def run():
"""
Run a job locally, in consume mode
"""
global BUDGET_KEEPER
global WORK_QUEUES
global JOB_PROCESSES
job_payload = flask.request.get_json(force=True, silent=True)
if job_payload and not isinstance(job_payload, dict):
return error('The action did not receive a dictionary as an argument.')
try:
runtime = job_payload['runtime_name']
verify_runtime_name(runtime)
except Exception as e:
return error(str(e))
job_key = job_payload['job_key']
logger.info('Received job {}'.format(job_key))
BUDGET_KEEPER.last_usage_time = time.time()
BUDGET_KEEPER.update_config(job_payload['config']['standalone'])
BUDGET_KEEPER.jobs[job_key] = 'running'
exec_mode = job_payload['config']['standalone'].get('exec_mode', 'consume')
if exec_mode == 'consume':
# Consume mode runs the job locally
pull_runtime = STANDALONE_CONFIG.get('pull_runtime', False)
try:
localhost_handler = LocalhostHandler({
'runtime': runtime,
'pull_runtime': pull_runtime
})
localhost_handler.invoke(job_payload)
except Exception as e:
logger.error(e)
elif exec_mode == 'create':
# Create mode runs the job in worker VMs
work_queue = MP_MANAGER.Queue()
WORK_QUEUES[job_key] = work_queue
jp = mp.Process(target=run_job_process, args=(job_payload, work_queue))
jp.daemon = True
jp.start()
JOB_PROCESSES[job_key] = jp
act_id = str(uuid.uuid4()).replace('-', '')[:12]
response = flask.jsonify({'activationId': act_id})
response.status_code = 202
return response
def run_worker(master_ip, work_queue):
"""
Run a job
"""
global budget_keeper
global localhos_handler
global last_job_key
pull_runtime = stanbdalone_config.get('pull_runtime', False)
localhos_handler = LocalhostHandler({'pull_runtime': pull_runtime})
while True:
url = f'http://{master_ip}:{SA_SERVICE_PORT}/get-task/{work_queue}'
logger.debug(f'Getting task from {url}')
try:
resp = requests.get(url)
except Exception:
time.sleep(1)
continue
if resp.status_code != 200:
if stanbdalone_config.get('exec_mode') == 'reuse':
time.sleep(1)
continue
else:
logger.debug('All tasks completed'.format(url))
return
job_payload = resp.json()
logger.debug(job_payload)
logger.debug('Got tasks {}'.format(', '.join(job_payload['call_ids'])))
try:
runtime = job_payload['runtime_name']
verify_runtime_name(runtime)
except Exception:
return
job_key = job_payload['job_key']
last_job_key = job_key
budget_keeper.last_usage_time = time.time()
budget_keeper.update_config(job_payload['config']['standalone'])
budget_keeper.jobs[job_key] = 'running'
try:
localhos_handler.invoke(job_payload)
except Exception as e:
logger.error(e)
wait_job_completed(job_key)
