def run_job_local(work_queue): """ Localhost jobs manager process for consume mode """ global localhos_handler global last_job_key pull_runtime = standalone_config.get('pull_runtime', False) def wait_job_completed(job_key): done = os.path.join(JOBS_DIR, job_key+'.done') while True: if os.path.isfile(done): break time.sleep(1) try: localhos_handler = LocalhostHandler({'pull_runtime': pull_runtime}) while True: job_payload = work_queue.get() job_key = job_payload['job_key'] last_job_key = job_key job_payload['config']['lithops']['backend'] = 'localhost' localhos_handler.invoke(job_payload) wait_job_completed(job_key) except Exception as e: logger.error(e)
def run_worker(master_ip, job_key): """ Run a job """ global BUDGET_KEEPER while True: url = 'http://{}:{}/get-task/{}'.format(master_ip, STANDALONE_SERVICE_PORT, job_key) logger.info('Getting task from {}'.format(url)) resp = requests.get(url) if resp.status_code != 200: logger.info('All tasks completed'.format(url)) return job_payload = resp.json() logger.info(job_payload) logger.info('Got tasks {}'.format(', '.join(job_payload['call_ids']))) try: runtime = job_payload['runtime_name'] verify_runtime_name(runtime) except Exception: return BUDGET_KEEPER.last_usage_time = time.time() BUDGET_KEEPER.update_config(job_payload['config']['standalone']) BUDGET_KEEPER.jobs[job_payload['job_key']] = 'running' pull_runtime = STANDALONE_CONFIG.get('pull_runtime', False) localhost_handler = LocalhostHandler({'runtime': runtime, 'pull_runtime': pull_runtime}) localhost_handler.invoke(job_payload) wait_job_completed(job_key)
def run(): """ Run a job locally, in consume mode """ global BUDGET_KEEPER global WORK_QUEUES global JOB_PROCESSES job_payload = flask.request.get_json(force=True, silent=True) if job_payload and not isinstance(job_payload, dict): return error('The action did not receive a dictionary as an argument.') try: runtime = job_payload['runtime_name'] verify_runtime_name(runtime) except Exception as e: return error(str(e)) job_key = job_payload['job_key'] logger.info('Received job {}'.format(job_key)) BUDGET_KEEPER.last_usage_time = time.time() BUDGET_KEEPER.update_config(job_payload['config']['standalone']) BUDGET_KEEPER.jobs[job_key] = 'running' exec_mode = job_payload['config']['standalone'].get('exec_mode', 'consume') if exec_mode == 'consume': # Consume mode runs the job locally pull_runtime = STANDALONE_CONFIG.get('pull_runtime', False) try: localhost_handler = LocalhostHandler({ 'runtime': runtime, 'pull_runtime': pull_runtime }) localhost_handler.invoke(job_payload) except Exception as e: logger.error(e) elif exec_mode == 'create': # Create mode runs the job in worker VMs work_queue = MP_MANAGER.Queue() WORK_QUEUES[job_key] = work_queue jp = mp.Process(target=run_job_process, args=(job_payload, work_queue)) jp.daemon = True jp.start() JOB_PROCESSES[job_key] = jp act_id = str(uuid.uuid4()).replace('-', '')[:12] response = flask.jsonify({'activationId': act_id}) response.status_code = 202 return response
def run_worker(master_ip, work_queue): """ Run a job """ global budget_keeper global localhos_handler global last_job_key pull_runtime = stanbdalone_config.get('pull_runtime', False) localhos_handler = LocalhostHandler({'pull_runtime': pull_runtime}) while True: url = f'http://{master_ip}:{SA_SERVICE_PORT}/get-task/{work_queue}' logger.debug(f'Getting task from {url}') try: resp = requests.get(url) except Exception: time.sleep(1) continue if resp.status_code != 200: if stanbdalone_config.get('exec_mode') == 'reuse': time.sleep(1) continue else: logger.debug('All tasks completed'.format(url)) return job_payload = resp.json() logger.debug(job_payload) logger.debug('Got tasks {}'.format(', '.join(job_payload['call_ids']))) try: runtime = job_payload['runtime_name'] verify_runtime_name(runtime) except Exception: return job_key = job_payload['job_key'] last_job_key = job_key budget_keeper.last_usage_time = time.time() budget_keeper.update_config(job_payload['config']['standalone']) budget_keeper.jobs[job_key] = 'running' try: localhos_handler.invoke(job_payload) except Exception as e: logger.error(e) wait_job_completed(job_key)