def run_job(encoded_payload): logger.info( "Lithops v{} - Starting kubernetes execution".format(__version__)) payload = b64str_to_dict(encoded_payload) setup_lithops_logger(payload['log_level']) job_key = payload['job_key'] idgiver_ip = os.environ['IDGIVER_POD_IP'] res = requests.get('http://{}:{}/getid/{}'.format(idgiver_ip, IDGIVER_PORT, job_key)) job_index = int(res.text) act_id = str(uuid.uuid4()).replace('-', '')[:12] os.environ['__LITHOPS_ACTIVATION_ID'] = act_id os.environ['__LITHOPS_BACKEND'] = 'k8s' logger.info("Activation ID: {} - Job Index: {}".format(act_id, job_index)) chunksize = payload['chunksize'] call_ids_ranges = [ call_ids_range for call_ids_range in iterchunks(payload['call_ids'], chunksize) ] call_ids = call_ids_ranges[job_index] data_byte_ranges = [ payload['data_byte_ranges'][int(call_id)] for call_id in call_ids ] payload['call_ids'] = call_ids payload['data_byte_ranges'] = data_byte_ranges function_handler(payload)
def main_job(action, encoded_payload): logger.info( "Lithops v{} - Starting Code Engine execution".format(__version__)) payload = b64str_to_dict(encoded_payload) setup_lithops_logger(payload['log_level']) if (action == 'preinstalls'): runtime_packages(payload) return {"Execution": "Finished"} job_index = int(os.environ['JOB_INDEX']) payload['JOB_INDEX'] = job_index logger.info("Action {}. Job Index {}".format(action, job_index)) act_id = str(uuid.uuid4()).replace('-', '')[:12] os.environ['__LITHOPS_ACTIVATION_ID'] = act_id chunksize = payload['chunksize'] call_ids_ranges = [ call_ids_range for call_ids_range in iterchunks(payload['call_ids'], chunksize) ] call_ids = call_ids_ranges[job_index] data_byte_ranges = [ payload['data_byte_ranges'][int(call_id)] for call_id in call_ids ] payload['call_ids'] = call_ids payload['data_byte_ranges'] = data_byte_ranges function_handler(payload) return {"Execution": "Finished"}
def run(): def error(): response = flask.jsonify({ 'error': 'The action did not receive a dictionary as an argument.' }) response.status_code = 404 return complete(response) message = flask.request.get_json(force=True, silent=True) if message and not isinstance(message, dict): return error() act_id = str(uuid.uuid4()).replace('-', '')[:12] os.environ['__LITHOPS_ACTIVATION_ID'] = act_id setup_logger(message['log_level']) if 'remote_invoker' in message: logger.info( "Lithops v{} - Starting Knative invoker".format(__version__)) function_invoker(message) else: logger.info( "Lithops v{} - Starting Knative execution".format(__version__)) function_handler(message) response = flask.jsonify({"activationId": act_id}) response.status_code = 202 return complete(response)
def _process_runner(self, worker_id): logger.debug('Localhost worker process {} started'.format(worker_id)) os.environ['__LITHOPS_LOCAL_EXECUTION'] = 'True' p_logger = logging.getLogger('lithops') while True: with io.StringIO() as buf, redirect_stdout(buf), redirect_stderr(buf): event = self.queue.get(block=True) if isinstance(event, ShutdownSentinel): break act_id = str(uuid.uuid4()).replace('-', '')[:12] os.environ['__LITHOPS_ACTIVATION_ID'] = act_id executor_id = event['executor_id'] job_id = event['job_id'] setup_logger(event['log_level']) p_logger.info("Lithops v{} - Starting execution".format(__version__)) function_handler(event) log_output = buf.getvalue() job_key = create_job_key(executor_id, job_id) log_file = os.path.join(LOGS_DIR, job_key+'.log') header = "Activation: '{}' ({})\n[\n".format(event['runtime_name'], act_id) tail = ']\n\n' output = log_output.replace('\n', '\n ', log_output.count('\n')-1) with open(log_file, 'a') as lf: lf.write(header+' '+output+tail) with open(FN_LOG_FILE, 'a') as lf: lf.write(header+' '+output+tail)
def main_job(action, encoded_payload): logger.info( "Lithops v{} - Starting Code Engine execution".format(__version__)) payload = b64str_to_dict(encoded_payload) setup_logger(payload['log_level']) if (action == 'preinstalls'): runtime_packages(payload) return {"Execution": "Finished"} job_index = os.environ['JOB_INDEX'] payload['JOB_INDEX'] = job_index logger.info("Action {}. Job Index {}".format(action, job_index)) act_id = str(uuid.uuid4()).replace('-', '')[:12] os.environ['__LITHOPS_ACTIVATION_ID'] = act_id payload['data_byte_range'] = payload['data_byte_range'][int(job_index)] payload['call_id'] = "{:05d}".format(int(job_index)) function_handler(payload) return {"Execution": "Finished"}
def main(event, context): # pub/sub event data is b64 encoded args = json.loads(base64.b64decode(event['data']).decode('utf-8')) setup_lithops_logger(args.get('log_level', 'INFO')) os.environ['__LITHOPS_ACTIVATION_ID'] = uuid.uuid4().hex os.environ['__LITHOPS_BACKEND'] = 'Google Cloud Functions' if 'get_preinstalls' in args: logger.info("Lithops v{} - Generating metadata".format(__version__)) internal_storage = InternalStorage( args['get_preinstalls']['storage_config']) object_key = '/'.join( [JOBS_PREFIX, args['get_preinstalls']['runtime_name'] + '.meta']) logger.info("Runtime metadata key {}".format(object_key)) runtime_meta = get_runtime_preinstalls() runtime_meta_json = json.dumps(runtime_meta) internal_storage.put_data(object_key, runtime_meta_json) elif 'remote_invoker' in args: logger.info( "Lithops v{} - Starting Google Cloud Functions invoker".format( __version__)) function_invoker(args) else: logger.info( "Lithops v{} - Starting Google Cloud Functions execution".format( __version__)) function_handler(args) return {"Execution": "Finished"}
def run(): sys.stdout = log_file_stream sys.stderr = log_file_stream job_filename = sys.argv[2] logger.info('Got {} job file'.format(job_filename)) with open(job_filename, 'rb') as jf: job_payload = json.load(jf) executor_id = job_payload['executor_id'] job_id = job_payload['job_id'] job_key = job_payload['job_key'] logger.info('ExecutorID {} | JobID {} - Starting execution'.format( executor_id, job_id)) act_id = str(uuid.uuid4()).replace('-', '')[:12] os.environ['__LITHOPS_ACTIVATION_ID'] = act_id os.environ['__LITHOPS_BACKEND'] = 'Localhost' try: function_handler(job_payload) except KeyboardInterrupt: pass done = os.path.join(JOBS_DIR, job_key + '.done') Path(done).touch() if os.path.exists(job_filename): os.remove(job_filename) logger.info('ExecutorID {} | JobID {} - Execution Finished'.format( executor_id, job_id))
def _process_runner(self, worker_id): logger.debug('Localhost worker process {} started'.format(worker_id)) while True: with io.StringIO() as buf, redirect_stdout(buf), redirect_stderr( buf): try: act_id = str(uuid.uuid4()).replace('-', '')[:12] os.environ['__LITHOPS_ACTIVATION_ID'] = act_id event = self.queue.get(block=True) if isinstance(event, ShutdownSentinel): break log_level = event['log_level'] default_logging_config(log_level) logger.info( "Lithops v{} - Starting execution".format(__version__)) event['extra_env']['__LITHOPS_LOCAL_EXECUTION'] = 'True' function_handler(event) except KeyboardInterrupt: break header = "Activation: '{}' ({})\n[\n".format( event['runtime_name'], act_id) tail = ']\n\n' output = buf.getvalue() output = output.replace('\n', '\n ', output.count('\n') - 1) with open(FN_LOG_FILE, 'a') as lf: lf.write(header + ' ' + output + tail)
def main(action, payload_decoded): logger.info( "Welcome to Lithops-Code-Engine entry point. Action {}".format(action)) payload = binary_to_dict(payload_decoded) log_level = payload['log_level'] default_logging_config(log_level) logger.info(payload) if (action == 'preinstals'): runtime_packages(payload) return {"Execution": "Finished"} job_index = os.environ['JOB_INDEX'] logger.info("Action {}. Job Index {}".format(action, job_index)) os.environ['__PW_ACTIVATION_ID'] = payload['activation_id'] payload['JOB_INDEX'] = job_index if 'remote_invoker' in payload: logger.info("Lithops v{} - Remote Invoker. Starting execution".format( __version__)) #function_invoker(payload) payload['data_byte_range'] = payload['job_description']['data_ranges'][ int(job_index)] for key in payload['job_description']: payload[key] = payload['job_description'][key] payload['host_submit_tstamp'] = payload['metadata'][ 'host_job_create_tstamp'] payload['call_id'] = "{:05d}".format(int(job_index)) function_handler(payload) else: logger.info("Lithops v{} - Starting execution".format(__version__)) function_handler(payload) return {"Execution": "Finished"}
def main(args): os.environ['__LITHOPS_ACTIVATION_ID'] = os.environ['__OW_ACTIVATION_ID'] if 'remote_invoker' in args: logger.info("Lithops v{} - Starting invoker".format(__version__)) function_invoker(args) else: logger.info("Lithops v{} - Starting execution".format(__version__)) function_handler(args) return {"Execution": "Finished"}
def main(event, context): args = json.loads(event) os.environ['__PW_ACTIVATION_ID'] = context.request_id if 'remote_invoker' in args: logger.info("Lithops v{} - Starting invoker".format(__version__)) function_invoker(args) else: logger.info("Lithops v{} - Starting execution".format(__version__)) function_handler(args) return {"Execution": "Finished"}
def _thread_runner(self, worker_id): logger.debug('Localhost worker process {} started'.format(worker_id)) os.environ['__LITHOPS_LOCAL_EXECUTION'] = 'True' while True: event = self.queue.get(block=True) if isinstance(event, ShutdownSentinel): break act_id = str(uuid.uuid4()).replace('-', '')[:12] os.environ['__LITHOPS_ACTIVATION_ID'] = act_id logger.info("Lithops v{} - Starting execution".format(__version__)) function_handler(event)
def main(event, context): logger.info("Starting GCP Functions function execution") args = json.loads(base64.b64decode(event['data']).decode('utf-8')) os.environ['__PW_ACTIVATION_ID'] = uuid.uuid4().hex if 'remote_invoker' in args: logger.info("Lithops v{} - Starting invoker".format(__version__)) function_invoker(args) else: logger.info("Lithops v{} - Starting execution".format(__version__)) function_handler(args) return {"Execution": "Finished"}
def lambda_handler(event, context): os.environ['__PW_ACTIVATION_ID'] = context.aws_request_id os.environ['__OW_ACTIVATION_ID'] = context.aws_request_id if 'remote_invoker' in event: logger.info("Lithops v{} - Starting invoker".format(__version__)) function_invoker(event) else: logger.info("Lithops v{} - Starting execution".format(__version__)) function_handler(event) return {"Execution": "Finished"}
def _process_runner(self, worker_id): logger.debug('Localhost worker process {} started'.format(worker_id)) while True: event = self.queue.get(block=True) if isinstance(event, ShutdownSentinel): break act_id = str(uuid.uuid4()).replace('-', '')[:12] os.environ['__LITHOPS_ACTIVATION_ID'] = act_id event['extra_env']['__LITHOPS_LOCAL_EXECUTION'] = 'True' function_handler(event)
def main(args): os.environ['__LITHOPS_ACTIVATION_ID'] = os.environ['__OW_ACTIVATION_ID'] setup_logger(args['log_level'], sys.stdout, LOGGER_FORMAT_SHORT) if 'remote_invoker' in args: logger.info("Lithops v{} - Starting invoker".format(__version__)) function_invoker(args) else: logger.info("Lithops v{} - Starting execution".format(__version__)) function_handler(args) return {"Execution": "Finished"}
def _local_handler(self, event): """ Handler to run local functions. """ if not self.log_active: old_stdout = sys.stdout sys.stdout = open(os.devnull, 'w') event['extra_env']['__PW_LOCAL_EXECUTION'] = 'True' act_id = str(uuid.uuid4()).replace('-', '')[:12] os.environ['__PW_ACTIVATION_ID'] = act_id function_handler(event) if not self.log_active: sys.stdout = old_stdout
def main(msgIn: func.QueueMessage): try: args = json.loads(msgIn.get_body()) except: args = msgIn.get_json() os.environ['__PW_ACTIVATION_ID'] = str(msgIn.id) if 'remote_invoker' in args: logger.info("Lithops v{} - Starting invoker".format(__version__)) function_invoker(args) else: logger.info("Lithops v{} - Starting execution".format(__version__)) function_handler(args) return {"Execution": "Finished"}
def main(args): os.environ['__LITHOPS_ACTIVATION_ID'] = os.environ['__OW_ACTIVATION_ID'] setup_logger(args['log_level'], sys.stdout, LOGGER_FORMAT_SHORT) if 'get_preinstalls' in args: logger.info("Lithops v{} - Generating metadata".format(__version__)) return get_runtime_preinstalls() elif 'remote_invoker' in args: logger.info("Lithops v{} - Starting IBM CF invoker".format(__version__)) function_invoker(args) else: logger.info("Lithops v{} - Starting IBM CF execution".format(__version__)) function_handler(args) return {"Execution": "Finished"}
def lambda_handler(event, context): os.environ['__LITHOPS_ACTIVATION_ID'] = context.aws_request_id setup_lithops_logger(event.get('log_level', logging.INFO)) if 'get_preinstalls' in event: logger.info("Lithops v{} - Generating metadata".format(__version__)) return get_runtime_preinstalls() elif 'remote_invoker' in event: logger.info("Lithops v{} - Starting invoker".format(__version__)) function_invoker(event) else: logger.info("Lithops v{} - Starting execution".format(__version__)) function_handler(event) return {"Execution": "Finished"}
def run_job(encoded_payload): logger.info( "Lithops v{} - Starting kubernetes execution".format(__version__)) payload = b64str_to_dict(encoded_payload) setup_lithops_logger(payload['log_level']) total_calls = payload['total_calls'] job_key = payload['job_key'] master_ip = os.environ['MASTER_POD_IP'] chunksize = payload['chunksize'] call_ids_ranges = [ call_ids_range for call_ids_range in iterchunks(payload['call_ids'], chunksize) ] data_byte_ranges = payload['data_byte_ranges'] job_finished = False while not job_finished: job_index = None while job_index is None: try: url = f'http://{master_ip}:{MASTER_PORT}/getid/{job_key}/{total_calls}' res = requests.get(url) job_index = int(res.text) except Exception: time.sleep(0.1) if job_index == -1: job_finished = True continue act_id = str(uuid.uuid4()).replace('-', '')[:12] os.environ['__LITHOPS_ACTIVATION_ID'] = act_id os.environ['__LITHOPS_BACKEND'] = 'k8s' logger.info("Activation ID: {} - Job Index: {}".format( act_id, job_index)) call_ids = call_ids_ranges[job_index] dbr = [data_byte_ranges[int(call_id)] for call_id in call_ids] payload['call_ids'] = call_ids payload['data_byte_ranges'] = dbr function_handler(payload)
def main(args): os.environ['__LITHOPS_ACTIVATION_ID'] = os.environ['__OW_ACTIVATION_ID'] log_level = args['log_level'] ow_logging_config(log_level) if 'remote_invoker' in args: logger.info( "Lithops v{} - Starting OpenWhisk Functions invoker".format( __version__)) function_invoker(args) else: logger.info( "Lithops v{} - Starting OpenWhisk Functions execution".format( __version__)) function_handler(args) return {"Execution": "Finished"}
def main(msgIn: func.QueueMessage, msgOut: func.Out[func.QueueMessage]): try: args = json.loads(msgIn.get_body()) except Exception: args = msgIn.get_json() os.environ['__LITHOPS_ACTIVATION_ID'] = str(msgIn.id) setup_lithops_logger(args['log_level']) if 'get_preinstalls' in args: logger.info("Lithops v{} - Generating metadata".format(__version__)) runtime_meta = get_runtime_preinstalls() msgOut.set(json.dumps(runtime_meta)) elif 'remote_invoker' in args: logger.info("Lithops v{} - Starting invoker".format(__version__)) function_invoker(args) else: logger.info("Lithops v{} - Starting execution".format(__version__)) function_handler(args)
def main_http(req: func.HttpRequest, context: func.Context) -> str: payload = req.get_json() setup_lithops_logger(payload['log_level']) os.environ['__LITHOPS_ACTIVATION_ID'] = context.invocation_id os.environ['__LITHOPS_BACKEND'] = 'Azure Functions (http)' if 'get_preinstalls' in payload: logger.info("Lithops v{} - Generating metadata".format(__version__)) runtime_meta = get_runtime_preinstalls() return json.dumps(runtime_meta) elif 'remote_invoker' in payload: logger.info("Lithops v{} - Starting Azure Functions (http) invoker".format(__version__)) function_invoker(payload) else: logger.info("Lithops v{} - Starting Azure Functions (http) execution".format(__version__)) function_handler(payload) return context.invocation_id
def main_queue(msgIn: func.QueueMessage, msgOut: func.Out[func.QueueMessage]): try: payload = json.loads(msgIn.get_body()) except Exception: payload = msgIn.get_json() setup_lithops_logger(payload['log_level']) os.environ['__LITHOPS_ACTIVATION_ID'] = str(msgIn.id) os.environ['__LITHOPS_BACKEND'] = 'Azure Functions (event)' if 'get_preinstalls' in payload: logger.info("Lithops v{} - Generating metadata".format(__version__)) runtime_meta = get_runtime_preinstalls() msgOut.set(json.dumps(runtime_meta)) elif 'remote_invoker' in payload: logger.info("Lithops v{} - Starting Azure Functions (event) invoker".format(__version__)) function_invoker(payload) else: logger.info("Lithops v{} - Starting Azure Functions (event) execution".format(__version__)) function_handler(payload)
def main(event, context): args = json.loads(event) os.environ['__LITHOPS_ACTIVATION_ID'] = context.request_id os.environ['__LITHOPS_BACKEND'] = 'Alibaba Function Compute' setup_lithops_logger(args['log_level']) if 'get_preinstalls' in event: logger.info("Lithops v{} - Generating metadata".format(__version__)) return get_runtime_preinstalls() elif 'remote_invoker' in args: logger.info( "Lithops v{} - Starting Alibaba Function Compute invoker".format( __version__)) function_invoker(args) else: logger.info( "Lithops v{} - Starting Alibaba Function Compute execution".format( __version__)) function_handler(args) return {"Execution": "Finished"}
lithops_payload = json.loads(lithops_payload_json) setup_lithops_logger(lithops_payload.get('log_level', logging.INFO)) logger.info( "Lithops v{} - Starting AWS Batch execution".format(__version__)) job_index = int(os.environ.get('AWS_BATCH_JOB_ARRAY_INDEX', 0)) lithops_payload['JOB_INDEX'] = job_index logger.info('Job index {}'.format(job_index)) act_id = str(uuid.uuid4()).replace('-', '')[:12] os.environ['__LITHOPS_ACTIVATION_ID'] = act_id chunksize = lithops_payload['chunksize'] call_ids_ranges = [ call_ids_range for call_ids_range in iterchunks( lithops_payload['call_ids'], chunksize) ] call_ids = call_ids_ranges[job_index] data_byte_ranges = [ lithops_payload['data_byte_ranges'][int(call_id)] for call_id in call_ids ] lithops_payload['call_ids'] = call_ids lithops_payload['data_byte_ranges'] = data_byte_ranges function_handler(lithops_payload) else: raise Exception('Unknown action {}'.format(action))