def main(action, payload_decoded): logger.info( "Welcome to Lithops-Code-Engine entry point. Action {}".format(action)) payload = binary_to_dict(payload_decoded) log_level = payload['log_level'] default_logging_config(log_level) logger.info(payload) if (action == 'preinstals'): runtime_packages(payload) return {"Execution": "Finished"} job_index = os.environ['JOB_INDEX'] logger.info("Action {}. Job Index {}".format(action, job_index)) os.environ['__PW_ACTIVATION_ID'] = payload['activation_id'] payload['JOB_INDEX'] = job_index if 'remote_invoker' in payload: logger.info("Lithops v{} - Remote Invoker. Starting execution".format( __version__)) #function_invoker(payload) payload['data_byte_range'] = payload['job_description']['data_ranges'][ int(job_index)] for key in payload['job_description']: payload[key] = payload['job_description'][key] payload['host_submit_tstamp'] = payload['metadata'][ 'host_job_create_tstamp'] payload['call_id'] = "{:05d}".format(int(job_index)) function_handler(payload) else: logger.info("Lithops v{} - Starting execution".format(__version__)) function_handler(payload) return {"Execution": "Finished"}
def _process_runner(self, worker_id): logger.debug('Localhost worker process {} started'.format(worker_id)) while True: with io.StringIO() as buf, redirect_stdout(buf), redirect_stderr( buf): try: act_id = str(uuid.uuid4()).replace('-', '')[:12] os.environ['__LITHOPS_ACTIVATION_ID'] = act_id event = self.queue.get(block=True) if isinstance(event, ShutdownSentinel): break log_level = event['log_level'] default_logging_config(log_level) logger.info( "Lithops v{} - Starting execution".format(__version__)) event['extra_env']['__LITHOPS_LOCAL_EXECUTION'] = 'True' function_handler(event) except KeyboardInterrupt: break header = "Activation: '{}' ({})\n[\n".format( event['runtime_name'], act_id) tail = ']\n\n' output = buf.getvalue() output = output.replace('\n', '\n ', output.count('\n') - 1) with open(FN_LOG_FILE, 'a') as lf: lf.write(header + ' ' + output + tail)
def run(): def error(): response = flask.jsonify({ 'error': 'The action did not receive a dictionary as an argument.' }) response.status_code = 404 return complete(response) message = flask.request.get_json(force=True, silent=True) if message and not isinstance(message, dict): return error() act_id = str(uuid.uuid4()).replace('-', '')[:12] os.environ['__LITHOPS_ACTIVATION_ID'] = act_id log_level = message['log_level'] default_logging_config(log_level) if 'remote_invoker' in message: logger.info( "Lithops v{} - Starting Knative invoker".format(__version__)) function_invoker(message) else: logger.info( "Lithops v{} - Starting Knative execution".format(__version__)) function_handler(message) response = flask.jsonify({"activationId": act_id}) response.status_code = 202 return complete(response)
def lambda_handler(event, context): os.environ['__PW_ACTIVATION_ID'] = context.aws_request_id os.environ['__OW_ACTIVATION_ID'] = context.aws_request_id log_level = event['log_level'] default_logging_config(log_level) if 'remote_invoker' in event: logger.info("Lithops v{} - Starting invoker".format(__version__)) function_invoker(event) else: logger.info("Lithops v{} - Starting execution".format(__version__)) function_handler(event) return {"Execution": "Finished"}
def __init__(self, type=None, config=None, backend=None, storage=None, runtime=None, runtime_memory=None, rabbitmq_monitor=None, workers=None, remote_invoker=None, log_level=None): if type is None: config = default_config(copy.deepcopy(config)) type = config['lithops']['executor'] if log_level: default_logging_config(log_level) config_ow = {'lithops': {'executor': type}, type: {}} if runtime is not None: config_ow[type]['runtime'] = runtime if backend is not None: config_ow[type]['backend'] = backend if runtime_memory is not None: config_ow[type]['runtime_memory'] = int(runtime_memory) if remote_invoker is not None: config_ow[type]['remote_invoker'] = remote_invoker if storage is not None: config_ow['lithops']['storage'] = storage if workers is not None: config_ow['lithops']['workers'] = workers if rabbitmq_monitor is not None: config_ow['lithops']['rabbitmq_monitor'] = rabbitmq_monitor self.config = default_config(copy.deepcopy(config), config_ow) self.log_active = logger.getEffectiveLevel() != logging.WARNING self.is_lithops_worker = is_lithops_worker() self.executor_id = create_executor_id() self.data_cleaner = self.config['lithops'].get('data_cleaner', True) self.rabbitmq_monitor = self.config['lithops'].get( 'rabbitmq_monitor', False) if self.rabbitmq_monitor: if 'rabbitmq' in self.config and 'amqp_url' in self.config[ 'rabbitmq']: self.rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url') else: raise Exception("You cannot use rabbitmq_mnonitor since " "'amqp_url' is not present in configuration") self.storage_config = extract_storage_config(self.config) self.internal_storage = InternalStorage(self.storage_config) self.storage = self.internal_storage.storage self.futures = [] self.total_jobs = 0 self.cleaned_jobs = set() self.last_call = None if type == 'localhost': localhost_config = extract_localhost_config(self.config) self.compute_handler = LocalhostHandler(localhost_config) self.invoker = StandaloneInvoker(self.config, self.executor_id, self.internal_storage, self.compute_handler) elif type == 'serverless': serverless_config = extract_serverless_config(self.config) self.compute_handler = ServerlessHandler(serverless_config, self.storage_config) self.invoker = ServerlessInvoker(self.config, self.executor_id, self.internal_storage, self.compute_handler) elif type == 'standalone': standalone_config = extract_standalone_config(self.config) self.compute_handler = StandaloneHandler(standalone_config) self.invoker = StandaloneInvoker(self.config, self.executor_id, self.internal_storage, self.compute_handler) else: raise Exception("Function executor type must be one of " "'localhost', 'serverless' or 'standalone'") logger.info('{} Executor created with ID: {}'.format( type.capitalize(), self.executor_id))
def __init__(self, config=None, runtime=None, runtime_memory=None, compute_backend=None, compute_backend_region=None, storage_backend=None, storage_backend_region=None, workers=None, rabbitmq_monitor=None, remote_invoker=None, log_level=None): """ Initialize a FunctionExecutor class. :param config: Settings passed in here will override those in config file. Default None. :param runtime: Runtime name to use. Default None. :param runtime_memory: memory to use in the runtime. Default None. :param compute_backend: Name of the compute backend to use. Default None. :param compute_backend_region: Name of the compute backend region to use. Default None. :param storage_backend: Name of the storage backend to use. Default None. :param storage_backend_region: Name of the storage backend region to use. Default None. :param workers: Max number of concurrent workers. :param rabbitmq_monitor: use rabbitmq as the monitoring system. Default None. :param log_level: log level to use during the execution. Default None. :return `FunctionExecutor` object. """ self.is_lithops_function = is_lithops_function() if log_level: default_logging_config(log_level) self.log_active = logger.getEffectiveLevel() != logging.WARNING # Overwrite lithops config parameters pw_config_ow = {} if runtime is not None: pw_config_ow['runtime'] = runtime if runtime_memory is not None: pw_config_ow['runtime_memory'] = int(runtime_memory) if compute_backend is not None: pw_config_ow['compute_backend'] = compute_backend if compute_backend_region is not None: pw_config_ow['compute_backend_region'] = compute_backend_region if storage_backend is not None: pw_config_ow['storage_backend'] = storage_backend if storage_backend_region is not None: pw_config_ow['storage_backend_region'] = storage_backend_region if workers is not None: pw_config_ow['workers'] = workers if rabbitmq_monitor is not None: pw_config_ow['rabbitmq_monitor'] = rabbitmq_monitor if remote_invoker is not None: pw_config_ow['remote_invoker'] = remote_invoker self.config = default_config(copy.deepcopy(config), pw_config_ow) self.executor_id = create_executor_id() logger.debug('FunctionExecutor created with ID: {}'.format(self.executor_id)) self.data_cleaner = self.config['lithops'].get('data_cleaner', True) self.auto_dismantle = self.config['lithops'].get('auto_dismantle', True) self.rabbitmq_monitor = self.config['lithops'].get('rabbitmq_monitor', False) if self.rabbitmq_monitor: if 'rabbitmq' in self.config and 'amqp_url' in self.config['rabbitmq']: self.rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url') else: raise Exception("You cannot use rabbitmq_mnonitor since 'amqp_url'" " is not present in configuration") storage_config = extract_storage_config(self.config) self.internal_storage = InternalStorage(storage_config) self.storage = self.internal_storage.storage self.invoker = FunctionInvoker(self.config, self.executor_id, self.internal_storage) self.futures = [] self.total_jobs = 0 self.cleaned_jobs = set() self.last_call = None