def invoke(self, docker_image_name, memory=None, payload={}, return_result=False): """ Invoke function """ action_name = self._format_action_name(docker_image_name) in_q_name = self._format_queue_name(action_name, az_config.IN_QUEUE) in_queue = self.queue_service.get_queue_client(in_q_name) msg = in_queue.send_message(dict_to_b64str(payload)) activation_id = msg.id if return_result: out_q_name = self._format_queue_name(action_name, az_config.OUT_QUEUE) out_queue = self.queue_service.get_queue_client(out_q_name) msg = [] while not msg: time.sleep(1) msg = out_queue.receive_message() out_queue.clear_messages() return b64str_to_dict(msg.content) return activation_id
def main_job(action, encoded_payload): logger.info( "Lithops v{} - Starting Code Engine execution".format(__version__)) payload = b64str_to_dict(encoded_payload) setup_logger(payload['log_level']) if (action == 'preinstalls'): runtime_packages(payload) return {"Execution": "Finished"} job_index = os.environ['JOB_INDEX'] payload['JOB_INDEX'] = job_index logger.info("Action {}. Job Index {}".format(action, job_index)) act_id = str(uuid.uuid4()).replace('-', '')[:12] os.environ['__LITHOPS_ACTIVATION_ID'] = act_id payload['data_byte_range'] = payload['data_byte_range'][int(job_index)] payload['call_id'] = "{:05d}".format(int(job_index)) function_handler(payload) return {"Execution": "Finished"}
def main_job(action, encoded_payload): logger.info( "Lithops v{} - Starting Code Engine execution".format(__version__)) payload = b64str_to_dict(encoded_payload) setup_lithops_logger(payload['log_level']) if (action == 'preinstalls'): runtime_packages(payload) return {"Execution": "Finished"} job_index = int(os.environ['JOB_INDEX']) payload['JOB_INDEX'] = job_index logger.info("Action {}. Job Index {}".format(action, job_index)) act_id = str(uuid.uuid4()).replace('-', '')[:12] os.environ['__LITHOPS_ACTIVATION_ID'] = act_id chunksize = payload['chunksize'] call_ids_ranges = [ call_ids_range for call_ids_range in iterchunks(payload['call_ids'], chunksize) ] call_ids = call_ids_ranges[job_index] data_byte_ranges = [ payload['data_byte_ranges'][int(call_id)] for call_id in call_ids ] payload['call_ids'] = call_ids payload['data_byte_ranges'] = data_byte_ranges function_handler(payload) return {"Execution": "Finished"}
def run_job(encoded_payload): logger.info( "Lithops v{} - Starting kubernetes execution".format(__version__)) payload = b64str_to_dict(encoded_payload) setup_lithops_logger(payload['log_level']) job_key = payload['job_key'] idgiver_ip = os.environ['IDGIVER_POD_IP'] res = requests.get('http://{}:{}/getid/{}'.format(idgiver_ip, IDGIVER_PORT, job_key)) job_index = int(res.text) act_id = str(uuid.uuid4()).replace('-', '')[:12] os.environ['__LITHOPS_ACTIVATION_ID'] = act_id os.environ['__LITHOPS_BACKEND'] = 'k8s' logger.info("Activation ID: {} - Job Index: {}".format(act_id, job_index)) chunksize = payload['chunksize'] call_ids_ranges = [ call_ids_range for call_ids_range in iterchunks(payload['call_ids'], chunksize) ] call_ids = call_ids_ranges[job_index] data_byte_ranges = [ payload['data_byte_ranges'][int(call_id)] for call_id in call_ids ] payload['call_ids'] = call_ids payload['data_byte_ranges'] = data_byte_ranges function_handler(payload)
def invoke(self, docker_image_name, memory=None, payload={}, return_result=False): """ Invoke function """ action_name = self._format_action_name(docker_image_name, memory) if self.invocation_type == 'event': in_q_name = self._format_queue_name(action_name, az_config.IN_QUEUE) in_queue = self.queue_service.get_queue_client(in_q_name) msg = in_queue.send_message(dict_to_b64str(payload)) activation_id = msg.id if return_result: out_q_name = self._format_queue_name(action_name, az_config.OUT_QUEUE) out_queue = self.queue_service.get_queue_client(out_q_name) msg = [] while not msg: time.sleep(1) msg = out_queue.receive_message() out_queue.clear_messages() return b64str_to_dict(msg.content) elif self.invocation_type == 'http': endpoint = "https://{}.azurewebsites.net".format(action_name) parsed_url = urlparse(endpoint) ctx = ssl._create_unverified_context() conn = http.client.HTTPSConnection(parsed_url.netloc, context=ctx) route = "/api/lithops_handler" if return_result: conn.request("GET", route, body=json.dumps(payload, default=str)) resp = conn.getresponse() data = json.loads(resp.read().decode("utf-8")) conn.close() return data else: # logger.debug('Invoking calls {}'.format(', '.join(payload['call_ids']))) conn.request("POST", route, body=json.dumps(payload, default=str)) resp = conn.getresponse() if resp.status == 429: time.sleep(0.2) conn.close() return None activation_id = resp.read().decode("utf-8") conn.close() return activation_id
def extract_runtime_meta(encoded_payload): logger.info("Lithops v{} - Generating metadata".format(__version__)) payload = b64str_to_dict(encoded_payload) setup_lithops_logger(payload['log_level']) runtime_meta = get_runtime_preinstalls() internal_storage = InternalStorage(payload) status_key = '/'.join([JOBS_PREFIX, payload['runtime_name'] + '.meta']) logger.info("Runtime metadata key {}".format(status_key)) dmpd_response_status = json.dumps(runtime_meta) internal_storage.put_data(status_key, dmpd_response_status)
def run_job(encoded_payload): logger.info( "Lithops v{} - Starting kubernetes execution".format(__version__)) payload = b64str_to_dict(encoded_payload) setup_lithops_logger(payload['log_level']) total_calls = payload['total_calls'] job_key = payload['job_key'] master_ip = os.environ['MASTER_POD_IP'] chunksize = payload['chunksize'] call_ids_ranges = [ call_ids_range for call_ids_range in iterchunks(payload['call_ids'], chunksize) ] data_byte_ranges = payload['data_byte_ranges'] job_finished = False while not job_finished: job_index = None while job_index is None: try: url = f'http://{master_ip}:{MASTER_PORT}/getid/{job_key}/{total_calls}' res = requests.get(url) job_index = int(res.text) except Exception: time.sleep(0.1) if job_index == -1: job_finished = True continue act_id = str(uuid.uuid4()).replace('-', '')[:12] os.environ['__LITHOPS_ACTIVATION_ID'] = act_id os.environ['__LITHOPS_BACKEND'] = 'k8s' logger.info("Activation ID: {} - Job Index: {}".format( act_id, job_index)) call_ids = call_ids_ranges[job_index] dbr = [data_byte_ranges[int(call_id)] for call_id in call_ids] payload['call_ids'] = call_ids payload['data_byte_ranges'] = dbr function_handler(payload)