def invoke(self,
docker_image_name,
memory=None,
payload={},
return_result=False):
"""
Invoke function
"""
action_name = self._format_action_name(docker_image_name)
in_q_name = self._format_queue_name(action_name, az_config.IN_QUEUE)
in_queue = self.queue_service.get_queue_client(in_q_name)
msg = in_queue.send_message(dict_to_b64str(payload))
activation_id = msg.id
if return_result:
out_q_name = self._format_queue_name(action_name,
az_config.OUT_QUEUE)
out_queue = self.queue_service.get_queue_client(out_q_name)
msg = []
while not msg:
time.sleep(1)
msg = out_queue.receive_message()
out_queue.clear_messages()
return b64str_to_dict(msg.content)
return activation_id
def main_job(action, encoded_payload):
logger.info(
"Lithops v{} - Starting Code Engine execution".format(__version__))
payload = b64str_to_dict(encoded_payload)
setup_logger(payload['log_level'])
if (action == 'preinstalls'):
runtime_packages(payload)
return {"Execution": "Finished"}
job_index = os.environ['JOB_INDEX']
payload['JOB_INDEX'] = job_index
logger.info("Action {}. Job Index {}".format(action, job_index))
act_id = str(uuid.uuid4()).replace('-', '')[:12]
os.environ['__LITHOPS_ACTIVATION_ID'] = act_id
payload['data_byte_range'] = payload['data_byte_range'][int(job_index)]
payload['call_id'] = "{:05d}".format(int(job_index))
function_handler(payload)
return {"Execution": "Finished"}
def main_job(action, encoded_payload):
logger.info(
"Lithops v{} - Starting Code Engine execution".format(__version__))
payload = b64str_to_dict(encoded_payload)
setup_lithops_logger(payload['log_level'])
if (action == 'preinstalls'):
runtime_packages(payload)
return {"Execution": "Finished"}
job_index = int(os.environ['JOB_INDEX'])
payload['JOB_INDEX'] = job_index
logger.info("Action {}. Job Index {}".format(action, job_index))
act_id = str(uuid.uuid4()).replace('-', '')[:12]
os.environ['__LITHOPS_ACTIVATION_ID'] = act_id
chunksize = payload['chunksize']
call_ids_ranges = [
call_ids_range
for call_ids_range in iterchunks(payload['call_ids'], chunksize)
]
call_ids = call_ids_ranges[job_index]
data_byte_ranges = [
payload['data_byte_ranges'][int(call_id)] for call_id in call_ids
]
payload['call_ids'] = call_ids
payload['data_byte_ranges'] = data_byte_ranges
function_handler(payload)
return {"Execution": "Finished"}
def run_job(encoded_payload):
logger.info(
"Lithops v{} - Starting kubernetes execution".format(__version__))
payload = b64str_to_dict(encoded_payload)
setup_lithops_logger(payload['log_level'])
job_key = payload['job_key']
idgiver_ip = os.environ['IDGIVER_POD_IP']
res = requests.get('http://{}:{}/getid/{}'.format(idgiver_ip, IDGIVER_PORT,
job_key))
job_index = int(res.text)
act_id = str(uuid.uuid4()).replace('-', '')[:12]
os.environ['__LITHOPS_ACTIVATION_ID'] = act_id
os.environ['__LITHOPS_BACKEND'] = 'k8s'
logger.info("Activation ID: {} - Job Index: {}".format(act_id, job_index))
chunksize = payload['chunksize']
call_ids_ranges = [
call_ids_range
for call_ids_range in iterchunks(payload['call_ids'], chunksize)
]
call_ids = call_ids_ranges[job_index]
data_byte_ranges = [
payload['data_byte_ranges'][int(call_id)] for call_id in call_ids
]
payload['call_ids'] = call_ids
payload['data_byte_ranges'] = data_byte_ranges
function_handler(payload)
def invoke(self,
docker_image_name,
memory=None,
payload={},
return_result=False):
"""
Invoke function
"""
action_name = self._format_action_name(docker_image_name, memory)
if self.invocation_type == 'event':
in_q_name = self._format_queue_name(action_name,
az_config.IN_QUEUE)
in_queue = self.queue_service.get_queue_client(in_q_name)
msg = in_queue.send_message(dict_to_b64str(payload))
activation_id = msg.id
if return_result:
out_q_name = self._format_queue_name(action_name,
az_config.OUT_QUEUE)
out_queue = self.queue_service.get_queue_client(out_q_name)
msg = []
while not msg:
time.sleep(1)
msg = out_queue.receive_message()
out_queue.clear_messages()
return b64str_to_dict(msg.content)
elif self.invocation_type == 'http':
endpoint = "https://{}.azurewebsites.net".format(action_name)
parsed_url = urlparse(endpoint)
ctx = ssl._create_unverified_context()
conn = http.client.HTTPSConnection(parsed_url.netloc, context=ctx)
route = "/api/lithops_handler"
if return_result:
conn.request("GET",
route,
body=json.dumps(payload, default=str))
resp = conn.getresponse()
data = json.loads(resp.read().decode("utf-8"))
conn.close()
return data
else:
# logger.debug('Invoking calls {}'.format(', '.join(payload['call_ids'])))
conn.request("POST",
route,
body=json.dumps(payload, default=str))
resp = conn.getresponse()
if resp.status == 429:
time.sleep(0.2)
conn.close()
return None
activation_id = resp.read().decode("utf-8")
conn.close()
return activation_id
def extract_runtime_meta(encoded_payload):
logger.info("Lithops v{} - Generating metadata".format(__version__))
payload = b64str_to_dict(encoded_payload)
setup_lithops_logger(payload['log_level'])
runtime_meta = get_runtime_preinstalls()
internal_storage = InternalStorage(payload)
status_key = '/'.join([JOBS_PREFIX, payload['runtime_name'] + '.meta'])
logger.info("Runtime metadata key {}".format(status_key))
dmpd_response_status = json.dumps(runtime_meta)
internal_storage.put_data(status_key, dmpd_response_status)
def run_job(encoded_payload):
logger.info(
"Lithops v{} - Starting kubernetes execution".format(__version__))
payload = b64str_to_dict(encoded_payload)
setup_lithops_logger(payload['log_level'])
total_calls = payload['total_calls']
job_key = payload['job_key']
master_ip = os.environ['MASTER_POD_IP']
chunksize = payload['chunksize']
call_ids_ranges = [
call_ids_range
for call_ids_range in iterchunks(payload['call_ids'], chunksize)
]
data_byte_ranges = payload['data_byte_ranges']
job_finished = False
while not job_finished:
job_index = None
while job_index is None:
try:
url = f'http://{master_ip}:{MASTER_PORT}/getid/{job_key}/{total_calls}'
res = requests.get(url)
job_index = int(res.text)
except Exception:
time.sleep(0.1)
if job_index == -1:
job_finished = True
continue
act_id = str(uuid.uuid4()).replace('-', '')[:12]
os.environ['__LITHOPS_ACTIVATION_ID'] = act_id
os.environ['__LITHOPS_BACKEND'] = 'k8s'
logger.info("Activation ID: {} - Job Index: {}".format(
act_id, job_index))
call_ids = call_ids_ranges[job_index]
dbr = [data_byte_ranges[int(call_id)] for call_id in call_ids]
payload['call_ids'] = call_ids
payload['data_byte_ranges'] = dbr
function_handler(payload)
