def run_job(encoded_payload):
logger.info(
"Lithops v{} - Starting kubernetes execution".format(__version__))
payload = b64str_to_dict(encoded_payload)
setup_lithops_logger(payload['log_level'])
job_key = payload['job_key']
idgiver_ip = os.environ['IDGIVER_POD_IP']
res = requests.get('http://{}:{}/getid/{}'.format(idgiver_ip, IDGIVER_PORT,
job_key))
job_index = int(res.text)
act_id = str(uuid.uuid4()).replace('-', '')[:12]
os.environ['__LITHOPS_ACTIVATION_ID'] = act_id
os.environ['__LITHOPS_BACKEND'] = 'k8s'
logger.info("Activation ID: {} - Job Index: {}".format(act_id, job_index))
chunksize = payload['chunksize']
call_ids_ranges = [
call_ids_range
for call_ids_range in iterchunks(payload['call_ids'], chunksize)
]
call_ids = call_ids_ranges[job_index]
data_byte_ranges = [
payload['data_byte_ranges'][int(call_id)] for call_id in call_ids
]
payload['call_ids'] = call_ids
payload['data_byte_ranges'] = data_byte_ranges
function_handler(payload)
def main_job(action, encoded_payload):
logger.info(
"Lithops v{} - Starting Code Engine execution".format(__version__))
payload = b64str_to_dict(encoded_payload)
setup_lithops_logger(payload['log_level'])
if (action == 'preinstalls'):
runtime_packages(payload)
return {"Execution": "Finished"}
job_index = int(os.environ['JOB_INDEX'])
payload['JOB_INDEX'] = job_index
logger.info("Action {}. Job Index {}".format(action, job_index))
act_id = str(uuid.uuid4()).replace('-', '')[:12]
os.environ['__LITHOPS_ACTIVATION_ID'] = act_id
chunksize = payload['chunksize']
call_ids_ranges = [
call_ids_range
for call_ids_range in iterchunks(payload['call_ids'], chunksize)
]
call_ids = call_ids_ranges[job_index]
data_byte_ranges = [
payload['data_byte_ranges'][int(call_id)] for call_id in call_ids
]
payload['call_ids'] = call_ids
payload['data_byte_ranges'] = data_byte_ranges
function_handler(payload)
return {"Execution": "Finished"}
def run():
def error():
response = flask.jsonify({
'error':
'The action did not receive a dictionary as an argument.'
})
response.status_code = 404
return complete(response)
message = flask.request.get_json(force=True, silent=True)
if message and not isinstance(message, dict):
return error()
act_id = str(uuid.uuid4()).replace('-', '')[:12]
os.environ['__LITHOPS_ACTIVATION_ID'] = act_id
setup_logger(message['log_level'])
if 'remote_invoker' in message:
logger.info(
"Lithops v{} - Starting Knative invoker".format(__version__))
function_invoker(message)
else:
logger.info(
"Lithops v{} - Starting Knative execution".format(__version__))
function_handler(message)
response = flask.jsonify({"activationId": act_id})
response.status_code = 202
return complete(response)
def _process_runner(self, worker_id):
logger.debug('Localhost worker process {} started'.format(worker_id))
os.environ['__LITHOPS_LOCAL_EXECUTION'] = 'True'
p_logger = logging.getLogger('lithops')
while True:
with io.StringIO() as buf, redirect_stdout(buf), redirect_stderr(buf):
event = self.queue.get(block=True)
if isinstance(event, ShutdownSentinel):
break
act_id = str(uuid.uuid4()).replace('-', '')[:12]
os.environ['__LITHOPS_ACTIVATION_ID'] = act_id
executor_id = event['executor_id']
job_id = event['job_id']
setup_logger(event['log_level'])
p_logger.info("Lithops v{} - Starting execution".format(__version__))
function_handler(event)
log_output = buf.getvalue()
job_key = create_job_key(executor_id, job_id)
log_file = os.path.join(LOGS_DIR, job_key+'.log')
header = "Activation: '{}' ({})\n[\n".format(event['runtime_name'], act_id)
tail = ']\n\n'
output = log_output.replace('\n', '\n ', log_output.count('\n')-1)
with open(log_file, 'a') as lf:
lf.write(header+' '+output+tail)
with open(FN_LOG_FILE, 'a') as lf:
lf.write(header+' '+output+tail)
def main_job(action, encoded_payload):
logger.info(
"Lithops v{} - Starting Code Engine execution".format(__version__))
payload = b64str_to_dict(encoded_payload)
setup_logger(payload['log_level'])
if (action == 'preinstalls'):
runtime_packages(payload)
return {"Execution": "Finished"}
job_index = os.environ['JOB_INDEX']
payload['JOB_INDEX'] = job_index
logger.info("Action {}. Job Index {}".format(action, job_index))
act_id = str(uuid.uuid4()).replace('-', '')[:12]
os.environ['__LITHOPS_ACTIVATION_ID'] = act_id
payload['data_byte_range'] = payload['data_byte_range'][int(job_index)]
payload['call_id'] = "{:05d}".format(int(job_index))
function_handler(payload)
return {"Execution": "Finished"}
def main(event, context):
# pub/sub event data is b64 encoded
args = json.loads(base64.b64decode(event['data']).decode('utf-8'))
setup_lithops_logger(args.get('log_level', 'INFO'))
os.environ['__LITHOPS_ACTIVATION_ID'] = uuid.uuid4().hex
os.environ['__LITHOPS_BACKEND'] = 'Google Cloud Functions'
if 'get_preinstalls' in args:
logger.info("Lithops v{} - Generating metadata".format(__version__))
internal_storage = InternalStorage(
args['get_preinstalls']['storage_config'])
object_key = '/'.join(
[JOBS_PREFIX, args['get_preinstalls']['runtime_name'] + '.meta'])
logger.info("Runtime metadata key {}".format(object_key))
runtime_meta = get_runtime_preinstalls()
runtime_meta_json = json.dumps(runtime_meta)
internal_storage.put_data(object_key, runtime_meta_json)
elif 'remote_invoker' in args:
logger.info(
"Lithops v{} - Starting Google Cloud Functions invoker".format(
__version__))
function_invoker(args)
else:
logger.info(
"Lithops v{} - Starting Google Cloud Functions execution".format(
__version__))
function_handler(args)
return {"Execution": "Finished"}
def run():
sys.stdout = log_file_stream
sys.stderr = log_file_stream
job_filename = sys.argv[2]
logger.info('Got {} job file'.format(job_filename))
with open(job_filename, 'rb') as jf:
job_payload = json.load(jf)
executor_id = job_payload['executor_id']
job_id = job_payload['job_id']
job_key = job_payload['job_key']
logger.info('ExecutorID {} | JobID {} - Starting execution'.format(
executor_id, job_id))
act_id = str(uuid.uuid4()).replace('-', '')[:12]
os.environ['__LITHOPS_ACTIVATION_ID'] = act_id
os.environ['__LITHOPS_BACKEND'] = 'Localhost'
try:
function_handler(job_payload)
except KeyboardInterrupt:
pass
done = os.path.join(JOBS_DIR, job_key + '.done')
Path(done).touch()
if os.path.exists(job_filename):
os.remove(job_filename)
logger.info('ExecutorID {} | JobID {} - Execution Finished'.format(
executor_id, job_id))
def _process_runner(self, worker_id):
logger.debug('Localhost worker process {} started'.format(worker_id))
while True:
with io.StringIO() as buf, redirect_stdout(buf), redirect_stderr(
buf):
try:
act_id = str(uuid.uuid4()).replace('-', '')[:12]
os.environ['__LITHOPS_ACTIVATION_ID'] = act_id
event = self.queue.get(block=True)
if isinstance(event, ShutdownSentinel):
break
log_level = event['log_level']
default_logging_config(log_level)
logger.info(
"Lithops v{} - Starting execution".format(__version__))
event['extra_env']['__LITHOPS_LOCAL_EXECUTION'] = 'True'
function_handler(event)
except KeyboardInterrupt:
break
header = "Activation: '{}' ({})\n[\n".format(
event['runtime_name'], act_id)
tail = ']\n\n'
output = buf.getvalue()
output = output.replace('\n', '\n ', output.count('\n') - 1)
with open(FN_LOG_FILE, 'a') as lf:
lf.write(header + ' ' + output + tail)
def main(action, payload_decoded):
logger.info(
"Welcome to Lithops-Code-Engine entry point. Action {}".format(action))
payload = binary_to_dict(payload_decoded)
log_level = payload['log_level']
default_logging_config(log_level)
logger.info(payload)
if (action == 'preinstals'):
runtime_packages(payload)
return {"Execution": "Finished"}
job_index = os.environ['JOB_INDEX']
logger.info("Action {}. Job Index {}".format(action, job_index))
os.environ['__PW_ACTIVATION_ID'] = payload['activation_id']
payload['JOB_INDEX'] = job_index
if 'remote_invoker' in payload:
logger.info("Lithops v{} - Remote Invoker. Starting execution".format(
__version__))
#function_invoker(payload)
payload['data_byte_range'] = payload['job_description']['data_ranges'][
int(job_index)]
for key in payload['job_description']:
payload[key] = payload['job_description'][key]
payload['host_submit_tstamp'] = payload['metadata'][
'host_job_create_tstamp']
payload['call_id'] = "{:05d}".format(int(job_index))
function_handler(payload)
else:
logger.info("Lithops v{} - Starting execution".format(__version__))
function_handler(payload)
return {"Execution": "Finished"}
def main(args):
os.environ['__LITHOPS_ACTIVATION_ID'] = os.environ['__OW_ACTIVATION_ID']
if 'remote_invoker' in args:
logger.info("Lithops v{} - Starting invoker".format(__version__))
function_invoker(args)
else:
logger.info("Lithops v{} - Starting execution".format(__version__))
function_handler(args)
return {"Execution": "Finished"}
def main(event, context):
args = json.loads(event)
os.environ['__PW_ACTIVATION_ID'] = context.request_id
if 'remote_invoker' in args:
logger.info("Lithops v{} - Starting invoker".format(__version__))
function_invoker(args)
else:
logger.info("Lithops v{} - Starting execution".format(__version__))
function_handler(args)
return {"Execution": "Finished"}
def _thread_runner(self, worker_id):
logger.debug('Localhost worker process {} started'.format(worker_id))
os.environ['__LITHOPS_LOCAL_EXECUTION'] = 'True'
while True:
event = self.queue.get(block=True)
if isinstance(event, ShutdownSentinel):
break
act_id = str(uuid.uuid4()).replace('-', '')[:12]
os.environ['__LITHOPS_ACTIVATION_ID'] = act_id
logger.info("Lithops v{} - Starting execution".format(__version__))
function_handler(event)
def main(event, context):
logger.info("Starting GCP Functions function execution")
args = json.loads(base64.b64decode(event['data']).decode('utf-8'))
os.environ['__PW_ACTIVATION_ID'] = uuid.uuid4().hex
if 'remote_invoker' in args:
logger.info("Lithops v{} - Starting invoker".format(__version__))
function_invoker(args)
else:
logger.info("Lithops v{} - Starting execution".format(__version__))
function_handler(args)
return {"Execution": "Finished"}
def lambda_handler(event, context):
os.environ['__PW_ACTIVATION_ID'] = context.aws_request_id
os.environ['__OW_ACTIVATION_ID'] = context.aws_request_id
if 'remote_invoker' in event:
logger.info("Lithops v{} - Starting invoker".format(__version__))
function_invoker(event)
else:
logger.info("Lithops v{} - Starting execution".format(__version__))
function_handler(event)
return {"Execution": "Finished"}
def _process_runner(self, worker_id):
logger.debug('Localhost worker process {} started'.format(worker_id))
while True:
event = self.queue.get(block=True)
if isinstance(event, ShutdownSentinel):
break
act_id = str(uuid.uuid4()).replace('-', '')[:12]
os.environ['__LITHOPS_ACTIVATION_ID'] = act_id
event['extra_env']['__LITHOPS_LOCAL_EXECUTION'] = 'True'
function_handler(event)
def main(args):
os.environ['__LITHOPS_ACTIVATION_ID'] = os.environ['__OW_ACTIVATION_ID']
setup_logger(args['log_level'], sys.stdout, LOGGER_FORMAT_SHORT)
if 'remote_invoker' in args:
logger.info("Lithops v{} - Starting invoker".format(__version__))
function_invoker(args)
else:
logger.info("Lithops v{} - Starting execution".format(__version__))
function_handler(args)
return {"Execution": "Finished"}
def _local_handler(self, event):
"""
Handler to run local functions.
"""
if not self.log_active:
old_stdout = sys.stdout
sys.stdout = open(os.devnull, 'w')
event['extra_env']['__PW_LOCAL_EXECUTION'] = 'True'
act_id = str(uuid.uuid4()).replace('-', '')[:12]
os.environ['__PW_ACTIVATION_ID'] = act_id
function_handler(event)
if not self.log_active:
sys.stdout = old_stdout
def main(msgIn: func.QueueMessage):
try:
args = json.loads(msgIn.get_body())
except:
args = msgIn.get_json()
os.environ['__PW_ACTIVATION_ID'] = str(msgIn.id)
if 'remote_invoker' in args:
logger.info("Lithops v{} - Starting invoker".format(__version__))
function_invoker(args)
else:
logger.info("Lithops v{} - Starting execution".format(__version__))
function_handler(args)
return {"Execution": "Finished"}
def main(args):
os.environ['__LITHOPS_ACTIVATION_ID'] = os.environ['__OW_ACTIVATION_ID']
setup_logger(args['log_level'], sys.stdout, LOGGER_FORMAT_SHORT)
if 'get_preinstalls' in args:
logger.info("Lithops v{} - Generating metadata".format(__version__))
return get_runtime_preinstalls()
elif 'remote_invoker' in args:
logger.info("Lithops v{} - Starting IBM CF invoker".format(__version__))
function_invoker(args)
else:
logger.info("Lithops v{} - Starting IBM CF execution".format(__version__))
function_handler(args)
return {"Execution": "Finished"}
def lambda_handler(event, context):
os.environ['__LITHOPS_ACTIVATION_ID'] = context.aws_request_id
setup_lithops_logger(event.get('log_level', logging.INFO))
if 'get_preinstalls' in event:
logger.info("Lithops v{} - Generating metadata".format(__version__))
return get_runtime_preinstalls()
elif 'remote_invoker' in event:
logger.info("Lithops v{} - Starting invoker".format(__version__))
function_invoker(event)
else:
logger.info("Lithops v{} - Starting execution".format(__version__))
function_handler(event)
return {"Execution": "Finished"}
def run_job(encoded_payload):
logger.info(
"Lithops v{} - Starting kubernetes execution".format(__version__))
payload = b64str_to_dict(encoded_payload)
setup_lithops_logger(payload['log_level'])
total_calls = payload['total_calls']
job_key = payload['job_key']
master_ip = os.environ['MASTER_POD_IP']
chunksize = payload['chunksize']
call_ids_ranges = [
call_ids_range
for call_ids_range in iterchunks(payload['call_ids'], chunksize)
]
data_byte_ranges = payload['data_byte_ranges']
job_finished = False
while not job_finished:
job_index = None
while job_index is None:
try:
url = f'http://{master_ip}:{MASTER_PORT}/getid/{job_key}/{total_calls}'
res = requests.get(url)
job_index = int(res.text)
except Exception:
time.sleep(0.1)
if job_index == -1:
job_finished = True
continue
act_id = str(uuid.uuid4()).replace('-', '')[:12]
os.environ['__LITHOPS_ACTIVATION_ID'] = act_id
os.environ['__LITHOPS_BACKEND'] = 'k8s'
logger.info("Activation ID: {} - Job Index: {}".format(
act_id, job_index))
call_ids = call_ids_ranges[job_index]
dbr = [data_byte_ranges[int(call_id)] for call_id in call_ids]
payload['call_ids'] = call_ids
payload['data_byte_ranges'] = dbr
function_handler(payload)
def main(args):
os.environ['__LITHOPS_ACTIVATION_ID'] = os.environ['__OW_ACTIVATION_ID']
log_level = args['log_level']
ow_logging_config(log_level)
if 'remote_invoker' in args:
logger.info(
"Lithops v{} - Starting OpenWhisk Functions invoker".format(
__version__))
function_invoker(args)
else:
logger.info(
"Lithops v{} - Starting OpenWhisk Functions execution".format(
__version__))
function_handler(args)
return {"Execution": "Finished"}
def main(msgIn: func.QueueMessage, msgOut: func.Out[func.QueueMessage]):
try:
args = json.loads(msgIn.get_body())
except Exception:
args = msgIn.get_json()
os.environ['__LITHOPS_ACTIVATION_ID'] = str(msgIn.id)
setup_lithops_logger(args['log_level'])
if 'get_preinstalls' in args:
logger.info("Lithops v{} - Generating metadata".format(__version__))
runtime_meta = get_runtime_preinstalls()
msgOut.set(json.dumps(runtime_meta))
elif 'remote_invoker' in args:
logger.info("Lithops v{} - Starting invoker".format(__version__))
function_invoker(args)
else:
logger.info("Lithops v{} - Starting execution".format(__version__))
function_handler(args)
def main_http(req: func.HttpRequest, context: func.Context) -> str:
payload = req.get_json()
setup_lithops_logger(payload['log_level'])
os.environ['__LITHOPS_ACTIVATION_ID'] = context.invocation_id
os.environ['__LITHOPS_BACKEND'] = 'Azure Functions (http)'
if 'get_preinstalls' in payload:
logger.info("Lithops v{} - Generating metadata".format(__version__))
runtime_meta = get_runtime_preinstalls()
return json.dumps(runtime_meta)
elif 'remote_invoker' in payload:
logger.info("Lithops v{} - Starting Azure Functions (http) invoker".format(__version__))
function_invoker(payload)
else:
logger.info("Lithops v{} - Starting Azure Functions (http) execution".format(__version__))
function_handler(payload)
return context.invocation_id
def main_queue(msgIn: func.QueueMessage, msgOut: func.Out[func.QueueMessage]):
try:
payload = json.loads(msgIn.get_body())
except Exception:
payload = msgIn.get_json()
setup_lithops_logger(payload['log_level'])
os.environ['__LITHOPS_ACTIVATION_ID'] = str(msgIn.id)
os.environ['__LITHOPS_BACKEND'] = 'Azure Functions (event)'
if 'get_preinstalls' in payload:
logger.info("Lithops v{} - Generating metadata".format(__version__))
runtime_meta = get_runtime_preinstalls()
msgOut.set(json.dumps(runtime_meta))
elif 'remote_invoker' in payload:
logger.info("Lithops v{} - Starting Azure Functions (event) invoker".format(__version__))
function_invoker(payload)
else:
logger.info("Lithops v{} - Starting Azure Functions (event) execution".format(__version__))
function_handler(payload)
def main(event, context):
args = json.loads(event)
os.environ['__LITHOPS_ACTIVATION_ID'] = context.request_id
os.environ['__LITHOPS_BACKEND'] = 'Alibaba Function Compute'
setup_lithops_logger(args['log_level'])
if 'get_preinstalls' in event:
logger.info("Lithops v{} - Generating metadata".format(__version__))
return get_runtime_preinstalls()
elif 'remote_invoker' in args:
logger.info(
"Lithops v{} - Starting Alibaba Function Compute invoker".format(
__version__))
function_invoker(args)
else:
logger.info(
"Lithops v{} - Starting Alibaba Function Compute execution".format(
__version__))
function_handler(args)
return {"Execution": "Finished"}
lithops_payload = json.loads(lithops_payload_json)
setup_lithops_logger(lithops_payload.get('log_level', logging.INFO))
logger.info(
"Lithops v{} - Starting AWS Batch execution".format(__version__))
job_index = int(os.environ.get('AWS_BATCH_JOB_ARRAY_INDEX', 0))
lithops_payload['JOB_INDEX'] = job_index
logger.info('Job index {}'.format(job_index))
act_id = str(uuid.uuid4()).replace('-', '')[:12]
os.environ['__LITHOPS_ACTIVATION_ID'] = act_id
chunksize = lithops_payload['chunksize']
call_ids_ranges = [
call_ids_range for call_ids_range in iterchunks(
lithops_payload['call_ids'], chunksize)
]
call_ids = call_ids_ranges[job_index]
data_byte_ranges = [
lithops_payload['data_byte_ranges'][int(call_id)]
for call_id in call_ids
]
lithops_payload['call_ids'] = call_ids
lithops_payload['data_byte_ranges'] = data_byte_ranges
function_handler(lithops_payload)
else:
raise Exception('Unknown action {}'.format(action))