def build_runtime_docker(self, docker_image_name, dockerfile):
"""
Builds a new runtime from a Docker file and pushes it to the Docker hub
"""
logger.debug('Building new docker image from Dockerfile')
logger.debug('Docker image name: {}'.format(docker_image_name))
entry_point = os.path.join(os.path.dirname(__file__), 'entry_point.py')
create_handler_zip(az_config.FH_ZIP_LOCATION, entry_point, '__init__.py')
if dockerfile:
cmd = '{} build -t {} -f {} .'.format(az_config.DOCKER_PATH,
docker_image_name,
dockerfile)
else:
cmd = '{} build -t {} .'.format(az_config.DOCKER_PATH, docker_image_name)
if logger.getEffectiveLevel() != logging.DEBUG:
cmd = cmd + " >{} 2>&1".format(os.devnull)
logger.info('Building default runtime')
res = os.system(cmd)
if res != 0:
raise Exception('There was an error building the runtime')
cmd = '{} push {}'.format(az_config.DOCKER_PATH, docker_image_name)
if logger.getEffectiveLevel() != logging.DEBUG:
cmd = cmd + " >{} 2>&1".format(os.devnull)
res = os.system(cmd)
if res != 0:
raise Exception('There was an error pushing the runtime to the container registry')
logger.debug('Done!')
def create_runtime(self, docker_image_name, memory, timeout):
"""
Creates a new runtime into IBM CF namespace from an already built Docker image
"""
if docker_image_name == 'default':
docker_image_name = self._get_default_runtime_image_name()
logger.info('Creating new Lithops runtime based on Docker image {}'.format(docker_image_name))
self.cf_client.create_package(self.package)
action_name = self._format_action_name(docker_image_name, memory)
entry_point = os.path.join(os.path.dirname(__file__), 'entry_point.py')
create_handler_zip(ibmcf_config.FH_ZIP_LOCATION, entry_point, '__main__.py')
with open(ibmcf_config.FH_ZIP_LOCATION, "rb") as action_zip:
action_bin = action_zip.read()
self.cf_client.create_action(self.package, action_name, docker_image_name, code=action_bin,
memory=memory, is_binary=True, timeout=timeout * 1000)
self._delete_function_handler_zip()
runtime_meta = self._generate_runtime_meta(docker_image_name, memory)
return runtime_meta
def _setup_master_service(self):
"""
Setup lithops necessary packages and files in master VM instance
"""
logger.debug('Installing Lithops in {}'.format(self.backend.master))
ssh_client = self.backend.master.get_ssh_client()
src_proxy = os.path.join(os.path.dirname(__file__), 'worker.py')
create_handler_zip(LOCAL_FH_ZIP_LOCATION, src_proxy)
current_location = os.path.dirname(os.path.abspath(__file__))
controller_location = os.path.join(current_location, 'master.py')
logger.debug('Uploading lithops files to {}'.format(self.backend.master))
files_to_upload = [(LOCAL_FH_ZIP_LOCATION, '/tmp/lithops_standalone.zip'),
(controller_location, '/tmp/master.py'.format(STANDALONE_INSTALL_DIR))]
ssh_client.upload_multiple_local_files(files_to_upload)
os.remove(LOCAL_FH_ZIP_LOCATION)
vm_data = {'instance_name': self.backend.master.name,
'ip_address': self.backend.master.ip_address,
'instance_id': self.backend.master.instance_id}
script = get_master_setup_script(self.config, vm_data)
logger.debug('Executing lithops installation process on {}'.format(self.backend.master))
logger.debug('Be patient, initial installation process may take up to 5 minutes')
ssh_client.run_remote_command(script)
logger.debug('Lithops installation process completed')
def build_runtime(self, docker_image_name, dockerfile):
"""
Builds a new runtime from a Docker file and pushes it to the Docker hub
"""
logger.info('Building a new docker image from Dockerfile')
logger.info('Docker image name: {}'.format(docker_image_name))
# Project ID can contain '-'
expression = '^([-a-z0-9]+)/([-a-z0-9]+)(:[a-z0-9]+)?'
result = re.match(expression, docker_image_name)
if not result or result.group() != docker_image_name:
raise Exception(
"Invalid docker image name: '.' or '_' characters are not allowed"
)
entry_point = os.path.join(os.path.dirname(__file__), 'entry_point.py')
create_handler_zip(cr_config.FH_ZIP_LOCATION, entry_point,
'lithopsproxy.py')
# Dockerfile has to be called "Dockerfile" (and in cwd) for 'gcloud builds submit' to work
if dockerfile != "Dockerfile":
shutil.copyfile(dockerfile, "Dockerfile")
cmd = 'gcloud builds submit -t gcr.io/{}'.format(docker_image_name)
if not self.log_active:
cmd = cmd + " >{} 2>&1".format(os.devnull)
res = os.system(cmd)
if res != 0:
raise Exception('There was an error building the runtime')
self._delete_function_handler_zip()
def deploy_runtime(self, docker_image_name, memory, timeout):
"""
Deploys a new runtime into IBM CF namespace from an already built Docker image
"""
if docker_image_name == 'default':
docker_image_name = self._get_default_runtime_image_name()
logger.debug(
f"Deploying runtime: {docker_image_name} - Memory: {memory} Timeout: {timeout}"
)
self.cf_client.create_package(self.package)
action_name = self._format_function_name(docker_image_name, memory)
entry_point = os.path.join(os.path.dirname(__file__), 'entry_point.py')
create_handler_zip(ow_config.FH_ZIP_LOCATION, entry_point,
'__main__.py')
with open(ow_config.FH_ZIP_LOCATION, "rb") as action_zip:
action_bin = action_zip.read()
self.cf_client.create_action(self.package,
action_name,
docker_image_name,
code=action_bin,
memory=memory,
is_binary=True,
timeout=timeout * 1000)
self._delete_function_handler_zip()
return self._generate_runtime_meta(docker_image_name, memory)
def build_runtime(self, docker_image_name, dockerfile):
"""
Builds a new runtime from a Docker file and pushes it to the Docker hub
"""
logger.info('Building a new docker image from Dockerfile')
logger.info('Docker image name: {}'.format(docker_image_name))
entry_point = os.path.join(os.path.dirname(__file__), 'entry_point.py')
create_handler_zip(codeengine_config.FH_ZIP_LOCATION, entry_point,
'lithopsentry.py')
if dockerfile:
cmd = 'docker build -t {} -f {} .'.format(docker_image_name,
dockerfile)
else:
cmd = 'docker build -t {} .'.format(docker_image_name)
if not self.log_active:
cmd = cmd + " >{} 2>&1".format(os.devnull)
print(cmd)
res = os.system(cmd)
if res != 0:
raise Exception('There was an error building the runtime')
self._delete_function_handler_zip()
cmd = 'docker push {}'.format(docker_image_name)
if not self.log_active:
cmd = cmd + " >{} 2>&1".format(os.devnull)
res = os.system(cmd)
if res != 0:
raise Exception(
'There was an error pushing the runtime to the container registry'
)
def build_runtime(self, runtime_name, requirements_file=None):
try:
shutil.rmtree(az_config.BUILD_DIR)
except Exception:
pass
action_name = self._format_action_name(runtime_name)
build_dir = os.path.join(az_config.BUILD_DIR, action_name)
os.makedirs(build_dir, exist_ok=True)
logger.info('Building default runtime in {}'.format(build_dir))
action_dir = os.path.join(build_dir, az_config.ACTION_DIR)
os.makedirs(action_dir, exist_ok=True)
req_file = os.path.join(build_dir, 'requirements.txt')
with open(req_file, 'w') as reqf:
reqf.write(az_config.REQUIREMENTS_FILE)
if not is_unix_system():
if 'dev' in lithops.__version__:
reqf.write('git+https://github.com/lithops-cloud/lithops')
else:
reqf.write('lithops=={}'.format(lithops.__version__))
host_file = os.path.join(build_dir, 'host.json')
with open(host_file, 'w') as hstf:
hstf.write(az_config.HOST_FILE)
fn_file = os.path.join(action_dir, 'function.json')
if self.invocation_type == 'event':
with open(fn_file, 'w') as fnf:
in_q_name = self._format_queue_name(action_name, az_config.IN_QUEUE)
az_config.BINDINGS_QUEUE['bindings'][0]['queueName'] = in_q_name
out_q_name = self._format_queue_name(action_name, az_config.OUT_QUEUE)
az_config.BINDINGS_QUEUE['bindings'][1]['queueName'] = out_q_name
fnf.write(json.dumps(az_config.BINDINGS_QUEUE))
elif self.invocation_type == 'http':
with open(fn_file, 'w') as fnf:
fnf.write(json.dumps(az_config.BINDINGS_HTTP))
entry_point = os.path.join(os.path.dirname(__file__), 'entry_point.py')
main_file = os.path.join(action_dir, '__init__.py')
shutil.copy(entry_point, main_file)
if is_unix_system():
mod_dir = os.path.join(build_dir, az_config.ACTION_MODULES_DIR)
os.chdir(build_dir)
cmd = '{} -m pip install -U -t {} -r requirements.txt'.format(sys.executable, mod_dir)
if logger.getEffectiveLevel() != logging.DEBUG:
cmd = cmd + " >{} 2>&1".format(os.devnull)
os.system(cmd)
create_handler_zip(az_config.FH_ZIP_LOCATION, entry_point, '__init__.py')
archive = zipfile.ZipFile(az_config.FH_ZIP_LOCATION)
archive.extractall(path=mod_dir)
os.remove(mod_dir+'/__init__.py')
os.remove(az_config.FH_ZIP_LOCATION)
def _setup_lithops(self):
"""
Setup lithops necessary files and dirs in master VM instance
"""
logger.debug('Installing Lithops in {}'.format(self.backend.master))
ssh_client = self.backend.master.get_ssh_client()
# Upload local lithops version to remote VM instance
src_proxy = os.path.join(os.path.dirname(__file__), 'proxy.py')
create_handler_zip(LOCAL_FH_ZIP_LOCATION, src_proxy)
current_location = os.path.dirname(os.path.abspath(__file__))
ssh_location = os.path.join(current_location, '..', 'util',
'ssh_client.py')
controller_location = os.path.join(current_location, 'controller.py')
logger.debug('Uploading lithops files to {}'.format(
self.backend.master))
files_to_upload = [
(LOCAL_FH_ZIP_LOCATION, '/tmp/lithops_standalone.zip'),
(ssh_location, '/tmp/ssh_client.py'.format(REMOTE_INSTALL_DIR)),
(controller_location,
'/tmp/controller.py'.format(REMOTE_INSTALL_DIR))
]
ssh_client.upload_multiple_local_files(files_to_upload)
os.remove(LOCAL_FH_ZIP_LOCATION)
instance_data = {
'instance_name': self.backend.master.name,
'ip_address': self.backend.master.ip_address,
'instance_id': self.backend.master.instance_id
}
script = """
mv {0}/access.data .;
rm -R {0};
mkdir -p {0};
cp /tmp/lithops_standalone.zip {0};
mkdir -p /tmp/lithops;
mv access.data {0}/access.data;
test -f {0}/access.data || echo '{1}' > {0}/access.data;
test -f {0}/config || echo '{2}' > {0}/config;
mv /tmp/*.py '{0}';
apt-get install python3-paramiko -y >> /tmp/lithops/proxy.log 2>&1;
python3 {0}/controller.py setup; >> /tmp/lithops/proxy.log 2>&1;
""".format(REMOTE_INSTALL_DIR, json.dumps(instance_data),
json.dumps(self.config))
logger.debug('Executing lithops installation process on {}'.format(
self.backend.master))
logger.debug(
'Be patient, initial installation process may take up to 5 minutes'
)
ssh_client.run_remote_command(script)
logger.debug('Lithops installation process completed')
def build_runtime(self, runtime_name, runtime_file, extra_args=[]):
logger.debug('Building new docker image from Dockerfile')
logger.debug('Docker image name: {}'.format(runtime_name))
expression = '^([a-zA-Z0-9_-]+)(:[a-zA-Z0-9_-]+)+$'
result = re.match(expression, runtime_name)
if not result or result.group() != runtime_name:
raise Exception(
"Runtime name must satisfy regex {}".format(expression))
entry_point = os.path.join(os.path.dirname(__file__), 'entry_point.py')
create_handler_zip(os.path.join(os.getcwd(), RUNTIME_ZIP), entry_point)
res = self.ecr_client.get_authorization_token()
if res['ResponseMetadata']['HTTPStatusCode'] != 200:
raise Exception('Could not get ECR auth token: {}'.format(res))
auth_data = res['authorizationData'].pop()
ecr_token = base64.b64decode(
auth_data['authorizationToken']).split(b':')[1]
full_image_name, registry, repo_name = self._get_full_image_name(
runtime_name)
if runtime_file:
cmd = '{} build -t {} -f {} .'.format(batch_config.DOCKER_PATH,
full_image_name,
runtime_file)
else:
cmd = '{} build -t {} .'.format(batch_config.DOCKER_PATH,
full_image_name)
subprocess.check_call(cmd.split())
os.remove(RUNTIME_ZIP)
cmd = '{} login --username AWS --password-stdin {}'.format(
batch_config.DOCKER_PATH, registry)
subprocess.check_output(cmd.split(), input=ecr_token)
try:
self.ecr_client.create_repository(repositoryName=repo_name,
imageTagMutability='MUTABLE')
except self.ecr_client.exceptions.RepositoryAlreadyExistsException as e:
logger.info('Repository {} already exists'.format(repo_name))
cmd = '{} push {}'.format(batch_config.DOCKER_PATH, full_image_name)
subprocess.check_call(cmd.split())
logger.debug('Runtime {} built successfully'.format(runtime_name))
def build_runtime(self, docker_image_name, dockerfile, extra_args=[]):
"""
Builds a new runtime from a Docker file and pushes it to the Docker hub
"""
logger.debug('Building a new docker image from Dockerfile')
logger.debug('Docker image name: {}'.format(docker_image_name))
expression = '^([a-z0-9]+)/([-a-z0-9]+)(:[a-z0-9]+)?'
result = re.match(expression, docker_image_name)
if not result or result.group() != docker_image_name:
raise Exception(
"Invalid docker image name: All letters must be "
"lowercase and '.' or '_' characters are not allowed")
entry_point = os.path.join(os.path.dirname(__file__), 'entry_point.py')
create_handler_zip(kconfig.FH_ZIP_LOCATION, entry_point,
'lithopsproxy.py')
if dockerfile:
cmd = '{} build -t {} -f {} . '.format(kconfig.DOCKER_PATH,
docker_image_name,
dockerfile)
else:
cmd = '{} build -t {} .'.format(kconfig.DOCKER_PATH,
docker_image_name)
cmd = cmd + ' '.join(extra_args)
logger.info('Building default runtime')
if logger.getEffectiveLevel() != logging.DEBUG:
cmd = cmd + " >{} 2>&1".format(os.devnull)
res = os.system(cmd)
if res != 0:
raise Exception('There was an error building the runtime')
self._delete_function_handler_zip()
cmd = '{} push {}'.format(kconfig.DOCKER_PATH, docker_image_name)
if logger.getEffectiveLevel() != logging.DEBUG:
cmd = cmd + " >{} 2>&1".format(os.devnull)
res = os.system(cmd)
if res != 0:
raise Exception(
'There was an error pushing the runtime to the container registry'
)
def _create_handler_bin(remove=True):
"""
Create and return Lithops handler function as zip bytes
@param remove: True to delete the zip archive after building
@return: Lithops handler function as zip bytes
"""
current_location = os.path.dirname(os.path.abspath(__file__))
main_file = os.path.join(current_location, 'entry_point.py')
utils.create_handler_zip(LITHOPS_FUNCTION_ZIP, main_file, 'entry_point.py')
with open(LITHOPS_FUNCTION_ZIP, 'rb') as action_zip:
action_bin = action_zip.read()
if remove:
os.remove(LITHOPS_FUNCTION_ZIP)
return action_bin
def _setup_proxy(self):
logger.debug('Installing Lithops proxy in the VM instance')
logger.debug(
'Be patient, installation process can take up to 3 minutes '
'if this is the first time you use the VM instance')
service_file = '/etc/systemd/system/{}'.format(PROXY_SERVICE_NAME)
self.ssh_client.upload_data_to_file(self.ip_address,
PROXY_SERVICE_FILE, service_file)
cmd = 'rm -R {}; mkdir -p {}; '.format(REMOTE_INSTALL_DIR,
REMOTE_INSTALL_DIR)
cmd += 'systemctl daemon-reload; systemctl stop {}; '.format(
PROXY_SERVICE_NAME)
self.ssh_client.run_remote_command(self.ip_address, cmd)
config_file = os.path.join(REMOTE_INSTALL_DIR, 'config')
self.ssh_client.upload_data_to_file(self.ip_address,
json.dumps(self.config),
config_file)
src_proxy = os.path.join(os.path.dirname(__file__), 'proxy.py')
create_handler_zip(FH_ZIP_LOCATION, src_proxy)
self.ssh_client.upload_local_file(self.ip_address, FH_ZIP_LOCATION,
'/tmp/lithops_standalone.zip')
os.remove(FH_ZIP_LOCATION)
# Install dependenices
cmd = 'mkdir -p /tmp/lithops; '
cmd += 'apt-get update >> /tmp/lithops/proxy.log; '
cmd += 'apt-get install unzip python3-pip -y >> /tmp/lithops/proxy.log; '
cmd += 'pip3 install flask gevent pika==0.13.1 >> /tmp/lithops/proxy.log; '
cmd += 'unzip -o /tmp/lithops_standalone.zip -d {} > /dev/null 2>&1; '.format(
REMOTE_INSTALL_DIR)
cmd += 'rm /tmp/lithops_standalone.zip; '
cmd += 'chmod 644 {}; '.format(service_file)
# Start proxy service
cmd += 'systemctl daemon-reload; '
cmd += 'systemctl stop {}; '.format(PROXY_SERVICE_NAME)
cmd += 'systemctl enable {}; '.format(PROXY_SERVICE_NAME)
cmd += 'systemctl start {}; '.format(PROXY_SERVICE_NAME)
self.ssh_client.run_remote_command(self.ip_address,
cmd,
background=True)
def _setup_master_service(self):
"""
Setup lithops necessary packages and files in master VM instance
"""
logger.info(f'Installing Lithops in {self.backend.master}')
ssh_client = self.backend.master.get_ssh_client()
worker_path = os.path.join(os.path.dirname(__file__), 'worker.py')
master_path = os.path.join(os.path.dirname(__file__), 'master.py')
create_handler_zip(LOCAL_FH_ZIP_LOCATION, [master_path, worker_path])
logger.debug('Uploading lithops files to {}'.format(self.backend.master))
ssh_client.upload_local_file(LOCAL_FH_ZIP_LOCATION, '/tmp/lithops_standalone.zip')
os.remove(LOCAL_FH_ZIP_LOCATION)
vm_data = {'name': self.backend.master.name,
'instance_id': self.backend.master.instance_id,
'private_ip': self.backend.master.private_ip,
'delete_on_dismantle': self.backend.master.delete_on_dismantle,
'lithops_version': lithops_version}
logger.debug('Executing lithops installation process on {}'.format(self.backend.master))
logger.debug('Be patient, initial installation process may take up to 3 minutes')
remote_script = "/tmp/install_lithops.sh"
script = get_master_setup_script(self.config, vm_data)
ssh_client.upload_data_to_file(script, remote_script)
ssh_client.run_remote_command(f"chmod 777 {remote_script}; sudo {remote_script};")
try:
# Download the master VM public key generated with the installation script
# This public key will be used to create to worker
ssh_client.download_remote_file(
f'{self.backend.master.home_dir}/.ssh/id_rsa.pub',
f'{self.backend.cache_dir}/{self.backend.master.name}-id_rsa.pub')
except FileNotFoundError:
pass