def build_runtime_docker(self, docker_image_name, dockerfile): """ Builds a new runtime from a Docker file and pushes it to the Docker hub """ logger.debug('Building new docker image from Dockerfile') logger.debug('Docker image name: {}'.format(docker_image_name)) entry_point = os.path.join(os.path.dirname(__file__), 'entry_point.py') create_handler_zip(az_config.FH_ZIP_LOCATION, entry_point, '__init__.py') if dockerfile: cmd = '{} build -t {} -f {} .'.format(az_config.DOCKER_PATH, docker_image_name, dockerfile) else: cmd = '{} build -t {} .'.format(az_config.DOCKER_PATH, docker_image_name) if logger.getEffectiveLevel() != logging.DEBUG: cmd = cmd + " >{} 2>&1".format(os.devnull) logger.info('Building default runtime') res = os.system(cmd) if res != 0: raise Exception('There was an error building the runtime') cmd = '{} push {}'.format(az_config.DOCKER_PATH, docker_image_name) if logger.getEffectiveLevel() != logging.DEBUG: cmd = cmd + " >{} 2>&1".format(os.devnull) res = os.system(cmd) if res != 0: raise Exception('There was an error pushing the runtime to the container registry') logger.debug('Done!')
def create_runtime(self, docker_image_name, memory, timeout): """ Creates a new runtime into IBM CF namespace from an already built Docker image """ if docker_image_name == 'default': docker_image_name = self._get_default_runtime_image_name() logger.info('Creating new Lithops runtime based on Docker image {}'.format(docker_image_name)) self.cf_client.create_package(self.package) action_name = self._format_action_name(docker_image_name, memory) entry_point = os.path.join(os.path.dirname(__file__), 'entry_point.py') create_handler_zip(ibmcf_config.FH_ZIP_LOCATION, entry_point, '__main__.py') with open(ibmcf_config.FH_ZIP_LOCATION, "rb") as action_zip: action_bin = action_zip.read() self.cf_client.create_action(self.package, action_name, docker_image_name, code=action_bin, memory=memory, is_binary=True, timeout=timeout * 1000) self._delete_function_handler_zip() runtime_meta = self._generate_runtime_meta(docker_image_name, memory) return runtime_meta
def _setup_master_service(self): """ Setup lithops necessary packages and files in master VM instance """ logger.debug('Installing Lithops in {}'.format(self.backend.master)) ssh_client = self.backend.master.get_ssh_client() src_proxy = os.path.join(os.path.dirname(__file__), 'worker.py') create_handler_zip(LOCAL_FH_ZIP_LOCATION, src_proxy) current_location = os.path.dirname(os.path.abspath(__file__)) controller_location = os.path.join(current_location, 'master.py') logger.debug('Uploading lithops files to {}'.format(self.backend.master)) files_to_upload = [(LOCAL_FH_ZIP_LOCATION, '/tmp/lithops_standalone.zip'), (controller_location, '/tmp/master.py'.format(STANDALONE_INSTALL_DIR))] ssh_client.upload_multiple_local_files(files_to_upload) os.remove(LOCAL_FH_ZIP_LOCATION) vm_data = {'instance_name': self.backend.master.name, 'ip_address': self.backend.master.ip_address, 'instance_id': self.backend.master.instance_id} script = get_master_setup_script(self.config, vm_data) logger.debug('Executing lithops installation process on {}'.format(self.backend.master)) logger.debug('Be patient, initial installation process may take up to 5 minutes') ssh_client.run_remote_command(script) logger.debug('Lithops installation process completed')
def build_runtime(self, docker_image_name, dockerfile): """ Builds a new runtime from a Docker file and pushes it to the Docker hub """ logger.info('Building a new docker image from Dockerfile') logger.info('Docker image name: {}'.format(docker_image_name)) # Project ID can contain '-' expression = '^([-a-z0-9]+)/([-a-z0-9]+)(:[a-z0-9]+)?' result = re.match(expression, docker_image_name) if not result or result.group() != docker_image_name: raise Exception( "Invalid docker image name: '.' or '_' characters are not allowed" ) entry_point = os.path.join(os.path.dirname(__file__), 'entry_point.py') create_handler_zip(cr_config.FH_ZIP_LOCATION, entry_point, 'lithopsproxy.py') # Dockerfile has to be called "Dockerfile" (and in cwd) for 'gcloud builds submit' to work if dockerfile != "Dockerfile": shutil.copyfile(dockerfile, "Dockerfile") cmd = 'gcloud builds submit -t gcr.io/{}'.format(docker_image_name) if not self.log_active: cmd = cmd + " >{} 2>&1".format(os.devnull) res = os.system(cmd) if res != 0: raise Exception('There was an error building the runtime') self._delete_function_handler_zip()
def deploy_runtime(self, docker_image_name, memory, timeout): """ Deploys a new runtime into IBM CF namespace from an already built Docker image """ if docker_image_name == 'default': docker_image_name = self._get_default_runtime_image_name() logger.debug( f"Deploying runtime: {docker_image_name} - Memory: {memory} Timeout: {timeout}" ) self.cf_client.create_package(self.package) action_name = self._format_function_name(docker_image_name, memory) entry_point = os.path.join(os.path.dirname(__file__), 'entry_point.py') create_handler_zip(ow_config.FH_ZIP_LOCATION, entry_point, '__main__.py') with open(ow_config.FH_ZIP_LOCATION, "rb") as action_zip: action_bin = action_zip.read() self.cf_client.create_action(self.package, action_name, docker_image_name, code=action_bin, memory=memory, is_binary=True, timeout=timeout * 1000) self._delete_function_handler_zip() return self._generate_runtime_meta(docker_image_name, memory)
def build_runtime(self, docker_image_name, dockerfile): """ Builds a new runtime from a Docker file and pushes it to the Docker hub """ logger.info('Building a new docker image from Dockerfile') logger.info('Docker image name: {}'.format(docker_image_name)) entry_point = os.path.join(os.path.dirname(__file__), 'entry_point.py') create_handler_zip(codeengine_config.FH_ZIP_LOCATION, entry_point, 'lithopsentry.py') if dockerfile: cmd = 'docker build -t {} -f {} .'.format(docker_image_name, dockerfile) else: cmd = 'docker build -t {} .'.format(docker_image_name) if not self.log_active: cmd = cmd + " >{} 2>&1".format(os.devnull) print(cmd) res = os.system(cmd) if res != 0: raise Exception('There was an error building the runtime') self._delete_function_handler_zip() cmd = 'docker push {}'.format(docker_image_name) if not self.log_active: cmd = cmd + " >{} 2>&1".format(os.devnull) res = os.system(cmd) if res != 0: raise Exception( 'There was an error pushing the runtime to the container registry' )
def build_runtime(self, runtime_name, requirements_file=None): try: shutil.rmtree(az_config.BUILD_DIR) except Exception: pass action_name = self._format_action_name(runtime_name) build_dir = os.path.join(az_config.BUILD_DIR, action_name) os.makedirs(build_dir, exist_ok=True) logger.info('Building default runtime in {}'.format(build_dir)) action_dir = os.path.join(build_dir, az_config.ACTION_DIR) os.makedirs(action_dir, exist_ok=True) req_file = os.path.join(build_dir, 'requirements.txt') with open(req_file, 'w') as reqf: reqf.write(az_config.REQUIREMENTS_FILE) if not is_unix_system(): if 'dev' in lithops.__version__: reqf.write('git+https://github.com/lithops-cloud/lithops') else: reqf.write('lithops=={}'.format(lithops.__version__)) host_file = os.path.join(build_dir, 'host.json') with open(host_file, 'w') as hstf: hstf.write(az_config.HOST_FILE) fn_file = os.path.join(action_dir, 'function.json') if self.invocation_type == 'event': with open(fn_file, 'w') as fnf: in_q_name = self._format_queue_name(action_name, az_config.IN_QUEUE) az_config.BINDINGS_QUEUE['bindings'][0]['queueName'] = in_q_name out_q_name = self._format_queue_name(action_name, az_config.OUT_QUEUE) az_config.BINDINGS_QUEUE['bindings'][1]['queueName'] = out_q_name fnf.write(json.dumps(az_config.BINDINGS_QUEUE)) elif self.invocation_type == 'http': with open(fn_file, 'w') as fnf: fnf.write(json.dumps(az_config.BINDINGS_HTTP)) entry_point = os.path.join(os.path.dirname(__file__), 'entry_point.py') main_file = os.path.join(action_dir, '__init__.py') shutil.copy(entry_point, main_file) if is_unix_system(): mod_dir = os.path.join(build_dir, az_config.ACTION_MODULES_DIR) os.chdir(build_dir) cmd = '{} -m pip install -U -t {} -r requirements.txt'.format(sys.executable, mod_dir) if logger.getEffectiveLevel() != logging.DEBUG: cmd = cmd + " >{} 2>&1".format(os.devnull) os.system(cmd) create_handler_zip(az_config.FH_ZIP_LOCATION, entry_point, '__init__.py') archive = zipfile.ZipFile(az_config.FH_ZIP_LOCATION) archive.extractall(path=mod_dir) os.remove(mod_dir+'/__init__.py') os.remove(az_config.FH_ZIP_LOCATION)
def _setup_lithops(self): """ Setup lithops necessary files and dirs in master VM instance """ logger.debug('Installing Lithops in {}'.format(self.backend.master)) ssh_client = self.backend.master.get_ssh_client() # Upload local lithops version to remote VM instance src_proxy = os.path.join(os.path.dirname(__file__), 'proxy.py') create_handler_zip(LOCAL_FH_ZIP_LOCATION, src_proxy) current_location = os.path.dirname(os.path.abspath(__file__)) ssh_location = os.path.join(current_location, '..', 'util', 'ssh_client.py') controller_location = os.path.join(current_location, 'controller.py') logger.debug('Uploading lithops files to {}'.format( self.backend.master)) files_to_upload = [ (LOCAL_FH_ZIP_LOCATION, '/tmp/lithops_standalone.zip'), (ssh_location, '/tmp/ssh_client.py'.format(REMOTE_INSTALL_DIR)), (controller_location, '/tmp/controller.py'.format(REMOTE_INSTALL_DIR)) ] ssh_client.upload_multiple_local_files(files_to_upload) os.remove(LOCAL_FH_ZIP_LOCATION) instance_data = { 'instance_name': self.backend.master.name, 'ip_address': self.backend.master.ip_address, 'instance_id': self.backend.master.instance_id } script = """ mv {0}/access.data .; rm -R {0}; mkdir -p {0}; cp /tmp/lithops_standalone.zip {0}; mkdir -p /tmp/lithops; mv access.data {0}/access.data; test -f {0}/access.data || echo '{1}' > {0}/access.data; test -f {0}/config || echo '{2}' > {0}/config; mv /tmp/*.py '{0}'; apt-get install python3-paramiko -y >> /tmp/lithops/proxy.log 2>&1; python3 {0}/controller.py setup; >> /tmp/lithops/proxy.log 2>&1; """.format(REMOTE_INSTALL_DIR, json.dumps(instance_data), json.dumps(self.config)) logger.debug('Executing lithops installation process on {}'.format( self.backend.master)) logger.debug( 'Be patient, initial installation process may take up to 5 minutes' ) ssh_client.run_remote_command(script) logger.debug('Lithops installation process completed')
def build_runtime(self, runtime_name, runtime_file, extra_args=[]): logger.debug('Building new docker image from Dockerfile') logger.debug('Docker image name: {}'.format(runtime_name)) expression = '^([a-zA-Z0-9_-]+)(:[a-zA-Z0-9_-]+)+$' result = re.match(expression, runtime_name) if not result or result.group() != runtime_name: raise Exception( "Runtime name must satisfy regex {}".format(expression)) entry_point = os.path.join(os.path.dirname(__file__), 'entry_point.py') create_handler_zip(os.path.join(os.getcwd(), RUNTIME_ZIP), entry_point) res = self.ecr_client.get_authorization_token() if res['ResponseMetadata']['HTTPStatusCode'] != 200: raise Exception('Could not get ECR auth token: {}'.format(res)) auth_data = res['authorizationData'].pop() ecr_token = base64.b64decode( auth_data['authorizationToken']).split(b':')[1] full_image_name, registry, repo_name = self._get_full_image_name( runtime_name) if runtime_file: cmd = '{} build -t {} -f {} .'.format(batch_config.DOCKER_PATH, full_image_name, runtime_file) else: cmd = '{} build -t {} .'.format(batch_config.DOCKER_PATH, full_image_name) subprocess.check_call(cmd.split()) os.remove(RUNTIME_ZIP) cmd = '{} login --username AWS --password-stdin {}'.format( batch_config.DOCKER_PATH, registry) subprocess.check_output(cmd.split(), input=ecr_token) try: self.ecr_client.create_repository(repositoryName=repo_name, imageTagMutability='MUTABLE') except self.ecr_client.exceptions.RepositoryAlreadyExistsException as e: logger.info('Repository {} already exists'.format(repo_name)) cmd = '{} push {}'.format(batch_config.DOCKER_PATH, full_image_name) subprocess.check_call(cmd.split()) logger.debug('Runtime {} built successfully'.format(runtime_name))
def build_runtime(self, docker_image_name, dockerfile, extra_args=[]): """ Builds a new runtime from a Docker file and pushes it to the Docker hub """ logger.debug('Building a new docker image from Dockerfile') logger.debug('Docker image name: {}'.format(docker_image_name)) expression = '^([a-z0-9]+)/([-a-z0-9]+)(:[a-z0-9]+)?' result = re.match(expression, docker_image_name) if not result or result.group() != docker_image_name: raise Exception( "Invalid docker image name: All letters must be " "lowercase and '.' or '_' characters are not allowed") entry_point = os.path.join(os.path.dirname(__file__), 'entry_point.py') create_handler_zip(kconfig.FH_ZIP_LOCATION, entry_point, 'lithopsproxy.py') if dockerfile: cmd = '{} build -t {} -f {} . '.format(kconfig.DOCKER_PATH, docker_image_name, dockerfile) else: cmd = '{} build -t {} .'.format(kconfig.DOCKER_PATH, docker_image_name) cmd = cmd + ' '.join(extra_args) logger.info('Building default runtime') if logger.getEffectiveLevel() != logging.DEBUG: cmd = cmd + " >{} 2>&1".format(os.devnull) res = os.system(cmd) if res != 0: raise Exception('There was an error building the runtime') self._delete_function_handler_zip() cmd = '{} push {}'.format(kconfig.DOCKER_PATH, docker_image_name) if logger.getEffectiveLevel() != logging.DEBUG: cmd = cmd + " >{} 2>&1".format(os.devnull) res = os.system(cmd) if res != 0: raise Exception( 'There was an error pushing the runtime to the container registry' )
def _create_handler_bin(remove=True): """ Create and return Lithops handler function as zip bytes @param remove: True to delete the zip archive after building @return: Lithops handler function as zip bytes """ current_location = os.path.dirname(os.path.abspath(__file__)) main_file = os.path.join(current_location, 'entry_point.py') utils.create_handler_zip(LITHOPS_FUNCTION_ZIP, main_file, 'entry_point.py') with open(LITHOPS_FUNCTION_ZIP, 'rb') as action_zip: action_bin = action_zip.read() if remove: os.remove(LITHOPS_FUNCTION_ZIP) return action_bin
def _setup_proxy(self): logger.debug('Installing Lithops proxy in the VM instance') logger.debug( 'Be patient, installation process can take up to 3 minutes ' 'if this is the first time you use the VM instance') service_file = '/etc/systemd/system/{}'.format(PROXY_SERVICE_NAME) self.ssh_client.upload_data_to_file(self.ip_address, PROXY_SERVICE_FILE, service_file) cmd = 'rm -R {}; mkdir -p {}; '.format(REMOTE_INSTALL_DIR, REMOTE_INSTALL_DIR) cmd += 'systemctl daemon-reload; systemctl stop {}; '.format( PROXY_SERVICE_NAME) self.ssh_client.run_remote_command(self.ip_address, cmd) config_file = os.path.join(REMOTE_INSTALL_DIR, 'config') self.ssh_client.upload_data_to_file(self.ip_address, json.dumps(self.config), config_file) src_proxy = os.path.join(os.path.dirname(__file__), 'proxy.py') create_handler_zip(FH_ZIP_LOCATION, src_proxy) self.ssh_client.upload_local_file(self.ip_address, FH_ZIP_LOCATION, '/tmp/lithops_standalone.zip') os.remove(FH_ZIP_LOCATION) # Install dependenices cmd = 'mkdir -p /tmp/lithops; ' cmd += 'apt-get update >> /tmp/lithops/proxy.log; ' cmd += 'apt-get install unzip python3-pip -y >> /tmp/lithops/proxy.log; ' cmd += 'pip3 install flask gevent pika==0.13.1 >> /tmp/lithops/proxy.log; ' cmd += 'unzip -o /tmp/lithops_standalone.zip -d {} > /dev/null 2>&1; '.format( REMOTE_INSTALL_DIR) cmd += 'rm /tmp/lithops_standalone.zip; ' cmd += 'chmod 644 {}; '.format(service_file) # Start proxy service cmd += 'systemctl daemon-reload; ' cmd += 'systemctl stop {}; '.format(PROXY_SERVICE_NAME) cmd += 'systemctl enable {}; '.format(PROXY_SERVICE_NAME) cmd += 'systemctl start {}; '.format(PROXY_SERVICE_NAME) self.ssh_client.run_remote_command(self.ip_address, cmd, background=True)
def _setup_master_service(self): """ Setup lithops necessary packages and files in master VM instance """ logger.info(f'Installing Lithops in {self.backend.master}') ssh_client = self.backend.master.get_ssh_client() worker_path = os.path.join(os.path.dirname(__file__), 'worker.py') master_path = os.path.join(os.path.dirname(__file__), 'master.py') create_handler_zip(LOCAL_FH_ZIP_LOCATION, [master_path, worker_path]) logger.debug('Uploading lithops files to {}'.format(self.backend.master)) ssh_client.upload_local_file(LOCAL_FH_ZIP_LOCATION, '/tmp/lithops_standalone.zip') os.remove(LOCAL_FH_ZIP_LOCATION) vm_data = {'name': self.backend.master.name, 'instance_id': self.backend.master.instance_id, 'private_ip': self.backend.master.private_ip, 'delete_on_dismantle': self.backend.master.delete_on_dismantle, 'lithops_version': lithops_version} logger.debug('Executing lithops installation process on {}'.format(self.backend.master)) logger.debug('Be patient, initial installation process may take up to 3 minutes') remote_script = "/tmp/install_lithops.sh" script = get_master_setup_script(self.config, vm_data) ssh_client.upload_data_to_file(script, remote_script) ssh_client.run_remote_command(f"chmod 777 {remote_script}; sudo {remote_script};") try: # Download the master VM public key generated with the installation script # This public key will be used to create to worker ssh_client.download_remote_file( f'{self.backend.master.home_dir}/.ssh/id_rsa.pub', f'{self.backend.cache_dir}/{self.backend.master.name}-id_rsa.pub') except FileNotFoundError: pass