def build(dir, requirements_dir, docker_tag): """ Builds a Docker image that contains code under the given source root directory. Assumes that Docker is installed and running locally. :param dir: [str], source root directory :param requirements_dir: [str], path to requirements.txt """ sagify_module_path = os.path.relpath(os.path.join(dir, 'sagify/')) build_script_path = os.path.join(sagify_module_path, 'build.sh') dockerfile_path = os.path.join(sagify_module_path, 'Dockerfile') train_file_path = os.path.join(sagify_module_path, 'training', 'train') serve_file_path = os.path.join(sagify_module_path, 'prediction', 'serve') executor_file_path = os.path.join(sagify_module_path, 'executor.sh') if not os.path.isfile(build_script_path) or not os.path.isfile(train_file_path) or not \ os.path.isfile(serve_file_path): raise ValueError("This is not a sagify directory: {}".format(dir)) os.chmod(train_file_path, 0o777) os.chmod(serve_file_path, 0o777) os.chmod(executor_file_path, 0o777) target_dir_name = os.path.basename(os.path.normpath(dir)) output = subprocess.check_output([ "{}".format(build_script_path), "{}".format(os.path.relpath(dir)), "{}".format(os.path.relpath(target_dir_name)), "{}".format(dockerfile_path), "{}".format(os.path.relpath(requirements_dir)), docker_tag ]) logger.debug(output)
def build(obj): """ Command to build SageMaker app """ logger.info(ASCII_LOGO) logger.info( "Started building SageMaker Docker image. It will take some minutes...\n" ) try: config_file_path = os.path.join('.sagify.json') if not os.path.isfile(config_file_path): raise ValueError() config = ConfigManager(config_file_path).get_config() api_build.build(source_dir=config.sagify_module_dir, requirements_dir=config.requirements_dir, docker_tag=obj['docker_tag'], image_name=config.image_name, python_version=config.python_version) logger.info("Docker image built successfully!") except ValueError: logger.info("This is not a sagify directory: {}".format(dir)) sys.exit(-1) except subprocess.CalledProcessError as e: logger.debug(e.output) raise except Exception as e: logger.info("{}".format(e)) sys.exit(-1)
def push(dir, docker_tag, aws_region, iam_role_arn, aws_profile, external_id, image_name): """ Push Docker image to AWS ECS :param dir: [str], source root directory :param docker_tag: [str], the Docker tag for the image :param aws_region: [str], the AWS region to push the image to :param iam_role_arn: [str], the AWS role used to push the image to ECR :param aws_profile: [str], the AWS profile used to push the image to ECR :param external_id: [str], Optional external id used when using an IAM role :param image_name: [str], The name of the Docker image """ sagify_module_path = os.path.relpath(os.path.join(dir, 'sagify/')) push_script_path = os.path.join(sagify_module_path, 'push.sh') if not os.path.isfile(push_script_path): raise ValueError("This is not a sagify directory: {}".format(dir)) output = subprocess.check_output([ "{}".format(push_script_path), docker_tag, aws_region, iam_role_arn, aws_profile, external_id, image_name ]) logger.debug(output)
def push(obj, dir, aws_region, iam_role_arn, aws_profile, external_id): """ Command to push Docker image to AWS ECS """ logger.info(ASCII_LOGO) logger.info( "Started pushing Docker image to AWS ECS. It will take some time. Please, be patient...\n" ) if iam_role_arn is not None and aws_profile is not None: logger.error('Only one of iam-role-arn and aws-profile can be used.') sys.exit(2) try: api_push.push(dir=dir, docker_tag=obj['docker_tag'], aws_region=aws_region, iam_role_arn=iam_role_arn, aws_profile=aws_profile, external_id=external_id) logger.info("Docker image pushed to ECS successfully!") except ValueError: logger.info("This is not a sagify directory: {}".format(dir)) sys.exit(-1) except subprocess.CalledProcessError as e: logger.debug(e.output) raise except Exception as e: logger.info("{}".format(e)) return
def push(dir, docker_tag): """ Push Docker image to AWS ECS :param dir: [str], source root directory """ sagify_module_path = os.path.relpath(os.path.join(dir, 'sagify/')) push_script_path = os.path.join(sagify_module_path, 'push.sh') if not os.path.isfile(push_script_path): raise ValueError("This is not a sagify directory: {}".format(dir)) output = subprocess.check_output(["{}".format(push_script_path), docker_tag]) logger.debug(output)
def push(obj, aws_region, iam_role_arn, aws_profile, external_id): """ Command to push Docker image to AWS ECS """ logger.info(ASCII_LOGO) if iam_role_arn is not None and aws_profile is not None: logger.error('Only one of iam-role-arn and aws-profile can be used.') sys.exit(2) if iam_role_arn is not None: aws_profile = '' try: config_file_path = os.path.join('.sagify.json') if not os.path.isfile(config_file_path): raise ValueError() config = ConfigManager(config_file_path).get_config() image_name = config.image_name aws_region = config.aws_region if aws_region is None else aws_region aws_profile = config.aws_profile if ( aws_profile is None and iam_role_arn is None) else aws_profile external_id = "" if external_id is None else external_id iam_role_arn = "" if iam_role_arn is None else iam_role_arn logger.info( "Started pushing Docker image to AWS ECS. It will take some time. Please, be patient...\n" ) api_push.push(dir=config.sagify_module_dir, docker_tag=obj['docker_tag'], aws_region=aws_region, iam_role_arn=iam_role_arn, aws_profile=aws_profile, external_id=external_id, image_name=image_name) logger.info("Docker image pushed to ECS successfully!") except ValueError: logger.info("This is not a sagify directory: {}".format(dir)) sys.exit(-1) except subprocess.CalledProcessError as e: logger.debug(e.output) raise except Exception as e: logger.info("{}".format(e)) return
def deploy(obj, dir): """ Command to deploy ML model(s) locally """ logger.info(ASCII_LOGO) logger.info("Started local deployment at localhost:8080 ...\n") try: api_local.deploy(dir=dir, docker_tag=obj['docker_tag']) except ValueError: logger.info("This is not a sagify directory: {}".format(dir)) sys.exit(-1) except subprocess.CalledProcessError as e: logger.debug(e.output) raise except Exception as e: logger.info("{}".format(e)) return
def deploy(obj): """ Command to deploy ML model(s) locally """ logger.info(ASCII_LOGO) logger.info("Started local deployment at localhost:8080 ...\n") try: config = ConfigManager(os.path.join('.sagify.json')).get_config() api_local.deploy(dir=config.sagify_module_dir, docker_tag=obj['docker_tag'], image_name=config.image_name) except ValueError: logger.info("This is not a sagify directory: {}".format(dir)) sys.exit(-1) except subprocess.CalledProcessError as e: logger.debug(e.output) raise except Exception as e: logger.info("{}".format(e)) sys.exit(-1)
def train(obj, dir): """ Command to train ML model(s) locally """ logger.info(ASCII_LOGO) logger.info("Started local training...\n") try: api_local.train(dir=dir, docker_tag=obj['docker_tag']) logger.info("Local training completed successfully!") except ValueError: logger.info("This is not a sagify directory: {}".format(dir)) sys.exit(-1) except subprocess.CalledProcessError as e: logger.debug(e.output) raise except Exception as e: logger.info("{}".format(e)) return
def deploy(dir, docker_tag): """ Deploys ML models(s) locally :param dir: [str], source root directory :param docker_tag: [str], the Docker tag for the image """ sagify_module_path = os.path.join(dir, 'sagify') local_deploy_script_path = os.path.join(sagify_module_path, 'local_test', 'deploy_local.sh') test_path = os.path.join(sagify_module_path, 'local_test', 'test_dir') if not os.path.isdir(test_path): raise ValueError("This is not a sagify directory: {}".format(dir)) output = subprocess.check_output([ "{}".format(local_deploy_script_path), "{}".format(os.path.abspath(test_path)), docker_tag ]) logger.debug(output)
def train(dir, docker_tag, image_name): """ Trains ML model(s) locally :param dir: [str], source root directory :param docker_tag: [str], the Docker tag for the image :param image_name: [str], The name of the Docker image """ sagify_module_path = os.path.join(dir, 'sagify') local_train_script_path = os.path.join(sagify_module_path, 'local_test', 'train_local.sh') test_path = os.path.join(sagify_module_path, 'local_test', 'test_dir') if not os.path.isdir(test_path): raise ValueError("This is not a sagify directory: {}".format(dir)) output = subprocess.check_output([ "{}".format(local_train_script_path), "{}".format(os.path.abspath(test_path)), docker_tag, image_name ]) logger.debug(output)
def push(obj, dir): """ Command to push Docker image to AWS ECS """ logger.info(ASCII_LOGO) logger.info( "Started pushing Docker image to AWS ECS. It will take some time. Please, be patient...\n" ) try: api_push.push(dir=dir, docker_tag=obj['docker_tag']) logger.info("Docker image pushed to ECS successfully!") except ValueError: logger.info("This is not a sagify directory: {}".format(dir)) sys.exit(-1) except subprocess.CalledProcessError as e: logger.debug(e.output) raise except Exception as e: logger.info("{}".format(e)) return
def build(obj, dir, requirements_dir): """ Command to build SageMaker app """ logger.info(ASCII_LOGO) logger.info( "Started building SageMaker Docker image. It will take some minutes...\n" ) try: api_build.build(dir=dir, requirements_dir=requirements_dir, docker_tag=obj['docker_tag']) logger.info("Docker image built successfully!") except ValueError: logger.info("This is not a sagify directory: {}".format(dir)) sys.exit(-1) except subprocess.CalledProcessError as e: logger.debug(e.output) raise except Exception as e: logger.info("{}".format(e)) return