def build(dir, requirements_dir, docker_tag):
"""
Builds a Docker image that contains code under the given source root directory.
Assumes that Docker is installed and running locally.
:param dir: [str], source root directory
:param requirements_dir: [str], path to requirements.txt
"""
sagify_module_path = os.path.relpath(os.path.join(dir, 'sagify/'))
build_script_path = os.path.join(sagify_module_path, 'build.sh')
dockerfile_path = os.path.join(sagify_module_path, 'Dockerfile')
train_file_path = os.path.join(sagify_module_path, 'training', 'train')
serve_file_path = os.path.join(sagify_module_path, 'prediction', 'serve')
executor_file_path = os.path.join(sagify_module_path, 'executor.sh')
if not os.path.isfile(build_script_path) or not os.path.isfile(train_file_path) or not \
os.path.isfile(serve_file_path):
raise ValueError("This is not a sagify directory: {}".format(dir))
os.chmod(train_file_path, 0o777)
os.chmod(serve_file_path, 0o777)
os.chmod(executor_file_path, 0o777)
target_dir_name = os.path.basename(os.path.normpath(dir))
output = subprocess.check_output([
"{}".format(build_script_path), "{}".format(os.path.relpath(dir)),
"{}".format(os.path.relpath(target_dir_name)),
"{}".format(dockerfile_path),
"{}".format(os.path.relpath(requirements_dir)), docker_tag
])
logger.debug(output)
def build(obj):
"""
Command to build SageMaker app
"""
logger.info(ASCII_LOGO)
logger.info(
"Started building SageMaker Docker image. It will take some minutes...\n"
)
try:
config_file_path = os.path.join('.sagify.json')
if not os.path.isfile(config_file_path):
raise ValueError()
config = ConfigManager(config_file_path).get_config()
api_build.build(source_dir=config.sagify_module_dir,
requirements_dir=config.requirements_dir,
docker_tag=obj['docker_tag'],
image_name=config.image_name,
python_version=config.python_version)
logger.info("Docker image built successfully!")
except ValueError:
logger.info("This is not a sagify directory: {}".format(dir))
sys.exit(-1)
except subprocess.CalledProcessError as e:
logger.debug(e.output)
raise
except Exception as e:
logger.info("{}".format(e))
sys.exit(-1)
def push(dir, docker_tag, aws_region, iam_role_arn, aws_profile, external_id,
image_name):
"""
Push Docker image to AWS ECS
:param dir: [str], source root directory
:param docker_tag: [str], the Docker tag for the image
:param aws_region: [str], the AWS region to push the image to
:param iam_role_arn: [str], the AWS role used to push the image to ECR
:param aws_profile: [str], the AWS profile used to push the image to ECR
:param external_id: [str], Optional external id used when using an IAM role
:param image_name: [str], The name of the Docker image
"""
sagify_module_path = os.path.relpath(os.path.join(dir, 'sagify/'))
push_script_path = os.path.join(sagify_module_path, 'push.sh')
if not os.path.isfile(push_script_path):
raise ValueError("This is not a sagify directory: {}".format(dir))
output = subprocess.check_output([
"{}".format(push_script_path), docker_tag, aws_region, iam_role_arn,
aws_profile, external_id, image_name
])
logger.debug(output)
def push(obj, dir, aws_region, iam_role_arn, aws_profile, external_id):
"""
Command to push Docker image to AWS ECS
"""
logger.info(ASCII_LOGO)
logger.info(
"Started pushing Docker image to AWS ECS. It will take some time. Please, be patient...\n"
)
if iam_role_arn is not None and aws_profile is not None:
logger.error('Only one of iam-role-arn and aws-profile can be used.')
sys.exit(2)
try:
api_push.push(dir=dir,
docker_tag=obj['docker_tag'],
aws_region=aws_region,
iam_role_arn=iam_role_arn,
aws_profile=aws_profile,
external_id=external_id)
logger.info("Docker image pushed to ECS successfully!")
except ValueError:
logger.info("This is not a sagify directory: {}".format(dir))
sys.exit(-1)
except subprocess.CalledProcessError as e:
logger.debug(e.output)
raise
except Exception as e:
logger.info("{}".format(e))
return
def push(dir, docker_tag):
"""
Push Docker image to AWS ECS
:param dir: [str], source root directory
"""
sagify_module_path = os.path.relpath(os.path.join(dir, 'sagify/'))
push_script_path = os.path.join(sagify_module_path, 'push.sh')
if not os.path.isfile(push_script_path):
raise ValueError("This is not a sagify directory: {}".format(dir))
output = subprocess.check_output(["{}".format(push_script_path), docker_tag])
logger.debug(output)
def push(obj, aws_region, iam_role_arn, aws_profile, external_id):
"""
Command to push Docker image to AWS ECS
"""
logger.info(ASCII_LOGO)
if iam_role_arn is not None and aws_profile is not None:
logger.error('Only one of iam-role-arn and aws-profile can be used.')
sys.exit(2)
if iam_role_arn is not None:
aws_profile = ''
try:
config_file_path = os.path.join('.sagify.json')
if not os.path.isfile(config_file_path):
raise ValueError()
config = ConfigManager(config_file_path).get_config()
image_name = config.image_name
aws_region = config.aws_region if aws_region is None else aws_region
aws_profile = config.aws_profile if (
aws_profile is None and iam_role_arn is None) else aws_profile
external_id = "" if external_id is None else external_id
iam_role_arn = "" if iam_role_arn is None else iam_role_arn
logger.info(
"Started pushing Docker image to AWS ECS. It will take some time. Please, be patient...\n"
)
api_push.push(dir=config.sagify_module_dir,
docker_tag=obj['docker_tag'],
aws_region=aws_region,
iam_role_arn=iam_role_arn,
aws_profile=aws_profile,
external_id=external_id,
image_name=image_name)
logger.info("Docker image pushed to ECS successfully!")
except ValueError:
logger.info("This is not a sagify directory: {}".format(dir))
sys.exit(-1)
except subprocess.CalledProcessError as e:
logger.debug(e.output)
raise
except Exception as e:
logger.info("{}".format(e))
return
def deploy(obj, dir):
"""
Command to deploy ML model(s) locally
"""
logger.info(ASCII_LOGO)
logger.info("Started local deployment at localhost:8080 ...\n")
try:
api_local.deploy(dir=dir, docker_tag=obj['docker_tag'])
except ValueError:
logger.info("This is not a sagify directory: {}".format(dir))
sys.exit(-1)
except subprocess.CalledProcessError as e:
logger.debug(e.output)
raise
except Exception as e:
logger.info("{}".format(e))
return
def deploy(obj):
"""
Command to deploy ML model(s) locally
"""
logger.info(ASCII_LOGO)
logger.info("Started local deployment at localhost:8080 ...\n")
try:
config = ConfigManager(os.path.join('.sagify.json')).get_config()
api_local.deploy(dir=config.sagify_module_dir, docker_tag=obj['docker_tag'], image_name=config.image_name)
except ValueError:
logger.info("This is not a sagify directory: {}".format(dir))
sys.exit(-1)
except subprocess.CalledProcessError as e:
logger.debug(e.output)
raise
except Exception as e:
logger.info("{}".format(e))
sys.exit(-1)
def train(obj, dir):
"""
Command to train ML model(s) locally
"""
logger.info(ASCII_LOGO)
logger.info("Started local training...\n")
try:
api_local.train(dir=dir, docker_tag=obj['docker_tag'])
logger.info("Local training completed successfully!")
except ValueError:
logger.info("This is not a sagify directory: {}".format(dir))
sys.exit(-1)
except subprocess.CalledProcessError as e:
logger.debug(e.output)
raise
except Exception as e:
logger.info("{}".format(e))
return
def deploy(dir, docker_tag):
"""
Deploys ML models(s) locally
:param dir: [str], source root directory
:param docker_tag: [str], the Docker tag for the image
"""
sagify_module_path = os.path.join(dir, 'sagify')
local_deploy_script_path = os.path.join(sagify_module_path, 'local_test',
'deploy_local.sh')
test_path = os.path.join(sagify_module_path, 'local_test', 'test_dir')
if not os.path.isdir(test_path):
raise ValueError("This is not a sagify directory: {}".format(dir))
output = subprocess.check_output([
"{}".format(local_deploy_script_path),
"{}".format(os.path.abspath(test_path)), docker_tag
])
logger.debug(output)
def train(dir, docker_tag, image_name):
"""
Trains ML model(s) locally
:param dir: [str], source root directory
:param docker_tag: [str], the Docker tag for the image
:param image_name: [str], The name of the Docker image
"""
sagify_module_path = os.path.join(dir, 'sagify')
local_train_script_path = os.path.join(sagify_module_path, 'local_test',
'train_local.sh')
test_path = os.path.join(sagify_module_path, 'local_test', 'test_dir')
if not os.path.isdir(test_path):
raise ValueError("This is not a sagify directory: {}".format(dir))
output = subprocess.check_output([
"{}".format(local_train_script_path),
"{}".format(os.path.abspath(test_path)), docker_tag, image_name
])
logger.debug(output)
def push(obj, dir):
"""
Command to push Docker image to AWS ECS
"""
logger.info(ASCII_LOGO)
logger.info(
"Started pushing Docker image to AWS ECS. It will take some time. Please, be patient...\n"
)
try:
api_push.push(dir=dir, docker_tag=obj['docker_tag'])
logger.info("Docker image pushed to ECS successfully!")
except ValueError:
logger.info("This is not a sagify directory: {}".format(dir))
sys.exit(-1)
except subprocess.CalledProcessError as e:
logger.debug(e.output)
raise
except Exception as e:
logger.info("{}".format(e))
return
def build(obj, dir, requirements_dir):
"""
Command to build SageMaker app
"""
logger.info(ASCII_LOGO)
logger.info(
"Started building SageMaker Docker image. It will take some minutes...\n"
)
try:
api_build.build(dir=dir,
requirements_dir=requirements_dir,
docker_tag=obj['docker_tag'])
logger.info("Docker image built successfully!")
except ValueError:
logger.info("This is not a sagify directory: {}".format(dir))
sys.exit(-1)
except subprocess.CalledProcessError as e:
logger.debug(e.output)
raise
except Exception as e:
logger.info("{}".format(e))
return