def build(obj):
"""
Command to build SageMaker app
"""
logger.info(ASCII_LOGO)
logger.info(
"Started building SageMaker Docker image. It will take some minutes...\n"
)
try:
config_file_path = os.path.join('.sagify.json')
if not os.path.isfile(config_file_path):
raise ValueError()
config = ConfigManager(config_file_path).get_config()
api_build.build(source_dir=config.sagify_module_dir,
requirements_dir=config.requirements_dir,
docker_tag=obj['docker_tag'],
image_name=config.image_name,
python_version=config.python_version)
logger.info("Docker image built successfully!")
except ValueError:
logger.info("This is not a sagify directory: {}".format(dir))
sys.exit(-1)
except subprocess.CalledProcessError as e:
logger.debug(e.output)
raise
except Exception as e:
logger.info("{}".format(e))
sys.exit(-1)
def train(dir):
"""
Command to train ML model(s) locally
"""
logger.info(ASCII_LOGO)
logger.info("Started local training...\n")
sagify_module_path = os.path.join(dir, 'sagify')
local_train_script_path = os.path.join(sagify_module_path, 'local_test',
'train_local.sh')
test_path = os.path.join(sagify_module_path, 'local_test', 'test_dir')
if not os.path.isdir(test_path):
logger.info("This is not a sagify directory: {}".format(dir))
sys.exit(-1)
try:
subprocess.check_output([
"{}".format(local_train_script_path),
"{}".format(os.path.abspath(test_path))
])
logger.info("Local training completed successfully!")
except Exception as e:
logger.info("{}".format(e))
return
def _configure(config_dir, image_name, aws_region, aws_profile, python_version,
requirements_dir):
try:
config_manager = ConfigManager(os.path.join(config_dir,
'.sagify.json'))
config = config_manager.get_config()
if image_name is not None:
config.image_name = image_name
if aws_region is not None:
config.aws_region = aws_region
if aws_profile is not None:
config.aws_profile = aws_profile
if python_version is not None:
config.python_version = python_version
if requirements_dir is not None:
config.requirements_dir = requirements_dir
config_manager.set_config(config)
logger.info("\nConfiguration updated successfully!\n")
except ValueError as e:
logger.info("{}".format(e))
sys.exit(-1)
def train(obj, input_s3_dir, output_s3_dir, hyperparams_file, ec2_type,
volume_size, time_out, aws_tags, iam_role_arn, external_id,
base_job_name, job_name, metric_names):
"""
Command to train ML model(s) on SageMaker
"""
logger.info(ASCII_LOGO)
logger.info("Started training on SageMaker...\n")
try:
s3_model_location = api_cloud.train(
dir=_config().sagify_module_dir,
input_s3_dir=input_s3_dir,
output_s3_dir=output_s3_dir,
hyperparams_file=hyperparams_file,
ec2_type=ec2_type,
volume_size=volume_size,
time_out=time_out,
docker_tag=obj['docker_tag'],
tags=aws_tags,
aws_role=iam_role_arn,
external_id=external_id,
base_job_name=base_job_name,
job_name=job_name,
metric_names=[_val.strip() for _val in metric_names.split(',')]
if metric_names else None)
logger.info("Training on SageMaker succeeded")
logger.info("Model S3 location: {}".format(s3_model_location))
except ValueError as e:
logger.info("{}".format(e))
sys.exit(-1)
def batch_transform(obj, s3_model_location, s3_input_location,
s3_output_location, num_instances, ec2_type, aws_tags,
iam_role_arn, external_id, wait, job_name):
"""
Command to execute a batch transform job given a trained ML model on SageMaker
"""
logger.info(ASCII_LOGO)
logger.info("Started configuration of batch transform on SageMaker ...\n")
try:
status = api_cloud.batch_transform(
dir=_config().sagify_module_dir,
s3_model_location=s3_model_location,
s3_input_location=s3_input_location,
s3_output_location=s3_output_location,
num_instances=num_instances,
ec2_type=ec2_type,
docker_tag=obj['docker_tag'],
aws_role=iam_role_arn,
external_id=external_id,
tags=aws_tags,
wait=wait,
job_name=job_name)
if wait:
logger.info(
"Batch transform on SageMaker finished with status: {}".format(
status))
else:
logger.info("Started batch transform on SageMaker successfully")
except ValueError as e:
logger.info("{}".format(e))
sys.exit(-1)
def train(obj, dir, job_name, input_s3_dir, output_s3_dir, hyperparams_file,
ec2_type, volume_size, time_out, aws_tags):
"""
Command to train ML model(s) on SageMaker
"""
logger.info(ASCII_LOGO)
logger.info("Started training on SageMaker...\n")
try:
s3_model_location = api_cloud.train(dir=dir,
job_name=job_name,
input_s3_dir=input_s3_dir,
output_s3_dir=output_s3_dir,
hyperparams_file=hyperparams_file,
ec2_type=ec2_type,
volume_size=volume_size,
time_out=time_out,
docker_tag=obj['docker_tag'],
tags=aws_tags)
logger.info("Training on SageMaker succeeded")
logger.info("Model S3 location: {}".format(s3_model_location))
except ValueError as e:
logger.info("{}".format(e))
sys.exit(-1)
def _read_config(input_dir):
config_file_path = os.path.join(input_dir, 'sagify', 'config.json')
if not os.path.isfile(config_file_path):
logger.info("This is not a sagify directory: {}".format(input_dir))
sys.exit(-1)
return ConfigManager(config_file_path).get_config()
def push(obj, dir, aws_region, iam_role_arn, aws_profile, external_id):
"""
Command to push Docker image to AWS ECS
"""
logger.info(ASCII_LOGO)
logger.info(
"Started pushing Docker image to AWS ECS. It will take some time. Please, be patient...\n"
)
if iam_role_arn is not None and aws_profile is not None:
logger.error('Only one of iam-role-arn and aws-profile can be used.')
sys.exit(2)
try:
api_push.push(dir=dir,
docker_tag=obj['docker_tag'],
aws_region=aws_region,
iam_role_arn=iam_role_arn,
aws_profile=aws_profile,
external_id=external_id)
logger.info("Docker image pushed to ECS successfully!")
except ValueError:
logger.info("This is not a sagify directory: {}".format(dir))
sys.exit(-1)
except subprocess.CalledProcessError as e:
logger.debug(e.output)
raise
except Exception as e:
logger.info("{}".format(e))
return
def __init__(self,
aws_profile,
aws_region,
aws_role=None,
external_id=None):
if aws_role and external_id:
logger.info(
"An IAM role and corresponding external id were provided. Attempting to assume that role..."
)
sts_client = boto3.client('sts')
assumedRoleObject = sts_client.assume_role(
RoleArn=aws_role,
RoleSessionName="SagifySession",
ExternalId=external_id)
credentials = assumedRoleObject['Credentials']
self.boto_session = boto3.Session(
aws_access_key_id=credentials['AccessKeyId'],
aws_secret_access_key=credentials['SecretAccessKey'],
aws_session_token=credentials['SessionToken'],
region_name=aws_region)
else:
logger.info(
"No IAM role provided. Using profile {} instead.".format(
aws_profile))
self.boto_session = boto3.Session(profile_name=aws_profile,
region_name=aws_region)
self.sagemaker_session = sage.Session(boto_session=self.boto_session)
self.role = sage.get_execution_role(
self.sagemaker_session) if aws_role is None else aws_role
def deploy(
obj,
s3_model_location,
num_instances,
ec2_type,
aws_tags,
iam_role_arn,
external_id,
endpoint_name
):
"""
Command to deploy ML model(s) on SageMaker
"""
logger.info(ASCII_LOGO)
logger.info("Started deployment on SageMaker ...\n")
try:
endpoint_name = api_cloud.deploy(
dir=_config().sagify_module_dir,
s3_model_location=s3_model_location,
num_instances=num_instances,
ec2_type=ec2_type,
docker_tag=obj['docker_tag'],
aws_role=iam_role_arn,
external_id=external_id,
tags=aws_tags,
endpoint_name=endpoint_name
)
logger.info("Model deployed to SageMaker successfully")
logger.info("Endpoint name: {}".format(endpoint_name))
except ValueError as e:
logger.info("{}".format(e))
sys.exit(-1)
def lightning_deploy(framework, s3_model_location, num_instances, ec2_type,
model_server_workers, aws_tags, aws_profile, aws_region,
iam_role_arn, external_id, endpoint_name,
extra_config_file):
"""
Command for lightning deployment of ML model(s) on SageMaker without code
"""
logger.info(ASCII_LOGO)
logger.info("Started lightning deployment on SageMaker ...\n")
try:
endpoint_name = api_cloud.lightning_deploy(
framework=framework,
s3_model_location=s3_model_location,
num_instances=num_instances,
ec2_type=ec2_type,
aws_region=aws_region,
model_server_workers=model_server_workers,
aws_profile=aws_profile,
aws_role=iam_role_arn,
external_id=external_id,
tags=aws_tags,
endpoint_name=endpoint_name,
extra_config_file=extra_config_file)
logger.info("Model deployed to SageMaker successfully")
logger.info("Endpoint name: {}".format(endpoint_name))
except ValueError as e:
logger.info("{}".format(e))
sys.exit(-1)
def configure(image_name, aws_region, aws_profile, python_version,
requirements_dir):
"""
Command to configure SageMaker template
"""
logger.info(ASCII_LOGO)
_configure('.', image_name, aws_region, aws_profile, python_version,
requirements_dir)
def _read_hyperparams_config(hyperparams_file_path):
if not os.path.isfile(hyperparams_file_path):
logger.info("The given hyperparams file {} doens't exist".format(
hyperparams_file_path))
sys.exit(-1)
with open(hyperparams_file_path) as _in_file:
return json.load(_in_file)
def train(dir, input_s3_dir, output_s3_dir, hyperparams_file, ec2_type,
volume_size, time_out):
"""
Command to train ML model(s) on SageMaker
"""
logger.info(ASCII_LOGO)
logger.info("Started training on SageMaker...\n")
config = _read_config(dir)
hyperparams_dict = _read_hyperparams_config(
hyperparams_file) if hyperparams_file else None
sage_maker_client = sagemaker.SageMakerClient(config.aws_profile,
config.aws_region)
s3_model_location = sage_maker_client.train(
image_name=config.image_name,
input_s3_data_location=input_s3_dir,
train_instance_count=1,
train_instance_type=ec2_type,
train_volume_size=volume_size,
train_max_run=time_out,
output_path=output_s3_dir,
hyperparameters=hyperparams_dict)
logger.info("Training on SageMaker succeeded")
logger.info("Model S3 location: {}".format(s3_model_location))
def push(obj, aws_region, iam_role_arn, aws_profile, external_id):
"""
Command to push Docker image to AWS ECS
"""
logger.info(ASCII_LOGO)
if iam_role_arn is not None and aws_profile is not None:
logger.error('Only one of iam-role-arn and aws-profile can be used.')
sys.exit(2)
if iam_role_arn is not None:
aws_profile = ''
try:
config_file_path = os.path.join('.sagify.json')
if not os.path.isfile(config_file_path):
raise ValueError()
config = ConfigManager(config_file_path).get_config()
image_name = config.image_name
aws_region = config.aws_region if aws_region is None else aws_region
aws_profile = config.aws_profile if (
aws_profile is None and iam_role_arn is None) else aws_profile
external_id = "" if external_id is None else external_id
iam_role_arn = "" if iam_role_arn is None else iam_role_arn
logger.info(
"Started pushing Docker image to AWS ECS. It will take some time. Please, be patient...\n"
)
api_push.push(dir=config.sagify_module_dir,
docker_tag=obj['docker_tag'],
aws_region=aws_region,
iam_role_arn=iam_role_arn,
aws_profile=aws_profile,
external_id=external_id,
image_name=image_name)
logger.info("Docker image pushed to ECS successfully!")
except ValueError:
logger.info("This is not a sagify directory: {}".format(dir))
sys.exit(-1)
except subprocess.CalledProcessError as e:
logger.debug(e.output)
raise
except Exception as e:
logger.info("{}".format(e))
return
def ask_for_python_version():
logger.info("Select Python interpreter:")
logger.info('{}'.format('\n'.join(['1 - Python3', '2 - Python2'])))
def _validate_python_option(input_value):
if int(input_value) not in {1, 2}:
raise BadParameter(
message="invalid choice: {}. (choose from 1, 2)".format(
str(input_value)))
return int(input_value)
chosen_python_index = click.prompt(
text="Choose from 1, 2",
default=1,
value_proc=lambda x: _validate_python_option(x))
return '3.6' if chosen_python_index == 1 else '2.7'
def init(dir):
"""
Command to initialize SageMaker template
"""
logger.info(ASCII_LOGO)
sagify_app_name = ask_for_app_name()
python_version = ask_for_python_version()
aws_profile, aws_region = ask_for_aws_details()
template_creation(app_name=sagify_app_name,
aws_profile=aws_profile,
aws_region=aws_region,
python_version=python_version,
output_dir=dir)
logger.info("\nsagify module is created! ヽ(´▽`)/")
def train(obj, dir):
"""
Command to train ML model(s) locally
"""
logger.info(ASCII_LOGO)
logger.info("Started local training...\n")
try:
api_local.train(dir=dir, docker_tag=obj['docker_tag'])
logger.info("Local training completed successfully!")
except ValueError:
logger.info("This is not a sagify directory: {}".format(dir))
sys.exit(-1)
except subprocess.CalledProcessError as e:
logger.debug(e.output)
raise
except Exception as e:
logger.info("{}".format(e))
return
def build(dir, requirements_dir):
"""
Command to build SageMaker app
"""
logger.info(ASCII_LOGO)
logger.info(
"Started building SageMaker Docker image. It will take some minutes...\n"
)
sagify_module_path = os.path.relpath(os.path.join(dir, 'sagify/'))
build_script_path = os.path.join(sagify_module_path, 'build.sh')
dockerfile_path = os.path.join(sagify_module_path, 'Dockerfile')
train_file_path = os.path.join(sagify_module_path, 'training', 'train')
serve_file_path = os.path.join(sagify_module_path, 'prediction', 'serve')
executor_file_path = os.path.join(sagify_module_path, 'executor.sh')
if not os.path.isfile(build_script_path) or not os.path.isfile(train_file_path) or not \
os.path.isfile(serve_file_path):
logger.info("This is not a sagify directory: {}".format(dir))
sys.exit(-1)
os.chmod(train_file_path, 0o777)
os.chmod(serve_file_path, 0o777)
os.chmod(executor_file_path, 0o777)
target_dir_name = os.path.basename(os.path.normpath(dir))
try:
subprocess.check_output([
"{}".format(build_script_path), "{}".format(os.path.relpath(dir)),
"{}".format(os.path.relpath(target_dir_name)),
"{}".format(dockerfile_path),
"{}".format(os.path.relpath(requirements_dir))
])
logger.info("Docker image built successfully!")
except Exception as e:
logger.info("{}".format(e))
return
def train(obj):
"""
Command to train ML model(s) locally
"""
logger.info(ASCII_LOGO)
logger.info("Started local training...\n")
try:
config = ConfigManager(os.path.join('.sagify.json')).get_config()
api_local.train(dir=config.sagify_module_dir, docker_tag=obj['docker_tag'], image_name=config.image_name)
logger.info("Local training completed successfully!")
except ValueError:
logger.info("This is not a sagify directory: {}".format(dir))
sys.exit(-1)
except subprocess.CalledProcessError as e:
logger.debug(e.output)
raise
except Exception as e:
logger.info("{}".format(e))
sys.exit(-1)
def ask_for_aws_details():
available_profiles = _get_local_aws_profiles()
if len(available_profiles) == 0:
logger.info("aws cli is not configured!")
return
valid_positions = list(range(1, len(available_profiles) + 1))
logger.info("Select AWS profile:")
logger.info('{}'.format('\n'.join([
'{} - {}'.format(pos, profile)
for pos, profile in zip(valid_positions, available_profiles)
])))
def _validate_profile_position(input_pos):
if int(input_pos) not in valid_positions:
raise BadParameter(
message="invalid choice: {}. (choose from {})".format(
input_pos, ', '.join([str(pos)
for pos in valid_positions])))
return int(input_pos) - 1
chosen_profile_index = click.prompt(
text="Choose from {}".format(', '.join(
[str(pos) for pos in valid_positions])),
default=1,
value_proc=lambda x: _validate_profile_position(x))
chosen_profile = available_profiles[chosen_profile_index]
chosen_region = click.prompt(text="Type in your preferred AWS region name",
default='us-east-1',
type=str)
return chosen_profile, chosen_region
def init():
"""
Command to initialize SageMaker template
"""
logger.info(ASCII_LOGO)
sagify_app_name = ask_for_app_name()
is_new_project = ask_if_existing_project_exists()
root_dir = None
if not is_new_project:
root_dir = ask_for_root_dir()
python_version = ask_for_python_version()
aws_profile, aws_region = ask_for_aws_details()
requirements_dir = ask_for_requirements_dir()
try:
api_initialize.init(
sagify_app_name=sagify_app_name,
aws_profile=aws_profile,
aws_region=aws_region,
python_version=python_version,
root_dir=root_dir if root_dir else 'src',
requirements_dir=requirements_dir
)
logger.info("\nsagify module is created! ヽ(´▽`)/")
except ValueError as e:
logger.info("{}".format(e))
sys.exit(-1)
def push(obj, dir):
"""
Command to push Docker image to AWS ECS
"""
logger.info(ASCII_LOGO)
logger.info(
"Started pushing Docker image to AWS ECS. It will take some time. Please, be patient...\n"
)
try:
api_push.push(dir=dir, docker_tag=obj['docker_tag'])
logger.info("Docker image pushed to ECS successfully!")
except ValueError:
logger.info("This is not a sagify directory: {}".format(dir))
sys.exit(-1)
except subprocess.CalledProcessError as e:
logger.debug(e.output)
raise
except Exception as e:
logger.info("{}".format(e))
return
def deploy(obj, dir, s3_model_location, model_name, num_instances, ec2_type,
aws_tags, vpc_configs):
"""
Command to deploy ML model(s) on SageMaker
"""
logger.info(ASCII_LOGO)
logger.info("Started deployment on SageMaker ...\n")
try:
endpoint_name = api_cloud.deploy(dir=dir,
s3_model_location=s3_model_location,
model_name=model_name,
vpc_configs=vpc_configs,
num_instances=num_instances,
ec2_type=ec2_type,
docker_tag=obj['docker_tag'],
tags=aws_tags)
logger.info("Model deployed to SageMaker successfully")
logger.info("Endpoint name: {}".format(endpoint_name))
except ValueError as e:
logger.info("{}".format(e))
sys.exit(-1)
def upload_data(dir, input_dir, s3_dir):
"""
Command to upload data to S3
"""
logger.info(ASCII_LOGO)
logger.info("Started uploading data to S3...\n")
try:
s3_path = api_cloud.upload_data(dir=dir,
input_dir=input_dir,
s3_dir=s3_dir)
logger.info("Data uploaded to {} successfully".format(s3_path))
except ValueError as e:
logger.info("{}".format(e))
sys.exit(-1)
def push(dir):
"""
Command to push Docker image to AWS ECS
"""
logger.info(ASCII_LOGO)
logger.info(
"Started pushing Docker image to AWS ECS. It will take some time. Please, be patient...\n"
)
sagify_module_path = os.path.relpath(os.path.join(dir, 'sagify/'))
push_script_path = os.path.join(sagify_module_path, 'push.sh')
if not os.path.isfile(push_script_path):
logger.info("This is not a sagify directory: {}".format(dir))
sys.exit(-1)
try:
subprocess.check_output(["{}".format(push_script_path)])
logger.info("Docker image pushed to ECS successfully!")
except Exception as e:
logger.info("{}".format(e))
return
def build(obj, dir, requirements_dir):
"""
Command to build SageMaker app
"""
logger.info(ASCII_LOGO)
logger.info(
"Started building SageMaker Docker image. It will take some minutes...\n"
)
try:
api_build.build(dir=dir,
requirements_dir=requirements_dir,
docker_tag=obj['docker_tag'])
logger.info("Docker image built successfully!")
except ValueError:
logger.info("This is not a sagify directory: {}".format(dir))
sys.exit(-1)
except subprocess.CalledProcessError as e:
logger.debug(e.output)
raise
except Exception as e:
logger.info("{}".format(e))
return
def template_creation(app_name, aws_profile, aws_region, python_version,
output_dir):
sagify_module_name = 'sagify'
sagify_exists = os.path.exists(os.path.join(output_dir,
sagify_module_name))
if sagify_exists:
logger.info("There is a sagify directory/module already. "
"Please, rename it in order to use sagify.")
sys.exit(-1)
Path(output_dir).mkdir(exist_ok=True)
Path(os.path.join(output_dir, '__init__.py')).touch()
cookiecutter(template=os.path.join(_FILE_DIR_PATH, '../template/'),
output_dir=output_dir,
no_input=True,
extra_context={
"project_slug": app_name,
"module_slug": sagify_module_name,
"aws_profile": aws_profile,
"aws_region": aws_region,
"python_version": python_version
})
def deploy(obj, dir):
"""
Command to deploy ML model(s) locally
"""
logger.info(ASCII_LOGO)
logger.info("Started local deployment at localhost:8080 ...\n")
try:
api_local.deploy(dir=dir, docker_tag=obj['docker_tag'])
except ValueError:
logger.info("This is not a sagify directory: {}".format(dir))
sys.exit(-1)
except subprocess.CalledProcessError as e:
logger.debug(e.output)
raise
except Exception as e:
logger.info("{}".format(e))
return
def deploy(dir, s3_model_location, num_instances, ec2_type):
"""
Command to deploy ML model(s) on SageMaker
"""
logger.info(ASCII_LOGO)
logger.info("Started deployment on SageMaker ...\n")
config = _read_config(dir)
sage_maker_client = sagemaker.SageMakerClient(config.aws_profile,
config.aws_region)
endpoint_name = sage_maker_client.deploy(
image_name=config.image_name,
s3_model_location=s3_model_location,
train_instance_count=num_instances,
train_instance_type=ec2_type)
logger.info("Model deployed to SageMaker successfully")
logger.info("Endpoint name: {}".format(endpoint_name))