def _template_creation(app_name, aws_profile, aws_region, python_version,
output_dir, requirements_dir):
sagify_module_name = 'sagify_base'
sagify_exists = os.path.exists(os.path.join(output_dir,
sagify_module_name))
if sagify_exists:
raise ValueError("There is a sagify directory/module already. "
"Please, rename it in order to use sagify.")
Path(output_dir).mkdir(exist_ok=True)
Path(os.path.join(output_dir, '__init__.py')).touch()
# Set 'sagify module' directory up
copy_tree(os.path.join(_FILE_DIR_PATH, '../template'), output_dir)
# Set configuration file up
config_manager = ConfigManager(os.path.join('.sagify.json'))
config = config_manager.get_config()
config.image_name = app_name
config.aws_region = aws_region
config.aws_profile = aws_profile
config.sagify_module_dir = output_dir
config.python_version = python_version
config.requirements_dir = requirements_dir
config_manager.set_config(config)
def _configure(config_dir, image_name, aws_region, aws_profile, python_version,
requirements_dir):
try:
config_manager = ConfigManager(os.path.join(config_dir,
'.sagify.json'))
config = config_manager.get_config()
if image_name is not None:
config.image_name = image_name
if aws_region is not None:
config.aws_region = aws_region
if aws_profile is not None:
config.aws_profile = aws_profile
if python_version is not None:
config.python_version = python_version
if requirements_dir is not None:
config.requirements_dir = requirements_dir
config_manager.set_config(config)
logger.info("\nConfiguration updated successfully!\n")
except ValueError as e:
logger.info("{}".format(e))
sys.exit(-1)
def tests(self):
with tempfile.TemporaryDirectory() as tmpdir:
for case in test_cases:
try:
updateConfig(tmpdir, case.image_name, case.aws_region,
case.aws_profile, case.python_version,
case.requirements_dir)
config = ConfigManager(os.path.join(
tmpdir, '.sagify.json')).get_config()
assert config.to_dict() == case.expected_config.to_dict()
except AssertionError as e:
e.args = ('Test Case: {}'.format(case.description), e.args)
raise
def test_config(tmpdir):
config_file = tmpdir.join('config.json')
config_file.write("""
{
"image_name": "keras-app-img",
"aws_profile": "sagemaker",
"aws_region": "us-east-1"
}
""")
config_manager = ConfigManager(str(config_file))
actual_config_obj = config_manager.get_config()
assert actual_config_obj.to_dict() == Config(
image_name="keras-app-img",
aws_profile="sagemaker",
aws_region="us-east-1").to_dict()
def build(obj):
"""
Command to build SageMaker app
"""
logger.info(ASCII_LOGO)
logger.info(
"Started building SageMaker Docker image. It will take some minutes...\n"
)
try:
config_file_path = os.path.join('.sagify.json')
if not os.path.isfile(config_file_path):
raise ValueError()
config = ConfigManager(config_file_path).get_config()
api_build.build(source_dir=config.sagify_module_dir,
requirements_dir=config.requirements_dir,
docker_tag=obj['docker_tag'],
image_name=config.image_name,
python_version=config.python_version)
logger.info("Docker image built successfully!")
except ValueError:
logger.info("This is not a sagify directory: {}".format(dir))
sys.exit(-1)
except subprocess.CalledProcessError as e:
logger.debug(e.output)
raise
except Exception as e:
logger.info("{}".format(e))
sys.exit(-1)
def _read_config(input_dir):
config_file_path = os.path.join(input_dir, 'sagify', 'config.json')
if not os.path.isfile(config_file_path):
logger.info("This is not a sagify directory: {}".format(input_dir))
sys.exit(-1)
return ConfigManager(config_file_path).get_config()
def listen_to_streaming_inference(output_topic_name, type, iam_role_arn,
external_id):
"""
Listen to predictions in a streaming inference pipeline
"""
logger.info(ASCII_LOGO)
logger.info("Experimental Functionality!\n")
logger.info(
"Started listening to predictions in this streaming inference pipeline ...\n"
)
try:
config_file_path = os.path.join('.sagify.json')
if not os.path.isfile(config_file_path):
raise ValueError()
config = ConfigManager(config_file_path).get_config()
for _prediction in api_cloud.listen_to_streaming_inference(
dir=config.sagify_module_dir,
output_topic_name=output_topic_name,
type=type,
aws_role=iam_role_arn,
external_id=external_id):
logger.info(_prediction)
except ValueError:
logger.info("This is not a sagify directory: {}".format(dir))
sys.exit(-1)
except Exception as e:
logger.info("{}".format(e))
sys.exit(-1)
def send_to_streaming_inference(input_features_file, input_topic_name, type,
iam_role_arn, external_id):
"""
Send features to a streaming inference pipeline
"""
logger.info(ASCII_LOGO)
logger.info("Experimental Functionality!\n")
logger.info(
"Started sending features to streaming inference pipeline ...\n")
try:
config_file_path = os.path.join('.sagify.json')
if not os.path.isfile(config_file_path):
raise ValueError()
config = ConfigManager(config_file_path).get_config()
api_cloud.send_to_streaming_inference(
dir=config.sagify_module_dir,
input_features_file=input_features_file,
input_topic_name=input_topic_name,
type=type,
aws_role=iam_role_arn,
external_id=external_id)
logger.info("Features have been sent!")
except ValueError:
logger.info("This is not a sagify directory: {}".format(dir))
sys.exit(-1)
except Exception as e:
logger.info("{}".format(e))
sys.exit(-1)
def delete_streaming_inference(name, input_topic_name, output_topic_name, type,
iam_role_arn, external_id):
"""
Command to delete a streaming inference pipeline
"""
logger.info(ASCII_LOGO)
logger.info("Experimental Functionality!\n")
logger.info("Started deleting streaming inference pipeline ...\n")
try:
config_file_path = os.path.join('.sagify.json')
if not os.path.isfile(config_file_path):
raise ValueError()
config = ConfigManager(config_file_path).get_config()
api_cloud.delete_streaming_inference(
dir=config.sagify_module_dir,
name=name,
input_topic_name=input_topic_name,
output_topic_name=output_topic_name,
type=type,
aws_role=iam_role_arn,
external_id=external_id)
logger.info("Streaming inference pipeline has been deleted!")
except ValueError:
logger.info("This is not a sagify directory: {}".format(dir))
sys.exit(-1)
except Exception as e:
logger.info("{}".format(e))
sys.exit(-1)
def _read_config(input_dir):
config_file_path = os.path.join('.sagify.json')
if not os.path.isfile(config_file_path):
raise ValueError(
"This is not a sagify directory: {}".format(input_dir))
return ConfigManager(config_file_path).get_config()
def test_config(tmpdir):
config_file = tmpdir.join('config.json')
config_file.write("""
{
"image_name": "keras-app-img",
"aws_profile": "sagemaker",
"aws_region": "us-east-1",
"python_version": "3.6",
"sagify_module_dir": "keras-app-img",
"requirements_dir": "requirements.txt"
}
""")
config_manager = ConfigManager(str(config_file))
actual_config_obj = config_manager.get_config()
assert actual_config_obj.to_dict() == Config(
image_name="keras-app-img",
aws_profile="sagemaker",
aws_region="us-east-1",
python_version="3.6",
sagify_module_dir="keras-app-img",
requirements_dir="requirements.txt").to_dict()
def push(obj, aws_region, iam_role_arn, aws_profile, external_id):
"""
Command to push Docker image to AWS ECS
"""
logger.info(ASCII_LOGO)
if iam_role_arn is not None and aws_profile is not None:
logger.error('Only one of iam-role-arn and aws-profile can be used.')
sys.exit(2)
if iam_role_arn is not None:
aws_profile = ''
try:
config_file_path = os.path.join('.sagify.json')
if not os.path.isfile(config_file_path):
raise ValueError()
config = ConfigManager(config_file_path).get_config()
image_name = config.image_name
aws_region = config.aws_region if aws_region is None else aws_region
aws_profile = config.aws_profile if (
aws_profile is None and iam_role_arn is None) else aws_profile
external_id = "" if external_id is None else external_id
iam_role_arn = "" if iam_role_arn is None else iam_role_arn
logger.info(
"Started pushing Docker image to AWS ECS. It will take some time. Please, be patient...\n"
)
api_push.push(dir=config.sagify_module_dir,
docker_tag=obj['docker_tag'],
aws_region=aws_region,
iam_role_arn=iam_role_arn,
aws_profile=aws_profile,
external_id=external_id,
image_name=image_name)
logger.info("Docker image pushed to ECS successfully!")
except ValueError:
logger.info("This is not a sagify directory: {}".format(dir))
sys.exit(-1)
except subprocess.CalledProcessError as e:
logger.debug(e.output)
raise
except Exception as e:
logger.info("{}".format(e))
return
def deploy(obj):
"""
Command to deploy ML model(s) locally
"""
logger.info(ASCII_LOGO)
logger.info("Started local deployment at localhost:8080 ...\n")
try:
config = ConfigManager(os.path.join('.sagify.json')).get_config()
api_local.deploy(dir=config.sagify_module_dir, docker_tag=obj['docker_tag'], image_name=config.image_name)
except ValueError:
logger.info("This is not a sagify directory: {}".format(dir))
sys.exit(-1)
except subprocess.CalledProcessError as e:
logger.debug(e.output)
raise
except Exception as e:
logger.info("{}".format(e))
sys.exit(-1)
def _config():
return ConfigManager('.sagify.json').get_config()