def _template_creation(app_name, aws_profile, aws_region, python_version, output_dir, requirements_dir): sagify_module_name = 'sagify_base' sagify_exists = os.path.exists(os.path.join(output_dir, sagify_module_name)) if sagify_exists: raise ValueError("There is a sagify directory/module already. " "Please, rename it in order to use sagify.") Path(output_dir).mkdir(exist_ok=True) Path(os.path.join(output_dir, '__init__.py')).touch() # Set 'sagify module' directory up copy_tree(os.path.join(_FILE_DIR_PATH, '../template'), output_dir) # Set configuration file up config_manager = ConfigManager(os.path.join('.sagify.json')) config = config_manager.get_config() config.image_name = app_name config.aws_region = aws_region config.aws_profile = aws_profile config.sagify_module_dir = output_dir config.python_version = python_version config.requirements_dir = requirements_dir config_manager.set_config(config)
def _configure(config_dir, image_name, aws_region, aws_profile, python_version, requirements_dir): try: config_manager = ConfigManager(os.path.join(config_dir, '.sagify.json')) config = config_manager.get_config() if image_name is not None: config.image_name = image_name if aws_region is not None: config.aws_region = aws_region if aws_profile is not None: config.aws_profile = aws_profile if python_version is not None: config.python_version = python_version if requirements_dir is not None: config.requirements_dir = requirements_dir config_manager.set_config(config) logger.info("\nConfiguration updated successfully!\n") except ValueError as e: logger.info("{}".format(e)) sys.exit(-1)
def tests(self): with tempfile.TemporaryDirectory() as tmpdir: for case in test_cases: try: updateConfig(tmpdir, case.image_name, case.aws_region, case.aws_profile, case.python_version, case.requirements_dir) config = ConfigManager(os.path.join( tmpdir, '.sagify.json')).get_config() assert config.to_dict() == case.expected_config.to_dict() except AssertionError as e: e.args = ('Test Case: {}'.format(case.description), e.args) raise
def test_config(tmpdir): config_file = tmpdir.join('config.json') config_file.write(""" { "image_name": "keras-app-img", "aws_profile": "sagemaker", "aws_region": "us-east-1" } """) config_manager = ConfigManager(str(config_file)) actual_config_obj = config_manager.get_config() assert actual_config_obj.to_dict() == Config( image_name="keras-app-img", aws_profile="sagemaker", aws_region="us-east-1").to_dict()
def build(obj): """ Command to build SageMaker app """ logger.info(ASCII_LOGO) logger.info( "Started building SageMaker Docker image. It will take some minutes...\n" ) try: config_file_path = os.path.join('.sagify.json') if not os.path.isfile(config_file_path): raise ValueError() config = ConfigManager(config_file_path).get_config() api_build.build(source_dir=config.sagify_module_dir, requirements_dir=config.requirements_dir, docker_tag=obj['docker_tag'], image_name=config.image_name, python_version=config.python_version) logger.info("Docker image built successfully!") except ValueError: logger.info("This is not a sagify directory: {}".format(dir)) sys.exit(-1) except subprocess.CalledProcessError as e: logger.debug(e.output) raise except Exception as e: logger.info("{}".format(e)) sys.exit(-1)
def _read_config(input_dir): config_file_path = os.path.join(input_dir, 'sagify', 'config.json') if not os.path.isfile(config_file_path): logger.info("This is not a sagify directory: {}".format(input_dir)) sys.exit(-1) return ConfigManager(config_file_path).get_config()
def listen_to_streaming_inference(output_topic_name, type, iam_role_arn, external_id): """ Listen to predictions in a streaming inference pipeline """ logger.info(ASCII_LOGO) logger.info("Experimental Functionality!\n") logger.info( "Started listening to predictions in this streaming inference pipeline ...\n" ) try: config_file_path = os.path.join('.sagify.json') if not os.path.isfile(config_file_path): raise ValueError() config = ConfigManager(config_file_path).get_config() for _prediction in api_cloud.listen_to_streaming_inference( dir=config.sagify_module_dir, output_topic_name=output_topic_name, type=type, aws_role=iam_role_arn, external_id=external_id): logger.info(_prediction) except ValueError: logger.info("This is not a sagify directory: {}".format(dir)) sys.exit(-1) except Exception as e: logger.info("{}".format(e)) sys.exit(-1)
def send_to_streaming_inference(input_features_file, input_topic_name, type, iam_role_arn, external_id): """ Send features to a streaming inference pipeline """ logger.info(ASCII_LOGO) logger.info("Experimental Functionality!\n") logger.info( "Started sending features to streaming inference pipeline ...\n") try: config_file_path = os.path.join('.sagify.json') if not os.path.isfile(config_file_path): raise ValueError() config = ConfigManager(config_file_path).get_config() api_cloud.send_to_streaming_inference( dir=config.sagify_module_dir, input_features_file=input_features_file, input_topic_name=input_topic_name, type=type, aws_role=iam_role_arn, external_id=external_id) logger.info("Features have been sent!") except ValueError: logger.info("This is not a sagify directory: {}".format(dir)) sys.exit(-1) except Exception as e: logger.info("{}".format(e)) sys.exit(-1)
def delete_streaming_inference(name, input_topic_name, output_topic_name, type, iam_role_arn, external_id): """ Command to delete a streaming inference pipeline """ logger.info(ASCII_LOGO) logger.info("Experimental Functionality!\n") logger.info("Started deleting streaming inference pipeline ...\n") try: config_file_path = os.path.join('.sagify.json') if not os.path.isfile(config_file_path): raise ValueError() config = ConfigManager(config_file_path).get_config() api_cloud.delete_streaming_inference( dir=config.sagify_module_dir, name=name, input_topic_name=input_topic_name, output_topic_name=output_topic_name, type=type, aws_role=iam_role_arn, external_id=external_id) logger.info("Streaming inference pipeline has been deleted!") except ValueError: logger.info("This is not a sagify directory: {}".format(dir)) sys.exit(-1) except Exception as e: logger.info("{}".format(e)) sys.exit(-1)
def _read_config(input_dir): config_file_path = os.path.join('.sagify.json') if not os.path.isfile(config_file_path): raise ValueError( "This is not a sagify directory: {}".format(input_dir)) return ConfigManager(config_file_path).get_config()
def test_config(tmpdir): config_file = tmpdir.join('config.json') config_file.write(""" { "image_name": "keras-app-img", "aws_profile": "sagemaker", "aws_region": "us-east-1", "python_version": "3.6", "sagify_module_dir": "keras-app-img", "requirements_dir": "requirements.txt" } """) config_manager = ConfigManager(str(config_file)) actual_config_obj = config_manager.get_config() assert actual_config_obj.to_dict() == Config( image_name="keras-app-img", aws_profile="sagemaker", aws_region="us-east-1", python_version="3.6", sagify_module_dir="keras-app-img", requirements_dir="requirements.txt").to_dict()
def push(obj, aws_region, iam_role_arn, aws_profile, external_id): """ Command to push Docker image to AWS ECS """ logger.info(ASCII_LOGO) if iam_role_arn is not None and aws_profile is not None: logger.error('Only one of iam-role-arn and aws-profile can be used.') sys.exit(2) if iam_role_arn is not None: aws_profile = '' try: config_file_path = os.path.join('.sagify.json') if not os.path.isfile(config_file_path): raise ValueError() config = ConfigManager(config_file_path).get_config() image_name = config.image_name aws_region = config.aws_region if aws_region is None else aws_region aws_profile = config.aws_profile if ( aws_profile is None and iam_role_arn is None) else aws_profile external_id = "" if external_id is None else external_id iam_role_arn = "" if iam_role_arn is None else iam_role_arn logger.info( "Started pushing Docker image to AWS ECS. It will take some time. Please, be patient...\n" ) api_push.push(dir=config.sagify_module_dir, docker_tag=obj['docker_tag'], aws_region=aws_region, iam_role_arn=iam_role_arn, aws_profile=aws_profile, external_id=external_id, image_name=image_name) logger.info("Docker image pushed to ECS successfully!") except ValueError: logger.info("This is not a sagify directory: {}".format(dir)) sys.exit(-1) except subprocess.CalledProcessError as e: logger.debug(e.output) raise except Exception as e: logger.info("{}".format(e)) return
def deploy(obj): """ Command to deploy ML model(s) locally """ logger.info(ASCII_LOGO) logger.info("Started local deployment at localhost:8080 ...\n") try: config = ConfigManager(os.path.join('.sagify.json')).get_config() api_local.deploy(dir=config.sagify_module_dir, docker_tag=obj['docker_tag'], image_name=config.image_name) except ValueError: logger.info("This is not a sagify directory: {}".format(dir)) sys.exit(-1) except subprocess.CalledProcessError as e: logger.debug(e.output) raise except Exception as e: logger.info("{}".format(e)) sys.exit(-1)
def _config(): return ConfigManager('.sagify.json').get_config()