def test_tf(m_tar, e_tar, time, strftime, sagemaker_session, tf_version): tf = TensorFlow(entry_point=SCRIPT_FILE, role=ROLE, sagemaker_session=sagemaker_session, training_steps=1000, evaluation_steps=10, train_instance_count=INSTANCE_COUNT, train_instance_type=INSTANCE_TYPE, framework_version=tf_version, requirements_file=REQUIREMENTS_FILE, source_dir=DATA_DIR) inputs = 's3://mybucket/train' s3_prefix = 's3://{}/{}/source/sourcedir.tar.gz'.format( BUCKET_NAME, JOB_NAME) e_tar.return_value = UploadedCode(s3_prefix=s3_prefix, script_name=SCRIPT_FILE) s3_prefix = 's3://{}/{}/sourcedir.tar.gz'.format(BUCKET_NAME, JOB_NAME) m_tar.return_value = UploadedCode(s3_prefix=s3_prefix, script_name=SCRIPT_FILE) tf.fit(inputs=inputs) call_names = [c[0] for c in sagemaker_session.method_calls] assert call_names == ['train', 'logs_for_job'] expected_train_args = _create_train_job(tf_version) expected_train_args['input_config'][0]['DataSource']['S3DataSource'][ 'S3Uri'] = inputs actual_train_args = sagemaker_session.method_calls[0][2] assert actual_train_args == expected_train_args model = tf.create_model() environment = { 'Environment': { 'SAGEMAKER_SUBMIT_DIRECTORY': 's3://{}/{}/sourcedir.tar.gz'.format(BUCKET_NAME, JOB_NAME), 'SAGEMAKER_PROGRAM': 'dummy_script.py', 'SAGEMAKER_REQUIREMENTS': 'dummy_requirements.txt', 'SAGEMAKER_ENABLE_CLOUDWATCH_METRICS': 'false', 'SAGEMAKER_REGION': 'us-west-2', 'SAGEMAKER_CONTAINER_LOG_LEVEL': '20' }, 'Image': create_image_uri('us-west-2', "tensorflow", INSTANCE_TYPE, tf_version, "py2"), 'ModelDataUrl': 's3://m/m.tar.gz' } assert environment == model.prepare_container_def(INSTANCE_TYPE) assert 'cpu' in model.prepare_container_def(INSTANCE_TYPE)['Image'] predictor = tf.deploy(1, INSTANCE_TYPE) assert isinstance(predictor, TensorFlowPredictor)
def _upload_code(self, key_prefix, repack=False): local_code = utils.get_config_value("local.local_code", self.sagemaker_session.config) if self.sagemaker_session.local_mode and local_code: self.uploaded_code = None elif not repack: bucket = self.bucket or self.sagemaker_session.default_bucket() self.uploaded_code = fw_utils.tar_and_upload_dir( session=self.sagemaker_session.boto_session, bucket=bucket, s3_key_prefix=key_prefix, script=self.entry_point, directory=self.source_dir, dependencies=self.dependencies, ) if repack: bucket = self.bucket or self.sagemaker_session.default_bucket() repacked_model_data = "s3://" + os.path.join( bucket, key_prefix, "model.tar.gz") utils.repack_model( inference_script=self.entry_point, source_directory=self.source_dir, dependencies=self.dependencies, model_uri=self.model_data, repacked_model_uri=repacked_model_data, sagemaker_session=self.sagemaker_session, ) self.repacked_model_data = repacked_model_data self.uploaded_code = UploadedCode( s3_prefix=self.repacked_model_data, script_name=os.path.basename(self.entry_point))
def test_model_mms_version(repack_model, sagemaker_session): model_kms_key = "kms-key" model = MXNetModel( MODEL_DATA, role=ROLE, entry_point=SCRIPT_PATH, framework_version=MXNetModel._LOWEST_MMS_VERSION, sagemaker_session=sagemaker_session, name="test-mxnet-model", model_kms_key=model_kms_key, ) predictor = model.deploy(1, GPU) repack_model.assert_called_once_with( inference_script=SCRIPT_PATH, source_directory=None, dependencies=[], model_uri=MODEL_DATA, repacked_model_uri="s3://mybucket/test-mxnet-model/model.tar.gz", sagemaker_session=sagemaker_session, kms_key=model_kms_key, ) assert model.model_data == MODEL_DATA assert model.repacked_model_data == "s3://mybucket/test-mxnet-model/model.tar.gz" assert model.uploaded_code == UploadedCode( s3_prefix="s3://mybucket/test-mxnet-model/model.tar.gz", script_name=os.path.basename(SCRIPT_PATH), ) assert isinstance(predictor, MXNetPredictor)
def attach(cls, training_job_name, sagemaker_session=None): """Attach to an existing training job. Create an Estimator bound to an existing training job, each subclass is responsible to implement ``_prepare_init_params_from_job_description()`` as this method delegates the actual conversion of a training job description to the arguments that the class constructor expects. After attaching, if the training job has a Complete status, it can be ``deploy()`` ed to create a SageMaker Endpoint and return a ``Predictor``. If the training job is in progress, attach will block and display log messages from the training job, until the training job completes. Args: training_job_name (str): The name of the training job to attach to. sagemaker_session (sagemaker.session.Session): Session object which manages interactions with Amazon SageMaker APIs and any other AWS services needed. If not specified, the estimator creates one using the default AWS configuration chain. Examples: >>> my_estimator.fit(wait=False) >>> training_job_name = my_estimator.latest_training_job.name Later on: >>> attached_estimator = Estimator.attach(training_job_name) >>> attached_estimator.deploy() Returns: Instance of the calling ``Estimator`` Class with the attached training job. """ estimator = super(Framework, cls).attach(training_job_name, sagemaker_session) estimator.uploaded_code = UploadedCode(estimator.source_dir, estimator.entry_point) return estimator
def test_tar_and_upload_dir_s3(sagemaker_session): bucket = 'mybucker' s3_key_prefix = 'something/source' script = 'mnist.py' directory = 's3://m' result = tar_and_upload_dir(sagemaker_session, bucket, s3_key_prefix, script, directory) assert result == UploadedCode('s3://m', 'mnist.py')
def test_tar_and_upload_dir_not_s3(sagemaker_session): bucket = 'mybucker' s3_key_prefix = 'something/source' script = os.path.basename(__file__) directory = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) result = tar_and_upload_dir(sagemaker_session, bucket, s3_key_prefix, script, directory) assert result == UploadedCode('s3://{}/{}/sourcedir.tar.gz'.format(bucket, s3_key_prefix), script)
def attach(cls, training_job_name, sagemaker_session=None, **kwargs): """Attach to an existing training job. Create an Estimator bound to an existing training job. After attaching, if the training job has a Complete status, it can be ``deploy()`` ed to create a SageMaker Endpoint and return a ``Predictor``. If the training job is in progress, attach will block and display log messages from the training job, until the training job completes. Args: training_job_name (str): The name of the training job to attach to. sagemaker_session (sagemaker.session.Session): Session object which manages interactions with Amazon SageMaker APIs and any other AWS services needed. If not specified, the estimator creates one using the default AWS configuration chain. **kwargs: Additional kwargs passed to the :class:`~sagemaker.estimator.Estimator` constructor. Returns: sagemaker.estimator.Framework: ``Estimator`` with the attached training job. """ sagemaker_session = sagemaker_session or Session() if training_job_name is not None: job_details = sagemaker_session.sagemaker_client.describe_training_job( TrainingJobName=training_job_name) init_params, hp, _ = cls._prepare_estimator_params_from_job_description( job_details) else: # this case is only valid when called from inheriting class and then the class must declare framework if not hasattr(cls, '__framework_name__'): raise ValueError('must specify training_job name') init_params = dict(kwargs) hp = init_params.pop('hyperparameters') # parameters for framework classes framework_init_params = dict() framework_init_params['entry_point'] = json.loads( hp.get(SCRIPT_PARAM_NAME)) framework_init_params['source_dir'] = json.loads( hp.get(DIR_PARAM_NAME)) framework_init_params['enable_cloudwatch_metrics'] = json.loads( hp.get(CLOUDWATCH_METRICS_PARAM_NAME)) framework_init_params['container_log_level'] = json.loads( hp.get(CONTAINER_LOG_LEVEL_PARAM_NAME)) # drop json and remove other SageMaker specific additions hyperparameters = {entry: json.loads(hp[entry]) for entry in hp} framework_init_params['hyperparameters'] = hyperparameters init_params.update(framework_init_params) estimator = cls(sagemaker_session=sagemaker_session, **init_params) estimator.latest_training_job = _TrainingJob( sagemaker_session=sagemaker_session, training_job_name=init_params['base_job_name']) estimator.latest_training_job.wait() estimator.uploaded_code = UploadedCode(estimator.source_dir, estimator.entry_point) return estimator
def attach(cls, training_job_name, sagemaker_session=None, model_channel_name="model"): """Attach to an existing training job. Create an Estimator bound to an existing training job, each subclass is responsible to implement ``_prepare_init_params_from_job_description()`` as this method delegates the actual conversion of a training job description to the arguments that the class constructor expects. After attaching, if the training job has a Complete status, it can be ``deploy()`` ed to create a SageMaker Endpoint and return a ``Predictor``. If the training job is in progress, attach will block and display log messages from the training job, until the training job completes. Examples: >>> my_estimator.fit(wait=False) >>> training_job_name = my_estimator.latest_training_job.name Later on: >>> attached_estimator = Estimator.attach(training_job_name) >>> attached_estimator.deploy() Args: training_job_name (str): The name of the training job to attach to. sagemaker_session (sagemaker.session.Session): Session object which manages interactions with Amazon SageMaker APIs and any other AWS services needed. If not specified, the estimator creates one using the default AWS configuration chain. model_channel_name (str): Name of the channel where pre-trained model data will be downloaded (default: 'model'). If no channel with the same name exists in the training job, this option will be ignored. Returns: Instance of the calling ``Estimator`` Class with the attached training job. """ sagemaker_session = sagemaker_session or Session() job_details = sagemaker_session.sagemaker_client.describe_training_job( TrainingJobName=training_job_name ) init_params = cls._prepare_init_params_from_job_description(job_details, model_channel_name) tags = sagemaker_session.sagemaker_client.list_tags( ResourceArn=job_details["TrainingJobArn"] )["Tags"] init_params.update(tags=tags) estimator = cls(sagemaker_session=sagemaker_session, **init_params) estimator.latest_training_job = _TrainingJob( sagemaker_session=sagemaker_session, job_name=training_job_name ) estimator._current_job_name = estimator.latest_training_job.name estimator.latest_training_job.wait() # pylint gets confused thinking that estimator is an EstimatorBase instance, but it actually # is a Framework or any of its derived classes. We can safely ignore the no-member errors. estimator.uploaded_code = UploadedCode( estimator.source_dir, estimator.entry_point # pylint: disable=no-member ) return estimator
def test_create_model_with_network_isolation(upload, sagemaker_session): source_dir = "s3://mybucket/source" repacked_model_data = "s3://mybucket/prefix/model.tar.gz" sklearn_model = SKLearnModel( model_data=source_dir, role=ROLE, sagemaker_session=sagemaker_session, entry_point=SCRIPT_PATH, enable_network_isolation=True, ) sklearn_model.uploaded_code = UploadedCode(s3_prefix=repacked_model_data, script_name="script") sklearn_model.repacked_model_data = repacked_model_data model_values = sklearn_model.prepare_container_def(CPU) assert model_values["Environment"]["SAGEMAKER_SUBMIT_DIRECTORY"] == "/opt/ml/model/code" assert model_values["ModelDataUrl"] == repacked_model_data
def test_model_mms_version(repack_model, sagemaker_session): model = MXNetModel(MODEL_DATA, role=ROLE, entry_point=SCRIPT_PATH, framework_version=MXNetModel._LOWEST_MMS_VERSION, sagemaker_session=sagemaker_session) predictor = model.deploy(1, GPU) repack_model.assert_called_once_with(inference_script=SCRIPT_PATH, source_directory=None, model_uri=MODEL_DATA, sagemaker_session=sagemaker_session) assert model.model_data == MODEL_DATA assert model.repacked_model_data == REPACKED_MODEL_DATA assert model.uploaded_code == UploadedCode( s3_prefix=REPACKED_MODEL_DATA, script_name=os.path.basename(SCRIPT_PATH)) assert isinstance(predictor, MXNetPredictor)
def uploaded_code( s3_prefix="s3://mocked_s3_uri_from_upload_data/my_job_name/source/sourcedir.tar.gz", script_name="processing_code.py", ): return UploadedCode(s3_prefix=s3_prefix, script_name=script_name)