def test_marketplace_transform_job_from_model_package(sagemaker_session,
cpu_instance_type):
data_path = os.path.join(DATA_DIR, "marketplace", "training")
shape = pandas.read_csv(data_path + "/iris.csv", header=None).drop([0],
axis=1)
TRANSFORM_WORKDIR = DATA_DIR + "/marketplace/transform"
shape.to_csv(TRANSFORM_WORKDIR + "/batchtransform_test.csv",
index=False,
header=False)
transform_input = sagemaker_session.upload_data(
TRANSFORM_WORKDIR, key_prefix="integ-test-data/marketplace/transform")
region = sagemaker_session.boto_region_name
account = REGION_ACCOUNT_MAP[region]
model_package_arn = MODEL_PACKAGE_ARN.format(
partition=_aws_partition(region), region=region, account=account)
model = ModelPackage(
role="SageMakerRole",
model_package_arn=model_package_arn,
sagemaker_session=sagemaker_session,
)
transformer = model.transformer(1, cpu_instance_type)
transformer.transform(transform_input, content_type="text/csv")
transformer.wait()
def test_marketplace_model(sagemaker_session, cpu_instance_type):
region = sagemaker_session.boto_region_name
account = REGION_ACCOUNT_MAP[region]
model_package_arn = MODEL_PACKAGE_ARN % (region, account)
def predict_wrapper(endpoint, session):
return sagemaker.RealTimePredictor(
endpoint, session, serializer=sagemaker.predictor.csv_serializer)
model = ModelPackage(
role="SageMakerRole",
model_package_arn=model_package_arn,
sagemaker_session=sagemaker_session,
predictor_cls=predict_wrapper,
)
endpoint_name = "test-marketplace-model-endpoint{}".format(
sagemaker_timestamp())
with timeout_and_delete_endpoint_by_name(endpoint_name,
sagemaker_session,
minutes=20):
predictor = model.deploy(1,
cpu_instance_type,
endpoint_name=endpoint_name)
data_path = os.path.join(DATA_DIR, "marketplace", "training")
shape = pandas.read_csv(os.path.join(data_path, "iris.csv"),
header=None)
a = [50 * i for i in range(3)]
b = [40 + i for i in range(10)]
indices = [i + j for i, j in itertools.product(a, b)]
test_data = shape.iloc[indices[:-1]]
test_x = test_data.iloc[:, 1:]
print(predictor.predict(test_x.values).decode("utf-8"))
def test_marketplace_model(sagemaker_session):
def predict_wrapper(endpoint, session):
return sagemaker.RealTimePredictor(
endpoint, session, serializer=sagemaker.predictor.csv_serializer)
model = ModelPackage(
role='SageMakerRole',
model_package_arn=(MODEL_PACKAGE_ARN %
sagemaker_session.boto_region_name),
sagemaker_session=sagemaker_session,
predictor_cls=predict_wrapper)
endpoint_name = 'test-marketplace-model-endpoint{}'.format(
sagemaker_timestamp())
with timeout_and_delete_endpoint_by_name(endpoint_name,
sagemaker_session,
minutes=20):
predictor = model.deploy(1,
'ml.m4.xlarge',
endpoint_name=endpoint_name)
data_path = os.path.join(DATA_DIR, 'marketplace', 'training')
shape = pandas.read_csv(os.path.join(data_path, 'iris.csv'),
header=None)
a = [50 * i for i in range(3)]
b = [40 + i for i in range(10)]
indices = [i + j for i, j in itertools.product(a, b)]
test_data = shape.iloc[indices[:-1]]
test_x = test_data.iloc[:, 1:]
print(predictor.predict(test_x.values).decode('utf-8'))
def test_marketplace_transform_job_from_model_package(sagemaker_session):
data_path = os.path.join(DATA_DIR, 'marketplace', 'training')
shape = pandas.read_csv(data_path + '/iris.csv', header=None).drop([0], axis=1)
TRANSFORM_WORKDIR = DATA_DIR + '/marketplace/transform'
shape.to_csv(TRANSFORM_WORKDIR + '/batchtransform_test.csv', index=False, header=False)
transform_input = sagemaker_session.upload_data(
TRANSFORM_WORKDIR,
key_prefix='integ-test-data/marketplace/transform')
model = ModelPackage(role='SageMakerRole',
model_package_arn=(MODEL_PACKAGE_ARN % sagemaker_session.boto_region_name),
sagemaker_session=sagemaker_session)
transformer = model.transformer(1, 'ml.m4.xlarge')
transformer.transform(transform_input, content_type='text/csv')
transformer.wait()
)
# Update the existing Endpoint
create_endpoint_api_response = sagemaker_boto_client.update_endpoint(
EndpointName=args.endpoint_name,
EndpointConfigName=ep_config_name
)
create_config('Y')
except ClientError as error:
# endpoint does not exist
if "Could not find endpoint" in error.response['Error']['Message']:
model_package_approved = get_approved_package(args.model_package_group_name)
model_package_arn = model_package_approved["ModelPackageArn"]
model = ModelPackage(role=args.role,
model_package_arn=model_package_arn,
sagemaker_session=sagemaker_session)
try:
model.deploy(initial_instance_count=args.initial_instance_count,
instance_type=args.endpoint_instance_type,
endpoint_name=args.endpoint_name)
create_config('Y')
except ClientError as error:
print(error.response['Error']['Message'])
create_config('N')
error_message = error.response["Error"]["Message"]
LOGGER.error("{}".format(stacktrace))
raise Exception(error_message)
else:
print(error.response['Error']['Message'])
create_config('N')
def cli():
"""
Simply run this script with '--help' to see all options.
"""
desc = "Pipeline to train a AWS Marketplace model on Woocommerce data and upload cross-sell product recommendations"
logging.basicConfig(format="%(asctime)s : %(levelname)s : %(message)s",
level=logging.INFO)
parser = argparse.ArgumentParser(description=desc)
parser.add_argument("-i",
"--input",
help="input path",
dest="input",
required=True)
parser.add_argument("-o",
"--output",
help="output_path",
dest="output",
required=True)
args = parser.parse_args()
logging.info("configure services")
config = Configuration("configs/config.json").parameters
s3 = S3(
region_name=config["s3"]["region_name"],
access_key=config["aws_keys"]["access_key"],
secret_key=config["aws_keys"]["secret_key"],
bucket=config["s3"]["bucket"],
)
woocommerce = Woocommerce(
url=config["woocommerce"]["url"],
consumer_key=config["woocommerce"]["consumer_key"],
consumer_secret=config["woocommerce"]["consumer_secret"],
)
logging.info("download data from Woocommerce")
woocommerce.download_data(args.input)
logging.info("upload data to S3")
s3.upload(args.input, config["s3"]["input_file"])
logging.info("configure Batch Transform Job")
model_package_arn = config["batch_transform_job"]["model_package_arn"]
iam = boto3.client("iam")
role = iam.get_role(
RoleName=config["batch_transform_job"]["role_name"])["Role"]["Arn"]
sagemaker_session = sagemaker.Session()
model = ModelPackage(
model_package_arn=model_package_arn,
role=role,
sagemaker_session=sagemaker_session,
)
output_path = "s3://" + os.path.join(config["s3"]["bucket"], "output")
transformer = model.transformer(
instance_count=config["batch_transform_job"]["instance_count"],
instance_type=config["batch_transform_job"]["instance_type"],
output_path=output_path,
)
input_path = "s3://" + os.path.join(config["s3"]["bucket"],
config["s3"]["input_file"])
transformer.transform(input_path, content_type="text/csv")
logging.info('batch transform job starting')
transformer.wait()
logging.info("downloading recommendations from S3")
bucket_folder = transformer.output_path.rsplit("/")[3]
key = os.path.join(bucket_folder,
config["s3"]["input_file"].split("/")[-1] + ".out")
s3.download(key, args.output)
logging.info("Push recommendations to Woocommerce")
woocommerce.upload_recommendations(args.output)
