def test_create(make_stubber, error_code):
kinesisanalyticsv2_client = boto3.client('kinesisanalyticsv2')
kinesisanalyticsv2_stubber = make_stubber(kinesisanalyticsv2_client)
app = KinesisAnalyticsApplicationV2(kinesisanalyticsv2_client)
app_name = 'test-app'
app_arn = f'arn:aws:kinesisanalytics:REGION:123456789012:application/{app_name}'
app_version_id = 1
role_arn = 'test-role-arn'
kinesisanalyticsv2_stubber.stub_create_application(app_name,
'SQL-1_0',
role_arn,
app_arn,
app_version_id,
error_code=error_code)
if error_code is None:
got_app_details = app.create(app_name, role_arn)
assert app.name == app_name
assert app.version_id == app_version_id
assert app.arn == app_arn
assert got_app_details['ApplicationName'] == app_name
else:
with pytest.raises(ClientError) as exc_info:
app.create(app_name, role_arn)
assert exc_info.value.response['Error']['Code'] == error_code
def test_add_input(make_stubber, error_code):
kinesisanalyticsv2_client = boto3.client('kinesisanalyticsv2')
kinesisanalyticsv2_stubber = make_stubber(kinesisanalyticsv2_client)
app = KinesisAnalyticsApplicationV2(kinesisanalyticsv2_client)
app.name = 'test-app'
app.version_id = 1
prefix = 'test-prefix'
stream_arn = 'test-arn'
input_schema = {
'RecordFormat': {
'RecordFormatType': 'JSON'
},
'RecordColumns': [{
'Name': 'test-data',
'Mapping': 'test-col',
'SqlType': 'VARCHAR(4)'
}]
}
kinesisanalyticsv2_stubber.stub_add_application_input(
app.name,
app.version_id,
prefix,
stream_arn,
input_schema,
error_code=error_code)
if error_code is None:
got_response = app.add_input(prefix, stream_arn, input_schema)
assert got_response['InputDescriptions'][0][
'InputSchema'] == input_schema
else:
with pytest.raises(ClientError) as exc_info:
app.add_input(prefix, stream_arn, input_schema)
assert exc_info.value.response['Error']['Code'] == error_code
def test_create_read_write_role(make_stubber, stub_runner, error_code):
iam_resource = boto3.resource('iam')
iam_stubber = make_stubber(iam_resource.meta.client)
policy = 'test-policy'
policy_arn = f'arn:aws:iam:REGION:123456789012:policy/{policy}'
role = 'test-role'
iam_stubber.stub_create_policy(policy, policy_arn, error_code=error_code)
if error_code is None:
iam_stubber.stub_create_role(role)
iam_stubber.stub_attach_role_policy(role, policy_arn)
iam_stubber.stub_get_policy(policy_arn)
if error_code is None:
got_role = KinesisAnalyticsApplicationV2.create_read_write_role(
'test', 'input-arn', 'output-arn', iam_resource)
assert got_role.name == role
else:
with pytest.raises(ClientError) as exc_info:
KinesisAnalyticsApplicationV2.create_read_write_role(
'test', 'input-arn', 'output-arn', iam_resource)
assert exc_info.value.response['Error']['Code'] == error_code
def test_stop(make_stubber, error_code):
kinesisanalyticsv2_client = boto3.client('kinesisanalyticsv2')
kinesisanalyticsv2_stubber = make_stubber(kinesisanalyticsv2_client)
app = KinesisAnalyticsApplicationV2(kinesisanalyticsv2_client)
app.name = 'test-app'
kinesisanalyticsv2_stubber.stub_stop_application(app.name,
error_code=error_code)
if error_code is None:
app.stop()
else:
with pytest.raises(ClientError) as exc_info:
app.stop()
assert exc_info.value.response['Error']['Code'] == error_code
def test_delete(make_stubber, error_code):
kinesisanalyticsv2_client = boto3.client('kinesisanalyticsv2')
kinesisanalyticsv2_stubber = make_stubber(kinesisanalyticsv2_client)
app = KinesisAnalyticsApplicationV2(kinesisanalyticsv2_client)
app.name = 'test-app'
app.create_timestamp = datetime.datetime.now()
kinesisanalyticsv2_stubber.stub_delete_application(app.name,
app.create_timestamp,
error_code=error_code)
if error_code is None:
app.delete()
else:
with pytest.raises(ClientError) as exc_info:
app.delete()
assert exc_info.value.response['Error']['Code'] == error_code
def test_discover_input_schema(make_stubber, error_code):
kinesisanalyticsv2_client = boto3.client('kinesisanalyticsv2')
kinesisanalyticsv2_stubber = make_stubber(kinesisanalyticsv2_client)
app = KinesisAnalyticsApplicationV2(kinesisanalyticsv2_client)
stream_arn = 'test-stream_arn'
role_arn = 'test-role-arn'
kinesisanalyticsv2_stubber.stub_discover_input_schema(
stream_arn, role_arn, error_code=error_code)
if error_code is None:
got_schema = app.discover_input_schema(stream_arn, role_arn)
assert 'RecordFormat' in got_schema
else:
with pytest.raises(ClientError) as exc_info:
app.discover_input_schema(stream_arn, role_arn)
assert exc_info.value.response['Error']['Code'] == error_code
def test_describe_snapshot(make_stubber, error_code):
kinesisanalyticsv2_client = boto3.client('kinesisanalyticsv2')
kinesisanalyticsv2_stubber = make_stubber(kinesisanalyticsv2_client)
app = KinesisAnalyticsApplicationV2(kinesisanalyticsv2_client)
app_name = 'test-app'
snapshot_name = 'test-snapshot'
kinesisanalyticsv2_stubber.stub_describe_application_snapshot(
app_name, snapshot_name, error_code=error_code)
if error_code is None:
got_snapshot = app.describe_snapshot(app_name, snapshot_name)
assert got_snapshot['SnapshotName'] == snapshot_name
else:
with pytest.raises(ClientError) as exc_info:
app.describe_snapshot(app_name, snapshot_name)
assert exc_info.value.response['Error']['Code'] == error_code
def test_update_code(make_stubber, error_code):
kinesisanalyticsv2_client = boto3.client('kinesisanalyticsv2')
kinesisanalyticsv2_stubber = make_stubber(kinesisanalyticsv2_client)
app = KinesisAnalyticsApplicationV2(kinesisanalyticsv2_client)
app.name = 'test-app'
app.version_id = 1
code = 'test-code'
kinesisanalyticsv2_stubber.stub_update_application(app.name,
app.version_id,
code,
error_code=error_code)
if error_code is None:
got_details = app.update_code(code)
assert got_details['ApplicationName'] == app.name
else:
with pytest.raises(ClientError) as exc_info:
app.update_code(code)
assert exc_info.value.response['Error']['Code'] == error_code
def test_describe(make_stubber, error_code):
kinesisanalyticsv2_client = boto3.client('kinesisanalyticsv2')
kinesisanalyticsv2_stubber = make_stubber(kinesisanalyticsv2_client)
app = KinesisAnalyticsApplicationV2(kinesisanalyticsv2_client)
app_name = 'test-app'
version_id = 1
app_arn = f'arn:aws:kinesisanalytics:REGION:123456789012:application{app_name}'
kinesisanalyticsv2_stubber.stub_describe_application(app_name,
version_id,
app_arn,
error_code=error_code)
if error_code is None:
got_details = app.describe(app_name)
assert got_details['ApplicationName'] == app_name
assert app.name == app_name
assert app.version_id == version_id
assert app.arn == app_arn
else:
with pytest.raises(ClientError) as exc_info:
app.describe(app_name)
assert exc_info.value.response['Error']['Code'] == error_code
def test_add_output(make_stubber, error_code):
kinesisanalyticsv2_client = boto3.client('kinesisanalyticsv2')
kinesisanalyticsv2_stubber = make_stubber(kinesisanalyticsv2_client)
app = KinesisAnalyticsApplicationV2(kinesisanalyticsv2_client)
app.name = 'test-app'
app.version_id = 1
stream_name = 'test-stream_name'
output_arn = 'test-arn'
kinesisanalyticsv2_stubber.stub_add_application_output(
app.name,
app.version_id,
stream_name,
output_arn,
error_code=error_code)
if error_code is None:
got_response = app.add_output(stream_name, output_arn)
assert got_response[0]['KinesisStreamsOutputDescription'][
'ResourceARN'] == output_arn
else:
with pytest.raises(ClientError) as exc_info:
app.add_output(stream_name, output_arn)
assert exc_info.value.response['Error']['Code'] == error_code
def usage_demo():
print('-' * 88)
print(
"Welcome to the demo of version 2 of the Amazon Kinesis Data Analytics API."
)
print('-' * 88)
logging.basicConfig(level=logging.INFO,
format='%(levelname)s: %(message)s')
kinesis_client = boto3.client('kinesis')
analytics_client = boto3.client('kinesisanalyticsv2')
iam_resource = boto3.resource('iam')
application = KinesisAnalyticsApplicationV2(analytics_client)
app_running_waiter = ApplicationRunningWaiter(analytics_client)
input_stream_name = 'doc-example-stream-input'
input_prefix = 'SOURCE_SQL_STREAM'
output_stream_name = 'doc-example-stream-output'
app_name = 'doc-example-app'
role_name = 'doc-example-kinesis-read-write'
print(f"Creating input stream {input_stream_name} and output stream "
f"{output_stream_name}.")
input_stream = KinesisStream(kinesis_client)
input_stream.create(input_stream_name)
output_stream = KinesisStream(kinesis_client)
output_stream.create(output_stream_name)
print(
"Starting data generator (on a separate thread) to put data into the "
"input stream.")
stream_thread = threading.Thread(target=generate,
args=(input_stream.name, kinesis_client,
False),
daemon=True)
stream_thread.start()
print(
f"Creating role {role_name} to let Kinesis Analytics read from the input "
f"stream and write to the output stream.")
role = application.create_read_write_role(role_name, input_stream.arn(),
output_stream.arn(),
iam_resource)
print("Waiting for role to be ready.")
time.sleep(10)
print(f"Creating application {app_name}.")
# Sometimes the role is still not ready and InvalidArgumentException is raised, so
# continue to retry if this happens.
app_data = exponential_retry('InvalidArgumentException')(
application.create)(app_name, role.arn)
pprint(app_data)
print(f"Discovering schema of input stream {input_stream.name}.")
input_schema = application.discover_input_schema(input_stream.arn(),
role.arn)
pprint(input_schema)
print("Adding input stream to the application.")
input_details = application.add_input(input_prefix, input_stream.arn(),
input_schema)
print("Input details:")
pprint(input_details)
print("Uploading SQL code to the application to process the input stream.")
with open('analyticsv2/example.sql') as code_file:
code = code_file.read()
application.update_code(code)
print("Adding output stream to the application.")
application.add_output('DESTINATION_SQL_STREAM', output_stream.arn())
print("Starting the application.")
application.start(input_details['InputDescriptions'][0]['InputId'])
print(
"Waiting for the application to start (this may take a minute or two)."
)
app_running_waiter.wait(application.name)
print("Application started. Getting records from the output stream.")
for records in output_stream.get_records(50):
if len(records) > 0:
print(*[rec['Data'].decode() for rec in records], sep='\n')
print("Cleaning up...")
application.delete()
input_stream.delete()
output_stream.delete()
print("Deleting read/write role.")
for policy in role.attached_policies.all():
role.detach_policy(PolicyArn=policy.arn)
policy.delete()
role.delete()
print("Thanks for watching!")
print('-' * 88)