def wait_for_stream_created(table_name): stream_name = get_kinesis_stream_name(table_name) stream = KinesisStream(id=stream_name, num_shards=1) kinesis = aws_stack.create_external_boto_client( "kinesis", env=get_environment(None)) stream.connect(kinesis) stream.wait_for()
def wait_for_stream_created(table_name): stream_name = get_kinesis_stream_name(table_name) stream = KinesisStream(id=stream_name, num_shards=1) kinesis = aws_stack.connect_to_service('kinesis', env=get_environment(None)) stream.connect(kinesis) stream.wait_for()
def test_stream_spec_and_region_replacement(self): ddbstreams = aws_stack.connect_to_service('dynamodbstreams') kinesis = aws_stack.connect_to_service('kinesis') table_name = 'ddb-%s' % short_uid() aws_stack.create_dynamodb_table( table_name, partition_key=PARTITION_KEY, stream_view_type='NEW_AND_OLD_IMAGES') table = self.dynamodb.Table(table_name) # assert ARN formats expected_arn_prefix = 'arn:aws:dynamodb:' + aws_stack.get_local_region() self.assertTrue(table.table_arn.startswith(expected_arn_prefix)) self.assertTrue(table.latest_stream_arn.startswith(expected_arn_prefix)) # assert stream has been created stream_tables = [s['TableName'] for s in ddbstreams.list_streams()['Streams']] self.assertIn(table_name, stream_tables) stream_name = get_kinesis_stream_name(table_name) self.assertIn(stream_name, kinesis.list_streams()['StreamNames']) # assert shard ID formats result = ddbstreams.describe_stream(StreamArn=table.latest_stream_arn)['StreamDescription'] self.assertIn('Shards', result) for shard in result['Shards']: self.assertRegex(shard['ShardId'], r'^shardId\-[0-9]{20}\-[a-zA-Z0-9]{1,36}$') # clean up delete_table(table_name) # assert stream has been deleted stream_tables = [s['TableName'] for s in ddbstreams.list_streams()['Streams']] self.assertNotIn(table_name, stream_tables) self.assertNotIn(stream_name, kinesis.list_streams()['StreamNames'])
def describe_stream( self, context: RequestContext, stream_arn: StreamArn, limit: PositiveIntegerObject = None, exclusive_start_shard_id: ShardId = None, ) -> DescribeStreamOutput: region = DynamoDBStreamsBackend.get() kinesis = aws_stack.connect_to_service("kinesis") for stream in region.ddb_streams.values(): if stream["StreamArn"] == stream_arn: # get stream details dynamodb = aws_stack.connect_to_service("dynamodb") table_name = table_name_from_stream_arn(stream["StreamArn"]) stream_name = get_kinesis_stream_name(table_name) stream_details = kinesis.describe_stream( StreamName=stream_name) table_details = dynamodb.describe_table(TableName=table_name) stream["KeySchema"] = table_details["Table"]["KeySchema"] stream["StreamStatus"] = STREAM_STATUS_MAP.get( stream_details["StreamDescription"]["StreamStatus"]) # Replace Kinesis ShardIDs with ones that mimic actual # DynamoDBStream ShardIDs. stream_shards = copy.deepcopy( stream_details["StreamDescription"]["Shards"]) start_index = 0 for index, shard in enumerate(stream_shards): shard["ShardId"] = get_shard_id(stream, shard["ShardId"]) shard.pop("HashKeyRange", None) # we want to ignore the shards before exclusive_start_shard_id parameters # we store the index where we encounter then slice the shards if exclusive_start_shard_id and exclusive_start_shard_id == shard[ "ShardId"]: start_index = index if exclusive_start_shard_id: # slicing the resulting shards after the exclusive_start_shard_id parameters stream_shards = stream_shards[start_index + 1:] stream["Shards"] = stream_shards stream_description = select_from_typed_dict( StreamDescription, stream) return DescribeStreamOutput( StreamDescription=stream_description) raise ResourceNotFoundException(f"Stream {stream_arn} was not found.")
def test_stream_spec_and_region_replacement(self): ddbstreams = aws_stack.connect_to_service("dynamodbstreams") kinesis = aws_stack.connect_to_service("kinesis") table_name = "ddb-%s" % short_uid() aws_stack.create_dynamodb_table( table_name, partition_key=PARTITION_KEY, stream_view_type="NEW_AND_OLD_IMAGES", ) table = self.dynamodb.Table(table_name) # assert ARN formats expected_arn_prefix = "arn:aws:dynamodb:" + aws_stack.get_local_region( ) self.assertTrue(table.table_arn.startswith(expected_arn_prefix)) self.assertTrue( table.latest_stream_arn.startswith(expected_arn_prefix)) # assert stream has been created stream_tables = [ s["TableName"] for s in ddbstreams.list_streams()["Streams"] ] self.assertIn(table_name, stream_tables) stream_name = get_kinesis_stream_name(table_name) self.assertIn(stream_name, kinesis.list_streams()["StreamNames"]) # assert shard ID formats result = ddbstreams.describe_stream( StreamArn=table.latest_stream_arn)["StreamDescription"] self.assertIn("Shards", result) for shard in result["Shards"]: self.assertRegex(shard["ShardId"], r"^shardId\-[0-9]{20}\-[a-zA-Z0-9]{1,36}$") # clean up delete_table(table_name) # assert stream has been deleted stream_tables = [ s["TableName"] for s in ddbstreams.list_streams()["Streams"] ] self.assertNotIn(table_name, stream_tables) self.assertNotIn(stream_name, kinesis.list_streams()["StreamNames"])
def test_stream_spec_and_region_replacement(self, dynamodb): ddbstreams = aws_stack.create_external_boto_client("dynamodbstreams") kinesis = aws_stack.create_external_boto_client("kinesis") table_name = "ddb-%s" % short_uid() aws_stack.create_dynamodb_table( table_name, partition_key=PARTITION_KEY, stream_view_type="NEW_AND_OLD_IMAGES", ) table = dynamodb.Table(table_name) # assert ARN formats expected_arn_prefix = "arn:aws:dynamodb:" + aws_stack.get_local_region( ) assert table.table_arn.startswith(expected_arn_prefix) assert table.latest_stream_arn.startswith(expected_arn_prefix) # assert stream has been created stream_tables = [ s["TableName"] for s in ddbstreams.list_streams()["Streams"] ] assert table_name in stream_tables stream_name = get_kinesis_stream_name(table_name) assert stream_name in kinesis.list_streams()["StreamNames"] # assert shard ID formats result = ddbstreams.describe_stream( StreamArn=table.latest_stream_arn)["StreamDescription"] assert "Shards" in result for shard in result["Shards"]: assert re.match(r"^shardId-[0-9]{20}-[a-zA-Z0-9]{1,36}$", shard["ShardId"]) # clean up delete_table(table_name) # assert stream has been deleted stream_tables = [ s["TableName"] for s in ddbstreams.list_streams()["Streams"] ] assert table_name not in stream_tables assert stream_name not in kinesis.list_streams()["StreamNames"]
def describe_stream( self, context: RequestContext, stream_arn: StreamArn, limit: PositiveIntegerObject = None, exclusive_start_shard_id: ShardId = None, ) -> DescribeStreamOutput: region = DynamoDBStreamsBackend.get() kinesis = aws_stack.connect_to_service("kinesis") result = {} for stream in region.ddb_streams.values(): if stream["StreamArn"] == stream_arn: result = {"StreamDescription": stream} # get stream details dynamodb = aws_stack.connect_to_service("dynamodb") table_name = table_name_from_stream_arn(stream["StreamArn"]) stream_name = get_kinesis_stream_name(table_name) stream_details = kinesis.describe_stream( StreamName=stream_name) table_details = dynamodb.describe_table(TableName=table_name) stream["KeySchema"] = table_details["Table"]["KeySchema"] stream["StreamStatus"] = STREAM_STATUS_MAP.get( stream_details["StreamDescription"]["StreamStatus"]) # Replace Kinesis ShardIDs with ones that mimic actual # DynamoDBStream ShardIDs. stream_shards = copy.deepcopy( stream_details["StreamDescription"]["Shards"]) for shard in stream_shards: shard["ShardId"] = shard_id(stream_name, shard["ShardId"]) shard.pop("HashKeyRange", None) stream["Shards"] = stream_shards return DescribeStreamOutput(**result) if not result: raise ResourceNotFoundException( f"Stream {stream_arn} was not found.")