def test_setup_shards(mocker):
mock_boto3_session = MagicMock()
mock_shard_reader = mocker.patch('kinesis.consumer.ShardReader')
consumer = KinesisConsumer('testing', boto3_session=mock_boto3_session)
mock_boto3_session.client.assert_called_with('kinesis')
consumer.kinesis_client.describe_stream.return_value = {
'StreamDescription': {
'Shards': [{
'ShardId': 'test-shard',
}]
}
}
consumer.kinesis_client.get_shard_iterator.return_value = {
'ShardIterator': 'test-iter'
}
consumer.setup_shards()
consumer.kinesis_client.describe_stream.assert_called_with(
StreamName='testing')
consumer.kinesis_client.get_shard_iterator.assert_called_with(
StreamName='testing', ShardId='test-shard', ShardIteratorType='LATEST')
mock_shard_reader.assert_called_with('test-shard',
'test-iter',
consumer.record_queue,
consumer.error_queue,
boto3_session=consumer.boto3_session,
sleep_time=consumer.reader_sleep_time)
def create_consumer(stream_name, session, db_name=None):
if db_name:
consumer = KinesisConsumer(stream_name=stream_name,
boto3_session=session,
state=DynamoDB(db_name, session))
else:
consumer = KinesisConsumer(stream_name=stream_name,
boto3_session=session)
return consumer
def start_consuming(args):
consumer = KinesisConsumer(stream_name=args.stream)
records_consumed = 0
for record in consumer:
records_consumed += 1
message = record.get("Data")
try:
message = json.dumps(json.loads(message.decode()),
indent=4,
sort_keys=True)
settings.RECORDS_CAPTURED.append(json.loads(message))
except ValueError:
settings.RECORDS_CAPTURED.append(message.decode())
finally:
print(highlight(message, JsonLexer(), TerminalFormatter()))
print()
print("Records consumed: %s" % records_consumed)
print()
import logging
from kinesis.consumer import KinesisConsumer
from kinesis.state import DynamoDB
log = logging.getLogger(__name__)
logging.basicConfig(
level=logging.DEBUG,
format='%(levelname)s %(process)d %(name)s:%(lineno)d %(message)s')
logging.getLogger('botocore').level = logging.INFO
logging.getLogger(
'botocore.vendored.requests.packages.urllib3').level = logging.WARN
consumer = KinesisConsumer(stream_name='borgstrom-test',
state=DynamoDB('kinesis-locks'))
for message in consumer:
log.info("Received message: %s", message)
def Form_Expression_Values():
expression = {':inc': 1, ':init': 0, ':w_l': item['W_L'], ':match_start': item['Match_Start_Time'], ':gold': item['Gold_Adv'], ':xp': item['XP_Adv'], ':process': item['processed'], ':kinesis_stamp': item['Kinesis_Timestamp']}
return expression
#Define DynamoDB Tables
dynamo_db = boto3.resource('dynamodb')
table = dynamo_db.Table('Parallel_Test')
teammates_table = dynamo_db.Table('Hero_Teammates')
counters_table = dynamo_db.Table('Hero_Counters')
#Initialize Kinesis Stream to read from
my_stream_name = 'Consumer_Test'
consumer = KinesisConsumer(stream_name = my_stream_name)
for rec in consumer:
info = json.loads(rec['Data'])
radiant_item = []
dire_item = []
teammates = []
counters = []
#If Gold/Xp advantages are missing, use 0 as a placeholder
if isinstance(info['radiant_xp_adv'],basestring):
radiant_xp_adv = int(info['radiant_xp_adv'].split(",")[-1][:-1])
else:
radiant_xp_adv = 0
if isinstance(info['radiant_gold_adv'][0],basestring):
radiant_gold_adv = int(info['radiant_gold_adv'][0].split(",")[-1][:-1])
else:
def consumer(self):
return KinesisConsumer(self.stream_name,
boto3_session=self.boto3_session,
state=self.consumer_state)