def disable_shard_monitoring(stream, metrics=['ALL']):
if not isinstance(stream, basestring):
stream = stream.id
role = get_iam_role_for_stream(stream)
kinesis_client = aws_common.connect_kinesis(role=role)
return kinesis_client.disable_enhanced_monitoring(
StreamName=stream, ShardLevelMetrics=metrics)
def perform_scaling(kinesis_stream):
downscale = get_downscale_shards(kinesis_stream)
upscale = get_upscale_shards(kinesis_stream)
action = 'NOTHING'
role = kinesis_monitoring.get_iam_role_for_stream(kinesis_stream)
kinesis_client = aws_common.connect_kinesis(role=role)
try:
if downscale:
action = 'DOWNSCALE(-%s)' % len(downscale)
for shard_pair in downscale:
LOG.info('Merging shards %s and %s of Kinesis stream %s' %
(shard_pair.shard1.id, shard_pair.shard2.id,
kinesis_stream.id))
kinesis_client.merge_shards(
StreamName=kinesis_stream.id,
ShardToMerge=shard_pair.shard1.id,
AdjacentShardToMerge=shard_pair.shard2.id)
elif upscale:
action = 'UPSCALE(+%s)' % len(upscale)
for shard in upscale:
LOG.info('Splitting shard %s of Kinesis stream %s' %
(shard.id, kinesis_stream.id))
new_start_key = shard.center_key()
kinesis_client.split_shard(StreamName=kinesis_stream.id,
ShardToSplit=shard.id,
NewStartingHashKey=new_start_key)
except Exception, e:
LOG.warning('Unable to re-scale stream %s: %s' %
(kinesis_stream.id, e))
def retrieve_stream_details(stream_name, role=None):
LOG.info('Getting details for Kinesis stream %s' % stream_name)
if role is None:
role = get_iam_role_for_stream(stream_name)
kinesis_client = aws_common.connect_kinesis(role=role)
out = kinesis_client.describe_stream(StreamName=stream_name)
stream_shards = out['StreamDescription']['Shards']
stream = themis.model.kinesis_model.KinesisStream(stream_name)
num_shards = len(stream_shards)
for shard in stream_shards:
if 'EndingSequenceNumber' not in shard['SequenceNumberRange']:
key_range = shard['HashKeyRange']
shard = themis.model.kinesis_model.KinesisShard(id=shard['ShardId'])
shard.start_key = key_range['StartingHashKey']
shard.end_key = key_range['EndingHashKey']
stream.shards.append(shard)
return stream
def init_kinesis_config(run_parallel=False, role=None):
cfg = themis.model.resources_model.ResourcesConfiguration()
def init_kinesis_stream_config(stream_name):
stream_config = retrieve_stream_details(stream_name, role=role)
cfg.kinesis.append(stream_config)
# load Kinesis streams
kinesis_client = aws_common.connect_kinesis(role=role)
try:
out = kinesis_client.list_streams()
if run_parallel:
common.parallelize(out['StreamNames'], init_kinesis_stream_config)
else:
for c in out['StreamNames']:
init_kinesis_stream_config(c)
except Exception, e:
LOG.info('Unable to list Kinesis streams using IAM role "%s"' % role)
def retrieve_stream_details(stream_name, role=None):
LOG.info('Getting details for Kinesis stream %s' % stream_name)
if role is None:
role = get_iam_role_for_stream(stream_name)
kinesis_client = aws_common.connect_kinesis(role=role)
out = kinesis_client.describe_stream(StreamName=stream_name)
stream_shards = out['StreamDescription']['Shards']
stream = themis.model.kinesis_model.KinesisStream(stream_name)
num_shards = len(stream_shards)
for shard in stream_shards:
if 'EndingSequenceNumber' not in shard['SequenceNumberRange']:
key_range = shard['HashKeyRange']
shard = themis.model.kinesis_model.KinesisShard(
id=shard['ShardId'])
shard.start_key = key_range['StartingHashKey']
shard.end_key = key_range['EndingHashKey']
stream.shards.append(shard)
return stream
def init_kinesis_config(run_parallel=False, role=None):
cfg = themis.model.resources_model.ResourcesConfiguration()
def init_kinesis_stream_config(stream_name):
stream_config = retrieve_stream_details(stream_name, role=role)
cfg.kinesis.append(stream_config)
# load Kinesis streams
kinesis_client = aws_common.connect_kinesis(role=role)
try:
out = kinesis_client.list_streams()
if run_parallel:
common.parallelize(out['StreamNames'], init_kinesis_stream_config)
else:
for c in out['StreamNames']:
init_kinesis_stream_config(c)
except Exception, e:
LOG.info('Unable to list Kinesis streams using IAM role "%s"' % role)
def perform_scaling(kinesis_stream):
if check_cooldown(kinesis_stream) == True:
LOG.info('Cooling down scaling for Kinesis stream %s' %
kinesis_stream.id)
return
downscale = get_downscale_shards(kinesis_stream)
upscale = get_upscale_shards(kinesis_stream)
action = 'NOTHING'
role = kinesis_monitoring.get_iam_role_for_stream(kinesis_stream)
try:
kinesis_client = aws_common.connect_kinesis(role=role)
if downscale:
action = 'DOWNSCALE(-%s)' % len(downscale)
scale_down(downscale, kinesis_client, kinesis_stream)
elif upscale:
action = 'UPSCALE(+%s)' % len(upscale)
scale_up(upscale, kinesis_client, kinesis_stream)
except Exception, e:
LOG.warning('Unable to re-scale stream %s: %s' %
(kinesis_stream.id, e))
return
def perform_scaling(kinesis_stream):
downscale = get_downscale_shards(kinesis_stream)
upscale = get_upscale_shards(kinesis_stream)
action = 'NOTHING'
role = kinesis_monitoring.get_iam_role_for_stream(kinesis_stream)
kinesis_client = aws_common.connect_kinesis(role=role)
try:
if downscale:
action = 'DOWNSCALE(-%s)' % len(downscale)
for shard_pair in downscale:
LOG.info('Merging shards %s and %s of Kinesis stream %s' %
(shard_pair.shard1.id, shard_pair.shard2.id, kinesis_stream.id))
kinesis_client.merge_shards(StreamName=kinesis_stream.id,
ShardToMerge=shard_pair.shard1.id, AdjacentShardToMerge=shard_pair.shard2.id)
elif upscale:
action = 'UPSCALE(+%s)' % len(upscale)
for shard in upscale:
LOG.info('Splitting shard %s of Kinesis stream %s' % (shard.id, kinesis_stream.id))
new_start_key = shard.center_key()
kinesis_client.split_shard(StreamName=kinesis_stream.id,
ShardToSplit=shard.id, NewStartingHashKey=new_start_key)
except Exception, e:
LOG.warning('Unable to re-scale stream %s: %s' % (kinesis_stream.id, e))
def disable_shard_monitoring(stream, metrics=['ALL']):
if not isinstance(stream, basestring):
stream = stream.id
role = get_iam_role_for_stream(stream)
kinesis_client = aws_common.connect_kinesis(role=role)
return kinesis_client.disable_enhanced_monitoring(StreamName=stream, ShardLevelMetrics=metrics)