def get_consumed_read_units_percent(table_name, time_frame=300): """ Returns the number of consumed read units in percent :type table_name: str :param table_name: Name of the DynamoDB table :type time_frame: int :param time_frame: How many seconds to look at :returns: int -- Number of consumed reads """ try: metrics = __get_aws_metric( table_name, time_frame, 'ConsumedReadCapacityUnits') except BotoServerError: raise if metrics: consumed_read_units = int( math.ceil(float(metrics[0]['Sum'])/float(time_frame))) else: consumed_read_units = 0 try: consumed_read_units_percent = int( math.ceil( float(consumed_read_units) / float(dynamodb.get_provisioned_table_read_units(table_name)) * 100)) except JSONResponseError: raise logger.info('{0} - Consumed read units: {1:d}%'.format( table_name, consumed_read_units_percent)) return consumed_read_units_percent
def get_consumed_read_units_percent(table_name, time_frame=300): """ Returns the number of consumed read units in percent :type table_name: str :param table_name: Name of the DynamoDB table :type time_frame: int :param time_frame: How many seconds to look at :returns: int -- Number of consumed reads """ metrics = cloudwatch_connection.get_metric_statistics( period=time_frame, start_time=datetime.utcnow()-timedelta(minutes=10, seconds=time_frame), end_time=datetime.utcnow()-timedelta(minutes=10), metric_name='ConsumedReadCapacityUnits', namespace='AWS/DynamoDB', statistics=['Sum'], dimensions={'TableName': table_name}, unit='Count') if metrics: consumed_read_units = int( math.ceil(float(metrics[0]['Sum'])/float(time_frame))) else: consumed_read_units = 0 consumed_read_units_percent = int( math.ceil( float(consumed_read_units) / float(dynamodb.get_provisioned_table_read_units(table_name)) * 100)) logger.info('{0} - Consumed read units: {1:d}%'.format( table_name, consumed_read_units_percent)) return consumed_read_units_percent
def __ensure_provisioning_reads(table_name, key_name): """ Ensure that provisioning is correct :type table_name: str :param table_name: Name of the DynamoDB table :type key_name: str :param key_name: Configuration option key name :returns: (bool, int) -- update_needed, updated_read_units """ if not get_table_option(key_name, 'enable_reads_autoscaling'): logger.info( '{0} - Autoscaling of reads has been disabled'.format(table_name)) return False, dynamodb.get_provisioned_table_read_units(table_name) update_needed = False try: updated_read_units = dynamodb.get_provisioned_table_read_units( table_name) consumed_read_units_percent = \ table_stats.get_consumed_read_units_percent(table_name) throttled_read_count = \ table_stats.get_throttled_read_event_count(table_name) reads_upper_threshold = \ get_table_option(key_name, 'reads_upper_threshold') reads_lower_threshold = \ get_table_option(key_name, 'reads_lower_threshold') throttled_reads_upper_threshold = \ get_table_option(key_name, 'throttled_reads_upper_threshold') increase_reads_with = \ get_table_option(key_name, 'increase_reads_with') increase_reads_unit = \ get_table_option(key_name, 'increase_reads_unit') decrease_reads_with = \ get_table_option(key_name, 'decrease_reads_with') decrease_reads_unit = \ get_table_option(key_name, 'decrease_reads_unit') max_provisioned_reads = \ get_table_option(key_name, 'max_provisioned_reads') except JSONResponseError: raise except BotoServerError: raise if (consumed_read_units_percent == 0 and not get_table_option( key_name, 'allow_scaling_down_reads_on_0_percent')): print('1') logger.info( '{0} - Scaling down reads is not done when usage is at 0%'.format( table_name)) elif consumed_read_units_percent >= reads_upper_threshold: if increase_reads_unit == 'percent': updated_provisioning = calculators.increase_reads_in_percent( updated_read_units, increase_reads_with, key_name, table_name) else: updated_provisioning = calculators.increase_reads_in_units( updated_read_units, increase_reads_with, key_name, table_name) if updated_read_units != updated_provisioning: update_needed = True updated_read_units = updated_provisioning elif throttled_read_count > throttled_reads_upper_threshold: if throttled_reads_upper_threshold > 0: if increase_reads_unit == 'percent': updated_provisioning = calculators.increase_reads_in_percent( updated_read_units, increase_reads_with, key_name, table_name) else: updated_provisioning = calculators.increase_reads_in_units( updated_read_units, increase_reads_with, key_name, table_name) if updated_read_units != updated_provisioning: update_needed = True updated_read_units = updated_provisioning elif consumed_read_units_percent <= reads_lower_threshold: if decrease_reads_unit == 'percent': updated_provisioning = calculators.decrease_reads_in_percent( updated_read_units, decrease_reads_with, key_name, table_name) else: updated_provisioning = calculators.decrease_reads_in_units( updated_read_units, decrease_reads_with, key_name, table_name) if updated_read_units != updated_provisioning: update_needed = True updated_read_units = updated_provisioning if max_provisioned_reads: if (int(updated_read_units) > int(max_provisioned_reads)): update_needed = True updated_read_units = int(max_provisioned_reads) logger.info( 'Will not increase writes over max-provisioned-reads ' 'limit ({0} writes)'.format(updated_read_units)) return update_needed, int(updated_read_units)
def __update_throughput(table_name, read_units, write_units, key_name): """ Update throughput on the DynamoDB table :type table_name: str :param table_name: Name of the DynamoDB table :type read_units: int :param read_units: New read unit provisioning :type write_units: int :param write_units: New write unit provisioning :type key_name: str :param key_name: Configuration option key name """ try: current_ru = dynamodb.get_provisioned_table_read_units(table_name) current_wu = dynamodb.get_provisioned_table_write_units(table_name) except JSONResponseError: raise # Check that we are in the right time frame if get_table_option(key_name, 'maintenance_windows'): if (not __is_maintenance_window(table_name, get_table_option( key_name, 'maintenance_windows'))): logger.warning( '{0} - Current time is outside maintenance window'.format( table_name)) return else: logger.info( '{0} - Current time is within maintenance window'.format( table_name)) # Check table status try: table_status = dynamodb.get_table_status(table_name) except JSONResponseError: raise logger.debug('{0} - Table status is {1}'.format(table_name, table_status)) if table_status != 'ACTIVE': logger.warning( '{0} - Not performing throughput changes when table ' 'is {1}'.format(table_name, table_status)) return # If this setting is True, we will only scale down when # BOTH reads AND writes are low if get_table_option(key_name, 'always_decrease_rw_together'): read_units, write_units = __calculate_always_decrease_rw_values( table_name, read_units, current_ru, write_units, current_wu) if read_units == current_ru and write_units == current_wu: logger.info('{0} - No changes to perform'.format(table_name)) return if not get_global_option('dry_run'): dynamodb.update_table_provisioning( table_name, key_name, int(read_units), int(write_units))
def __ensure_provisioning_reads(table_name, key_name): """ Ensure that provisioning is correct :type table_name: str :param table_name: Name of the DynamoDB table :type key_name: str :param key_name: Configuration option key name :returns: (bool, int) -- update_needed, updated_read_units """ update_needed = False try: updated_read_units = dynamodb.get_provisioned_table_read_units( table_name) consumed_read_units_percent = table_stats.\ get_consumed_read_units_percent(table_name) except JSONResponseError: raise throttled_read_count = table_stats.get_throttled_read_event_count( table_name) if (consumed_read_units_percent == 0 and not get_table_option( key_name, 'allow_scaling_down_reads_on_0_percent')): logger.info( '{0} - Scaling down reads is not done when usage is at 0%'.format( table_name)) elif (consumed_read_units_percent >= get_table_option(key_name, 'reads_upper_threshold')): if get_table_option(key_name, 'increase_reads_unit') == 'percent': updated_provisioning = calculators.increase_reads_in_percent( updated_read_units, get_table_option(key_name, 'increase_reads_with'), key_name, table_name) else: updated_provisioning = calculators.increase_reads_in_units( updated_read_units, get_table_option(key_name, 'increase_reads_with'), key_name, table_name) if updated_read_units != updated_provisioning: update_needed = True updated_read_units = updated_provisioning elif (throttled_read_count >= get_table_option(key_name, 'throttled_reads_upper_threshold')): if get_table_option(key_name, 'increase_reads_unit') == 'percent': updated_provisioning = calculators.increase_reads_in_percent( updated_read_units, get_table_option(key_name, 'increase_reads_with'), key_name, table_name) else: updated_provisioning = calculators.increase_reads_in_units( updated_read_units, get_table_option(key_name, 'increase_reads_with'), key_name, table_name) if updated_read_units != updated_provisioning: update_needed = True updated_read_units = updated_provisioning elif (consumed_read_units_percent <= get_table_option(key_name, 'reads_lower_threshold')): if get_table_option(key_name, 'decrease_reads_unit') == 'percent': updated_provisioning = calculators.decrease_reads_in_percent( updated_read_units, get_table_option(key_name, 'decrease_reads_with'), key_name, table_name) else: updated_provisioning = calculators.decrease_reads_in_units( updated_read_units, get_table_option(key_name, 'decrease_reads_with'), key_name, table_name) if updated_read_units != updated_provisioning: update_needed = True updated_read_units = updated_provisioning if get_table_option(key_name, 'max_provisioned_reads'): if (int(updated_read_units) > int(get_table_option(key_name, 'max_provisioned_reads'))): update_needed = True updated_read_units = int( get_table_option(key_name, 'max_provisioned_reads')) logger.info( 'Will not increase writes over max-provisioned-reads ' 'limit ({0} writes)'.format(updated_read_units)) return update_needed, int(updated_read_units)