def snapshot_volumes(self): ''' Loops through all EBS volumes and creates snapshots of them ''' log.info('Getting list of EBS volumes') volumes = self.get_volumes_to_snapshot() log.info('Found %d volumes', len(volumes)) for volume in volumes: description_parts = [self._prefix] description_parts.append(volume.id) if volume.attach_data.instance_id: description_parts.append(volume.attach_data.instance_id) if volume.attach_data.device: description_parts.append(volume.attach_data.device) description = ' '.join(description_parts) self._info(subject=_status.parse_status('snapshot_create', (volume.id, description)), src_volume=volume.id, src_tags=' '.join([':'.join(i) for i in volume.tags.items()]), category='snapshots') try: snapshot = self._retryInCaseOfException( volume.create_snapshot, description, src_volume=volume.id, category='snapshots', type='alert', severity='high') if volume.tags: snapshot.add_tags(self.remove_reserved_tags(volume.tags)) self._info(subject=_status.parse_status('snapshot_create_success', (snapshot.id, volume.id)), src_volume=volume.id, src_snapshot=snapshot.id, src_tags=' '.join([':'.join(i) for i in snapshot.tags.items()]), category='snapshots') except BotoServerError, e: if e.code == 'SnapshotLimitExceeded': raise BackupMonkeyException('%s: %s' % (_status.parse_status('snapshot_create_error', volume.id), e.message), subject=_status.parse_status('snapshot_create_error', volume.id), body=e.message, src_volume=volume.id, src_tags=' '.join([':'.join(i) for i in self.remove_reserved_tags(volume.tags).items()]), category='snapshots') else: log.error('%s: %s' % (_status.parse_status('snapshot_create_error', volume.id), e.message)) SplunkLogging.write( subject=_status.parse_status('snapshot_create_error', volume.id), body=e.message, src_volume=volume.id, src_tags=' '.join([':'.join(i) for i in self.remove_reserved_tags(volume.tags).items()]), category='snapshots', type='alarm', severity='critical')
def _retryInCaseOfException(self, func, *args, **kwargs): '''Retry with sleep in case of RequestLimitExceeded exception''' result = None for attempt in range(1, 6): try: result = func(*args) except BotoServerError, e: sleep_time = attempt + 5 log.error("Encountered Error %s on %s, waiting %d seconds then retrying", e.message, str(kwargs), sleep_time) splunk_kwargs = { 'subject':_status.parse_status('retry_after_sleep', (str(attempt), str(sleep_time))), 'body':e.message } splunk_kwargs.update(kwargs) SplunkLogging.write(**splunk_kwargs) time.sleep(sleep_time) continue except Exception, e: log.error("Encountered Error %s on %s", e.message, str(kwargs)) raise e
def _info(self, **kwargs): log.info('%s: %s' % (kwargs['subject'], kwargs['body']) if 'subject' in kwargs and 'body' in kwargs else kwargs['subject'] if 'subject' in kwargs else None) kwargs['severity'] = kwargs['severity'] if 'severity' in kwargs else 'informational' kwargs['type'] = kwargs['type'] if 'type' in kwargs else 'event' kwargs['src_region'] = self._region SplunkLogging.write(**kwargs)
try: self._retryInCaseOfException( snapshot.delete, src_snapshot=snapshot_id, category='snapshots', type='alert', severity='high') self._info(subject=_status.parse_status('snapshot_delete_success', (snapshot_id, snapshot_description)), src_snapshot=snapshot_id, category='snapshots') except BotoServerError, e: log.error('%s: %s' % (_status.parse_status('snapshot_delete_error', (snapshot_id, snapshot_description)), e.message)) SplunkLogging.write( subject=_status.parse_status('snapshot_delete_error', (snapshot_id, snapshot_description)), body=e.message, src_snapshot=snapshot_id, category='snapshots', type='alarm', severity='critical') return True def _retryInCaseOfException(self, func, *args, **kwargs): '''Retry with sleep in case of RequestLimitExceeded exception''' result = None for attempt in range(1, 6): try: result = func(*args) except BotoServerError, e: sleep_time = attempt + 5 log.error("Encountered Error %s on %s, waiting %d seconds then retrying", e.message, str(kwargs), sleep_time) splunk_kwargs = {
def __init__(self, *args, **kwargs): kwargs['type'] = 'alarm' kwargs['severity'] = 'critical' SplunkLogging.write(**kwargs) super(Exception, self).__init__(*args)