def snapshot_volumes(self):
''' Loops through all EBS volumes and creates snapshots of them '''
log.info('Getting list of EBS volumes')
volumes = self.get_volumes_to_snapshot()
log.info('Found %d volumes', len(volumes))
for volume in volumes:
description_parts = [self._prefix]
description_parts.append(volume.id)
if volume.attach_data.instance_id:
description_parts.append(volume.attach_data.instance_id)
if volume.attach_data.device:
description_parts.append(volume.attach_data.device)
description = ' '.join(description_parts)
self._info(subject=_status.parse_status('snapshot_create', (volume.id, description)),
src_volume=volume.id,
src_tags=' '.join([':'.join(i) for i in volume.tags.items()]),
category='snapshots')
try:
snapshot = self._retryInCaseOfException(
volume.create_snapshot, description,
src_volume=volume.id,
category='snapshots',
type='alert',
severity='high')
if volume.tags:
snapshot.add_tags(self.remove_reserved_tags(volume.tags))
self._info(subject=_status.parse_status('snapshot_create_success', (snapshot.id, volume.id)),
src_volume=volume.id,
src_snapshot=snapshot.id,
src_tags=' '.join([':'.join(i) for i in snapshot.tags.items()]),
category='snapshots')
except BotoServerError, e:
if e.code == 'SnapshotLimitExceeded':
raise BackupMonkeyException('%s: %s' % (_status.parse_status('snapshot_create_error', volume.id), e.message),
subject=_status.parse_status('snapshot_create_error', volume.id),
body=e.message,
src_volume=volume.id,
src_tags=' '.join([':'.join(i) for i in self.remove_reserved_tags(volume.tags).items()]),
category='snapshots')
else:
log.error('%s: %s' % (_status.parse_status('snapshot_create_error', volume.id), e.message))
SplunkLogging.write(
subject=_status.parse_status('snapshot_create_error', volume.id),
body=e.message,
src_volume=volume.id,
src_tags=' '.join([':'.join(i) for i in self.remove_reserved_tags(volume.tags).items()]),
category='snapshots',
type='alarm',
severity='critical')
def _retryInCaseOfException(self, func, *args, **kwargs):
'''Retry with sleep in case of RequestLimitExceeded exception'''
result = None
for attempt in range(1, 6):
try:
result = func(*args)
except BotoServerError, e:
sleep_time = attempt + 5
log.error("Encountered Error %s on %s, waiting %d seconds then retrying", e.message, str(kwargs), sleep_time)
splunk_kwargs = {
'subject':_status.parse_status('retry_after_sleep', (str(attempt), str(sleep_time))),
'body':e.message
}
splunk_kwargs.update(kwargs)
SplunkLogging.write(**splunk_kwargs)
time.sleep(sleep_time)
continue
except Exception, e:
log.error("Encountered Error %s on %s", e.message, str(kwargs))
raise e
def _info(self, **kwargs):
log.info('%s: %s' % (kwargs['subject'], kwargs['body']) if 'subject' in kwargs and 'body' in kwargs else kwargs['subject'] if 'subject' in kwargs else None)
kwargs['severity'] = kwargs['severity'] if 'severity' in kwargs else 'informational'
kwargs['type'] = kwargs['type'] if 'type' in kwargs else 'event'
kwargs['src_region'] = self._region
SplunkLogging.write(**kwargs)
try:
self._retryInCaseOfException(
snapshot.delete,
src_snapshot=snapshot_id,
category='snapshots',
type='alert',
severity='high')
self._info(subject=_status.parse_status('snapshot_delete_success', (snapshot_id, snapshot_description)),
src_snapshot=snapshot_id,
category='snapshots')
except BotoServerError, e:
log.error('%s: %s' % (_status.parse_status('snapshot_delete_error', (snapshot_id, snapshot_description)), e.message))
SplunkLogging.write(
subject=_status.parse_status('snapshot_delete_error', (snapshot_id, snapshot_description)),
body=e.message,
src_snapshot=snapshot_id,
category='snapshots',
type='alarm',
severity='critical')
return True
def _retryInCaseOfException(self, func, *args, **kwargs):
'''Retry with sleep in case of RequestLimitExceeded exception'''
result = None
for attempt in range(1, 6):
try:
result = func(*args)
except BotoServerError, e:
sleep_time = attempt + 5
log.error("Encountered Error %s on %s, waiting %d seconds then retrying", e.message, str(kwargs), sleep_time)
splunk_kwargs = {
def __init__(self, *args, **kwargs): kwargs['type'] = 'alarm' kwargs['severity'] = 'critical' SplunkLogging.write(**kwargs) super(Exception, self).__init__(*args)
