def rsync_snapshot(src_region_name, snapshot_id, dst_region_name,
src_inst=None, dst_inst=None):
"""Duplicate the snapshot into dst_region.
src_region_name, dst_region_name
Amazon region names. Allowed to be contracted, e.g.
`ap-southeast-1` will be recognized in `ap-south` or even
`ap-s`;
snapshot_id
snapshot to duplicate;
src_inst, dst_inst
will be used instead of creating new for temporary.
You'll need to open port 60000 for encrypted instances replication."""
src_conn = get_region_conn(src_region_name)
src_snap = src_conn.get_all_snapshots([snapshot_id])[0]
dst_conn = get_region_conn(dst_region_name)
_src_device = get_snap_device(src_snap)
_src_dev = re.match(r'^/dev/sda$', _src_device) # check for encryption
if _src_dev:
encr = True
logger.info('Found traces of encryption')
else:
encr = None
info = 'Going to transmit {snap.volume_size} GiB {snap} {snap.description}'
if src_snap.tags.get('Name'):
info += ' of {name}'
info += ' from {snap.region} to {dst}'
logger.info(info.format(snap=src_snap, dst=dst_conn.region,
name=src_snap.tags.get('Name')))
dst_snaps = dst_conn.get_all_snapshots(owner='self')
dst_snaps = [snp for snp in dst_snaps if not snp.status == 'error']
src_vol = get_snap_vol(src_snap)
vol_snaps = [snp for snp in dst_snaps if get_snap_vol(snp) == src_vol]
if vol_snaps:
dst_snap = sorted(vol_snaps, key=get_snap_time)[-1]
if get_snap_time(dst_snap) >= get_snap_time(src_snap):
kwargs = dict(src=src_snap, dst=dst_snap, dst_reg=dst_conn.region)
logger.info('Stepping over {src} - it\'s not newer than {dst} '
'{dst.description} in {dst_reg}'.format(**kwargs))
return
else:
dst_snap = create_empty_snapshot(dst_conn.region, src_snap.volume_size)
with nested(attach_snapshot(src_snap, inst=src_inst, encr=encr),
attach_snapshot(dst_snap, inst=dst_inst, encr=encr)) as (
(src_vol, src_mnt), (dst_vol, dst_mnt)):
update_snap(src_vol, src_mnt, dst_vol, dst_mnt, encr,
delete_old=not vol_snaps) # Delete only empty snapshots.
def get_relevant_snapshots(
conn, tag_name=DEFAULT_TAG_NAME, tag_value=DEFAULT_TAG_VALUE,
native_only=True, filters={'status': SNAP_STATUSES}):
"""Returns snapshots with proper description."""
if tag_name and tag_value:
filters.update({'tag:{0}'.format(tag_name): tag_value})
snaps = conn.get_all_snapshots(owner='self', filters=filters)
is_described = lambda snap: get_snap_vol(snap) and get_snap_time(snap)
snaps = [snp for snp in snaps if is_described(snp)]
if native_only:
is_native = lambda snp, reg: get_descr_attr(snp, 'Region') == reg.name
snaps = [snp for snp in snaps if is_native(snp, conn.region)]
return snaps
def get_relevant_snapshots(
conn, tag_name=DEFAULT_TAG_NAME, tag_value=DEFAULT_TAG_VALUE,
native_only=True, filters={'status': SNAP_STATUSES}):
"""Returns snapshots with proper description."""
if tag_name and tag_value:
filters.update({'tag:{0}'.format(tag_name): tag_value})
snaps = conn.get_all_snapshots(owner='self', filters=filters)
is_described = lambda snap: get_snap_vol(snap) and get_snap_time(snap)
snaps = [snp for snp in snaps if is_described(snp)]
if native_only:
is_native = lambda snp, reg: get_descr_attr(snp, 'Region') == reg.name
snaps = [snp for snp in snaps if is_native(snp, conn.region)]
return snaps
def rsync_region(src_region_name, dst_region_name, tag_name=None,
tag_value=None, native_only=True):
"""Duplicates latest snapshots with given tag into dst_region.
src_region_name, dst_region_name
every latest volume snapshot from src_region will be rsynced
to the dst_region;
tag_name, tag_value
snapshots will be filtered by tag. Tag will be fetched from
config by default, may be configured per region;
native_only
sync only snapshots, created in the src_region_name. True by
default."""
src_conn = get_region_conn(src_region_name)
dst_conn = get_region_conn(dst_region_name)
tag_name = tag_name or config.get(src_conn.region.name, 'TAG_NAME')
tag_value = tag_value or config.get(src_conn.region.name, 'TAG_VALUE')
filters = {'tag-key': tag_name, 'tag-value': tag_value}
snaps = src_conn.get_all_snapshots(owner='self', filters=filters)
snaps = [snp for snp in snaps if not snp.status == 'error']
_is_described = lambda snap: get_snap_vol(snap) and get_snap_time(snap)
snaps = [snp for snp in snaps if _is_described(snp)]
if native_only:
def is_native(snap, region):
return get_descr_attr(snap, 'Region') == region.name
snaps = [snp for snp in snaps if is_native(snp, src_conn.region)]
with nested(create_temp_inst(src_conn.region),
create_temp_inst(dst_conn.region)) as (src_inst, dst_inst):
snaps = sorted(snaps, key=get_snap_vol) # Prepare for grouping.
for vol, vol_snaps in groupby(snaps, get_snap_vol):
latest_snap = sorted(vol_snaps, key=get_snap_time)[-1]
for inst in src_inst, dst_inst:
logger.debug('Rebooting {0} in {0.region} '
'to refresh attachments'.format(inst))
inst.reboot()
args = (src_region_name, latest_snap.id, dst_region_name, src_inst,
dst_inst)
try:
rsync_snapshot(*args)
except:
logger.exception('rsync of {1} from {0} to {2} failed'.format(
*args))
def _trim_snapshots(region, dry_run=False):
"""Delete snapshots back in time in logarithmic manner.
dry_run
just print snapshot to be deleted.
Modified version of the `boto.ec2.connection.trim_snapshots
<http://pypi.python.org/pypi/boto/2.0>_`. Licensed under MIT license
by Mitch Garnaat, 2011."""
hourly_backups = config.getint('purge_backups', 'HOURLY_BACKUPS')
daily_backups = config.getint('purge_backups', 'DAILY_BACKUPS')
weekly_backups = config.getint('purge_backups', 'WEEKLY_BACKUPS')
monthly_backups = config.getint('purge_backups', 'MONTHLY_BACKUPS')
quarterly_backups = config.getint('purge_backups', 'QUARTERLY_BACKUPS')
yearly_backups = config.getint('purge_backups', 'YEARLY_BACKUPS')
# work with UTC time, which is what the snapshot start time is reported in
now = datetime.utcnow()
last_hour = datetime(now.year, now.month, now.day, now.hour)
last_midnight = datetime(now.year, now.month, now.day)
last_sunday = datetime(now.year, now.month,
now.day) - timedelta(days=(now.weekday() + 1) % 7)
last_month = datetime.now() - relativedelta(months=1)
last_year = datetime.now() - relativedelta(years=1)
other_years = datetime.now() - relativedelta(years=2)
start_of_month = datetime(now.year, now.month, 1)
target_backup_times = []
# there are no snapshots older than 1/1/2000
oldest_snapshot_date = datetime(2000, 1, 1)
for hour in range(0, hourly_backups):
target_backup_times.append(last_hour - timedelta(hours=hour))
for day in range(0, daily_backups):
target_backup_times.append(last_midnight - timedelta(days=day))
for week in range(0, weekly_backups):
target_backup_times.append(last_sunday - timedelta(weeks=week))
for month in range(0, monthly_backups):
target_backup_times.append(last_month - relativedelta(months=month))
for quart in range(0, quarterly_backups):
target_backup_times.append(last_year - relativedelta(months=4 * quart))
for year in range(0, yearly_backups):
target_backup_times.append(other_years - relativedelta(years=year))
one_day = timedelta(days=1)
while start_of_month > oldest_snapshot_date:
# append the start of the month to the list of snapshot dates to save:
target_backup_times.append(start_of_month)
# there's no timedelta setting for one month, so instead:
# decrement the day by one,
#so we go to the final day of the previous month...
start_of_month -= one_day
# ... and then go to the first day of that previous month:
start_of_month = datetime(start_of_month.year,
start_of_month.month, 1)
temp = []
for t in target_backup_times:
if temp.__contains__(t) == False:
temp.append(t)
target_backup_times = temp
target_backup_times.reverse() # make the oldest date first
# get all the snapshots, sort them by date and time,
#and organize them into one array for each volume:
conn = get_region_conn(region.name)
all_snapshots = conn.get_all_snapshots(owner='self')
# oldest first
all_snapshots.sort(cmp=lambda x, y: cmp(x.start_time, y.start_time))
snaps_for_each_volume = {}
for snap in all_snapshots:
# the snapshot name and the volume name are the same.
# The snapshot name is set from the volume
# name at the time the snapshot is taken
volume_name = get_snap_vol(snap)
if volume_name:
# only examine snapshots that have a volume name
snaps_for_volume = snaps_for_each_volume.get(volume_name)
if not snaps_for_volume:
snaps_for_volume = []
snaps_for_each_volume[volume_name] = snaps_for_volume
snaps_for_volume.append(snap)
# Do a running comparison of snapshot dates to desired time periods,
# keeping the oldest snapshot in each
# time period and deleting the rest:
for volume_name in snaps_for_each_volume:
snaps = snaps_for_each_volume[volume_name]
snaps = snaps[:-1]
# never delete the newest snapshot, so remove it from consideration
time_period_num = 0
snap_found_for_this_time_period = False
for snap in snaps:
check_this_snap = True
while (check_this_snap and
time_period_num < target_backup_times.__len__()):
if get_snap_time(snap) < target_backup_times[time_period_num]:
# the snap date is before the cutoff date.
# Figure out if it's the first snap in this
# date range and act accordingly
#(since both date the date ranges and the snapshots
# are sorted chronologically, we know this
#snapshot isn't in an earlier date range):
if snap_found_for_this_time_period:
if not snap.tags.get('preserve_snapshot'):
if dry_run:
logger.info('Dry-trimmed {0} {1} from {2}'
.format(snap, snap.description,
snap.start_time))
else:
# as long as the snapshot wasn't marked with
# the 'preserve_snapshot' tag, delete it:
try:
conn.delete_snapshot(snap.id)
except EC2ResponseError as err:
logger.exception(str(err))
else:
logger.info('Trimmed {0} {1} from {2}'
.format(snap, snap.description,
snap.start_time))
# go on and look at the next snapshot,
# leaving the time period alone
else:
# this was the first snapshot found for this time
# period. Leave it alone and look at the next snapshot:
snap_found_for_this_time_period = True
check_this_snap = False
else:
# the snap is after the cutoff date.
# Check it against the next cutoff date
time_period_num += 1
snap_found_for_this_time_period = False
def create_ami(region, snap_id, force=None, root_dev='/dev/sda1', zone_name=None,
default_arch=None, default_type='t1.micro', security_groups=''):
"""
Creates AMI image from given snapshot.
Force option removes prompt request and creates new instance from
created ami image.
region, snap_id
specify snapshot to be processed. Snapshot description in json
format will be used to restore instance with same parameters.
Will automaticaly process snapshots for same instance with near
time (10 minutes or shorter), but for other devices (/dev/sdb,
/dev/sdc, etc);
force
Run instance from ami after creation without confirmation. To
enable set value to "RUN";
default_arch
architecture to use if not mentioned in snapshot description;
default_type
instance type to use if not mentioned in snapshot description.
Used only if ``force`` is "RUN";
security_groups
list of AWS Security Groups names formatted as string separated
with semicolon ';'. Used only if ``force`` is "RUN".
"""
conn = get_region_conn(region)
snap = conn.get_all_snapshots(snapshot_ids=[snap_id, ])[0]
instance_id = get_snap_instance(snap)
_device = get_snap_device(snap)
snaps = conn.get_all_snapshots(owner='self')
snapshots = [snp for snp in snaps if
get_snap_instance(snp) == instance_id and
get_snap_device(snp) != _device and
abs(get_snap_time(snap) - get_snap_time(snp)) <= timedelta(minutes=10)]
snapshot = sorted(snapshots, key=get_snap_time,
reverse=True) if snapshots else None
# setup for building an EBS boot snapshot
default_arch = default_arch or config.get('DEFAULT', 'ARCHITECTURE')
arch = get_descr_attr(snap, 'Arch') or default_arch
kernel = config.get(conn.region.name, 'KERNEL' + arch.upper())
dev = re.match(r'^/dev/sda$', _device) # if our instance encrypted
if dev:
kernel = config.get(conn.region.name, 'KERNEL_ENCR_' + arch.upper())
ebs = EBSBlockDeviceType()
ebs.snapshot_id = snap_id
ebs.delete_on_termination = True
block_map = BlockDeviceMapping()
block_map[_device] = ebs
sdb = BlockDeviceType()
sdb.ephemeral_name = 'ephemeral0'
block_map['/dev/sdb'] = sdb
if snapshot:
for s in snapshot:
s_dev = get_snap_device(s)
s_ebs = EBSBlockDeviceType()
s_ebs.delete_on_termination = True
s_ebs.snapshot_id = s.id
block_map[s_dev] = s_ebs
name = 'Created {0} using access key {1}'.format(timestamp(),
conn.access_key)
name = name.replace(":", ".").replace(" ", "_")
# create the new AMI all options from snap JSON description:
wait_for(snap, '100%', limit=SNAP_TIME)
result = conn.register_image(
name=name,
description=snap.description,
architecture=get_descr_attr(snap, 'Arch') or default_arch,
root_device_name=get_descr_attr(snap, 'Root_dev_name') or root_dev,
block_device_map=block_map, kernel_id=kernel)
sleep(2)
image = conn.get_all_images(image_ids=[result, ])[0]
wait_for(image, 'available', limit=10 * 60)
add_tags(image, snap.tags)
logger.info('The new AMI ID = {0}'.format(result))
new_instance = None
if force == 'RUN':
instance_type = get_descr_attr(snap, 'Type') or default_type
new_instance = launch_instance_from_ami(
region, image.id, inst_type=instance_type,
security_groups=security_groups, zone_name=zone_name)
return image, new_instance
def _trim_snapshots(region, dry_run=False):
"""Delete snapshots back in time in logarithmic manner.
dry_run
just print snapshot to be deleted.
Modified version of the `boto.ec2.connection.trim_snapshots
<http://pypi.python.org/pypi/boto/2.0>_`. Licensed under MIT license
by Mitch Garnaat, 2011."""
hourly_backups = config.getint('purge_backups', 'HOURLY_BACKUPS')
daily_backups = config.getint('purge_backups', 'DAILY_BACKUPS')
weekly_backups = config.getint('purge_backups', 'WEEKLY_BACKUPS')
monthly_backups = config.getint('purge_backups', 'MONTHLY_BACKUPS')
quarterly_backups = config.getint('purge_backups', 'QUARTERLY_BACKUPS')
yearly_backups = config.getint('purge_backups', 'YEARLY_BACKUPS')
# work with UTC time, which is what the snapshot start time is reported in
now = datetime.utcnow()
last_hour = datetime(now.year, now.month, now.day, now.hour)
last_midnight = datetime(now.year, now.month, now.day)
last_sunday = datetime(now.year, now.month,
now.day) - timedelta(days=(now.weekday() + 1) % 7)
last_month = datetime.now() - relativedelta(months=1)
last_year = datetime.now() - relativedelta(years=1)
other_years = datetime.now() - relativedelta(years=2)
start_of_month = datetime(now.year, now.month, 1)
target_backup_times = []
# there are no snapshots older than 1/1/2000
oldest_snapshot_date = datetime(2000, 1, 1)
for hour in range(0, hourly_backups):
target_backup_times.append(last_hour - timedelta(hours=hour))
for day in range(0, daily_backups):
target_backup_times.append(last_midnight - timedelta(days=day))
for week in range(0, weekly_backups):
target_backup_times.append(last_sunday - timedelta(weeks=week))
for month in range(0, monthly_backups):
target_backup_times.append(last_month - relativedelta(months=month))
for quart in range(0, quarterly_backups):
target_backup_times.append(last_year - relativedelta(months=4 * quart))
for year in range(0, yearly_backups):
target_backup_times.append(other_years - relativedelta(years=year))
one_day = timedelta(days=1)
while start_of_month > oldest_snapshot_date:
# append the start of the month to the list of snapshot dates to save:
target_backup_times.append(start_of_month)
# there's no timedelta setting for one month, so instead:
# decrement the day by one,
#so we go to the final day of the previous month...
start_of_month -= one_day
# ... and then go to the first day of that previous month:
start_of_month = datetime(start_of_month.year,
start_of_month.month, 1)
temp = []
for t in target_backup_times:
if temp.__contains__(t) == False:
temp.append(t)
target_backup_times = temp
target_backup_times.reverse() # make the oldest date first
# get all the snapshots, sort them by date and time,
#and organize them into one array for each volume:
conn = get_region_conn(region.name)
all_snapshots = conn.get_all_snapshots(owner='self')
# oldest first
all_snapshots.sort(cmp=lambda x, y: cmp(x.start_time, y.start_time))
snaps_for_each_volume = {}
for snap in all_snapshots:
# the snapshot name and the volume name are the same.
# The snapshot name is set from the volume
# name at the time the snapshot is taken
volume_name = get_snap_vol(snap)
if volume_name:
# only examine snapshots that have a volume name
snaps_for_volume = snaps_for_each_volume.get(volume_name)
if not snaps_for_volume:
snaps_for_volume = []
snaps_for_each_volume[volume_name] = snaps_for_volume
snaps_for_volume.append(snap)
# Do a running comparison of snapshot dates to desired time periods,
# keeping the oldest snapshot in each
# time period and deleting the rest:
for volume_name in snaps_for_each_volume:
snaps = snaps_for_each_volume[volume_name]
snaps = snaps[:-1]
# never delete the newest snapshot, so remove it from consideration
time_period_num = 0
snap_found_for_this_time_period = False
for snap in snaps:
check_this_snap = True
while (check_this_snap and
time_period_num < target_backup_times.__len__()):
if get_snap_time(snap) < target_backup_times[time_period_num]:
# the snap date is before the cutoff date.
# Figure out if it's the first snap in this
# date range and act accordingly
#(since both date the date ranges and the snapshots
# are sorted chronologically, we know this
#snapshot isn't in an earlier date range):
if snap_found_for_this_time_period:
if not snap.tags.get('preserve_snapshot'):
if dry_run:
logger.info('Dry-trimmed {0} {1} from {2}'
.format(snap, snap.description,
snap.start_time))
else:
# as long as the snapshot wasn't marked with
# the 'preserve_snapshot' tag, delete it:
try:
conn.delete_snapshot(snap.id)
except EC2ResponseError as err:
logger.exception(str(err))
else:
logger.info('Trimmed {0} {1} from {2}'
.format(snap, snap.description,
snap.start_time))
# go on and look at the next snapshot,
# leaving the time period alone
else:
# this was the first snapshot found for this time
# period. Leave it alone and look at the next snapshot:
snap_found_for_this_time_period = True
check_this_snap = False
else:
# the snap is after the cutoff date.
# Check it against the next cutoff date
time_period_num += 1
snap_found_for_this_time_period = False
def create_ami(region,
snap_id,
force=None,
root_dev='/dev/sda1',
zone_name=None,
default_arch=None,
default_type='t1.micro',
security_groups=''):
"""
Creates AMI image from given snapshot.
Force option removes prompt request and creates new instance from
created ami image.
region, snap_id
specify snapshot to be processed. Snapshot description in json
format will be used to restore instance with same parameters.
Will automaticaly process snapshots for same instance with near
time (10 minutes or shorter), but for other devices (/dev/sdb,
/dev/sdc, etc);
force
Run instance from ami after creation without confirmation. To
enable set value to "RUN";
default_arch
architecture to use if not mentioned in snapshot description;
default_type
instance type to use if not mentioned in snapshot description.
Used only if ``force`` is "RUN";
security_groups
list of AWS Security Groups names formatted as string separated
with semicolon ';'. Used only if ``force`` is "RUN".
"""
conn = get_region_conn(region)
snap = conn.get_all_snapshots(snapshot_ids=[
snap_id,
])[0]
instance_id = get_snap_instance(snap)
_device = get_snap_device(snap)
snaps = conn.get_all_snapshots(owner='self')
snapshots = [
snp for snp in snaps if get_snap_instance(snp) == instance_id
and get_snap_device(snp) != _device and
abs(get_snap_time(snap) - get_snap_time(snp)) <= timedelta(minutes=10)
]
snapshot = sorted(snapshots, key=get_snap_time,
reverse=True) if snapshots else None
# setup for building an EBS boot snapshot
default_arch = default_arch or config.get('DEFAULT', 'ARCHITECTURE')
arch = get_descr_attr(snap, 'Arch') or default_arch
kernel = config.get(conn.region.name, 'KERNEL' + arch.upper())
dev = re.match(r'^/dev/sda$', _device) # if our instance encrypted
if dev:
kernel = config.get(conn.region.name, 'KERNEL_ENCR_' + arch.upper())
ebs = EBSBlockDeviceType()
ebs.snapshot_id = snap_id
ebs.delete_on_termination = True
block_map = BlockDeviceMapping()
block_map[_device] = ebs
sdb = BlockDeviceType()
sdb.ephemeral_name = 'ephemeral0'
block_map['/dev/sdb'] = sdb
if snapshot:
for s in snapshot:
s_dev = get_snap_device(s)
s_ebs = EBSBlockDeviceType()
s_ebs.delete_on_termination = True
s_ebs.snapshot_id = s.id
block_map[s_dev] = s_ebs
name = 'Created {0} using access key {1}'.format(timestamp(),
conn.access_key)
name = name.replace(":", ".").replace(" ", "_")
# create the new AMI all options from snap JSON description:
wait_for(snap, '100%', limit=SNAP_TIME)
result = conn.register_image(
name=name,
description=snap.description,
architecture=get_descr_attr(snap, 'Arch') or default_arch,
root_device_name=get_descr_attr(snap, 'Root_dev_name') or root_dev,
block_device_map=block_map,
kernel_id=kernel)
sleep(2)
image = conn.get_all_images(image_ids=[
result,
])[0]
wait_for(image, 'available', limit=10 * 60)
add_tags(image, snap.tags)
logger.info('The new AMI ID = {0}'.format(result))
info = ('\nEnter RUN if you want to launch instance using '
'just created {0}: '.format(image))
new_instance = None
if force == 'RUN' or raw_input(info).strip() == 'RUN':
instance_type = get_descr_attr(snap, 'Type') or default_type
new_instance = launch_instance_from_ami(
region,
image.id,
inst_type=instance_type,
security_groups=security_groups,
zone_name=zone_name)
return image, new_instance
