def rsync_snapshot(src_region_name, snapshot_id, dst_region_name,
src_inst=None, dst_inst=None):
"""Duplicate the snapshot into dst_region.
src_region_name, dst_region_name
Amazon region names. Allowed to be contracted, e.g.
`ap-southeast-1` will be recognized in `ap-south` or even
`ap-s`;
snapshot_id
snapshot to duplicate;
src_inst, dst_inst
will be used instead of creating new for temporary.
You'll need to open port 60000 for encrypted instances replication."""
src_conn = get_region_conn(src_region_name)
src_snap = src_conn.get_all_snapshots([snapshot_id])[0]
dst_conn = get_region_conn(dst_region_name)
_src_device = get_snap_device(src_snap)
_src_dev = re.match(r'^/dev/sda$', _src_device) # check for encryption
if _src_dev:
encr = True
logger.info('Found traces of encryption')
else:
encr = None
info = 'Going to transmit {snap.volume_size} GiB {snap} {snap.description}'
if src_snap.tags.get('Name'):
info += ' of {name}'
info += ' from {snap.region} to {dst}'
logger.info(info.format(snap=src_snap, dst=dst_conn.region,
name=src_snap.tags.get('Name')))
dst_snaps = dst_conn.get_all_snapshots(owner='self')
dst_snaps = [snp for snp in dst_snaps if not snp.status == 'error']
src_vol = get_snap_vol(src_snap)
vol_snaps = [snp for snp in dst_snaps if get_snap_vol(snp) == src_vol]
if vol_snaps:
dst_snap = sorted(vol_snaps, key=get_snap_time)[-1]
if get_snap_time(dst_snap) >= get_snap_time(src_snap):
kwargs = dict(src=src_snap, dst=dst_snap, dst_reg=dst_conn.region)
logger.info('Stepping over {src} - it\'s not newer than {dst} '
'{dst.description} in {dst_reg}'.format(**kwargs))
return
else:
dst_snap = create_empty_snapshot(dst_conn.region, src_snap.volume_size)
with nested(attach_snapshot(src_snap, inst=src_inst, encr=encr),
attach_snapshot(dst_snap, inst=dst_inst, encr=encr)) as (
(src_vol, src_mnt), (dst_vol, dst_mnt)):
update_snap(src_vol, src_mnt, dst_vol, dst_mnt, encr,
delete_old=not vol_snaps) # Delete only empty snapshots.
def rsync_snapshot(src_region_name, snapshot_id, dst_region_name,
src_inst=None, dst_inst=None, force=False):
"""Duplicate the snapshot into dst_region.
src_region_name, dst_region_name
Amazon region names. Allowed to be contracted, e.g.
`ap-southeast-1` will be recognized in `ap-south` or even
`ap-s`;
snapshot_id
snapshot to duplicate;
src_inst, dst_inst
will be used instead of creating new for temporary;
force
rsync snapshot even if newer version exist.
You'll need to open port 60000 for encrypted instances replication."""
src_conn = get_region_conn(src_region_name)
src_snap = src_conn.get_all_snapshots([snapshot_id])[0]
dst_conn = get_region_conn(dst_region_name)
_src_device = get_snap_device(src_snap)
_src_dev = re.match(r'^/dev/sda$', _src_device) # check for encryption
if _src_dev:
encr = True
logger.info('Found traces of encryption')
else:
encr = None
info = 'Going to transmit {snap.volume_size} GiB {snap} {snap.description}'
if src_snap.tags.get('Name'):
info += ' of {name}'
info += ' from {snap.region} to {dst}'
logger.info(info.format(snap=src_snap, dst=dst_conn.region,
name=src_snap.tags.get('Name')))
src_vol = get_snap_vol(src_snap)
dst_snaps = get_relevant_snapshots(dst_conn, native_only=False)
vol_snaps = [snp for snp in dst_snaps if get_snap_vol(snp) == src_vol]
def sync_mountpoints(src_snap, src_vol, src_mnt, dst_vol, dst_mnt):
# Marking temporary volume with snapshot's description.
dst_vol.add_tag(DESCRIPTION_TAG, src_snap.description)
snaps, vols = get_replicas(src_snap.description, dst_vol.connection)
if not force and snaps:
raise ReplicationCollisionError(
'Stepping over {snap} - it\'s already replicated as {snaps} '
'in {snaps[0].region}'.format(snap=src_snap, snaps=snaps))
if not force and len(vols) > 1:
timeout = src_snap.volume_size / REPLICATION_SPEED
get_vol_time = lambda vol: parse(vol.create_time)
def not_outdated(vol, now):
age = now - get_vol_time(vol)
return age.days * 24 * 60 * 60 + age.seconds < timeout
now = datetime.utcnow().replace(tzinfo=tzutc())
actual_vols = [vol for vol in vols if not_outdated(vol, now)]
hunged_vols = set(vols) - set(actual_vols)
if len(actual_vols) > 1:
oldest = sorted(actual_vols, key=get_vol_time)[0]
if dst_vol.id != oldest.id:
raise ReplicationCollisionError(
'Stepping over {snap} - it\'s already replicating to '
'{vol} in {vol.region}'.format(snap=src_snap,
vol=oldest))
if len(hunged_vols) > 1:
logger.warn(
'Replication to temporary {vols} created during '
'transmitting {snap} to {reg} qualified as hunged up. '
'Starting new replication process.'.format(
snap=src_snap, vols=hunged_vols, reg=dst_vol.region))
update_snap(src_vol, src_mnt, dst_vol, dst_mnt, encr)
if vol_snaps:
dst_snap = sorted(vol_snaps, key=get_snap_time)[-1]
with nested(
attach_snapshot(src_snap, inst=src_inst, encr=encr),
attach_snapshot(dst_snap, inst=dst_inst, encr=encr)) as (
(src_vol, src_mnt), (dst_vol, dst_mnt)):
sync_mountpoints(src_snap, src_vol, src_mnt, dst_vol, dst_mnt)
else:
with nested(
attach_snapshot(src_snap, inst=src_inst, encr=encr),
create_tmp_volume(dst_conn.region, src_snap.volume_size)) as (
(src_vol, src_mnt), (dst_vol, dst_mnt)):
sync_mountpoints(src_snap, src_vol, src_mnt, dst_vol, dst_mnt)
def create_ami(region, snap_id, force=None, root_dev='/dev/sda1', zone_name=None,
default_arch=None, default_type='t1.micro', security_groups=''):
"""
Creates AMI image from given snapshot.
Force option removes prompt request and creates new instance from
created ami image.
region, snap_id
specify snapshot to be processed. Snapshot description in json
format will be used to restore instance with same parameters.
Will automaticaly process snapshots for same instance with near
time (10 minutes or shorter), but for other devices (/dev/sdb,
/dev/sdc, etc);
force
Run instance from ami after creation without confirmation. To
enable set value to "RUN";
default_arch
architecture to use if not mentioned in snapshot description;
default_type
instance type to use if not mentioned in snapshot description.
Used only if ``force`` is "RUN";
security_groups
list of AWS Security Groups names formatted as string separated
with semicolon ';'. Used only if ``force`` is "RUN".
"""
conn = get_region_conn(region)
snap = conn.get_all_snapshots(snapshot_ids=[snap_id, ])[0]
instance_id = get_snap_instance(snap)
_device = get_snap_device(snap)
snaps = conn.get_all_snapshots(owner='self')
snapshots = [snp for snp in snaps if
get_snap_instance(snp) == instance_id and
get_snap_device(snp) != _device and
abs(get_snap_time(snap) - get_snap_time(snp)) <= timedelta(minutes=10)]
snapshot = sorted(snapshots, key=get_snap_time,
reverse=True) if snapshots else None
# setup for building an EBS boot snapshot
default_arch = default_arch or config.get('DEFAULT', 'ARCHITECTURE')
arch = get_descr_attr(snap, 'Arch') or default_arch
kernel = config.get(conn.region.name, 'KERNEL' + arch.upper())
dev = re.match(r'^/dev/sda$', _device) # if our instance encrypted
if dev:
kernel = config.get(conn.region.name, 'KERNEL_ENCR_' + arch.upper())
ebs = EBSBlockDeviceType()
ebs.snapshot_id = snap_id
ebs.delete_on_termination = True
block_map = BlockDeviceMapping()
block_map[_device] = ebs
sdb = BlockDeviceType()
sdb.ephemeral_name = 'ephemeral0'
block_map['/dev/sdb'] = sdb
if snapshot:
for s in snapshot:
s_dev = get_snap_device(s)
s_ebs = EBSBlockDeviceType()
s_ebs.delete_on_termination = True
s_ebs.snapshot_id = s.id
block_map[s_dev] = s_ebs
name = 'Created {0} using access key {1}'.format(timestamp(),
conn.access_key)
name = name.replace(":", ".").replace(" ", "_")
# create the new AMI all options from snap JSON description:
wait_for(snap, '100%', limit=SNAP_TIME)
result = conn.register_image(
name=name,
description=snap.description,
architecture=get_descr_attr(snap, 'Arch') or default_arch,
root_device_name=get_descr_attr(snap, 'Root_dev_name') or root_dev,
block_device_map=block_map, kernel_id=kernel)
sleep(2)
image = conn.get_all_images(image_ids=[result, ])[0]
wait_for(image, 'available', limit=10 * 60)
add_tags(image, snap.tags)
logger.info('The new AMI ID = {0}'.format(result))
new_instance = None
if force == 'RUN':
instance_type = get_descr_attr(snap, 'Type') or default_type
new_instance = launch_instance_from_ami(
region, image.id, inst_type=instance_type,
security_groups=security_groups, zone_name=zone_name)
return image, new_instance
def rsync_snapshot(src_region_name, snapshot_id, dst_region_name,
src_inst=None, dst_inst=None, force=False):
"""Duplicate the snapshot into dst_region.
src_region_name, dst_region_name
Amazon region names. Allowed to be contracted, e.g.
`ap-southeast-1` will be recognized in `ap-south` or even
`ap-s`;
snapshot_id
snapshot to duplicate;
src_inst, dst_inst
will be used instead of creating new for temporary;
force
rsync snapshot even if newer version exist.
You'll need to open port 60000 for encrypted instances replication."""
src_conn = get_region_conn(src_region_name)
src_snap = src_conn.get_all_snapshots([snapshot_id])[0]
dst_conn = get_region_conn(dst_region_name)
_src_device = get_snap_device(src_snap)
_src_dev = re.match(r'^/dev/sda$', _src_device) # check for encryption
if _src_dev:
encr = True
logger.info('Found traces of encryption')
else:
encr = None
info = 'Going to transmit {snap.volume_size} GiB {snap} {snap.description}'
if src_snap.tags.get('Name'):
info += ' of {name}'
info += ' from {snap.region} to {dst}'
logger.info(info.format(snap=src_snap, dst=dst_conn.region,
name=src_snap.tags.get('Name')))
src_vol = get_snap_vol(src_snap)
dst_snaps = get_relevant_snapshots(dst_conn, native_only=False)
vol_snaps = [snp for snp in dst_snaps if get_snap_vol(snp) == src_vol]
def sync_mountpoints(src_snap, src_vol, src_mnt, dst_vol, dst_mnt):
# Marking temporary volume with snapshot's description.
dst_vol.add_tag(DESCRIPTION_TAG, src_snap.description)
snaps, vols = get_replicas(src_snap.description, dst_vol.connection)
if not force and snaps:
raise ReplicationCollisionError(
'Stepping over {snap} - it\'s already replicated as {snaps} '
'in {snaps[0].region}'.format(snap=src_snap, snaps=snaps))
if not force and len(vols) > 1:
timeout = src_snap.volume_size / REPLICATION_SPEED
get_vol_time = lambda vol: parse(vol.create_time)
def not_outdated(vol, now):
age = now - get_vol_time(vol)
return age.days * 24 * 60 * 60 + age.seconds < timeout
now = datetime.utcnow().replace(tzinfo=tzutc())
actual_vols = [vol for vol in vols if not_outdated(vol, now)]
hunged_vols = set(vols) - set(actual_vols)
if len(actual_vols) > 1:
oldest = sorted(actual_vols, key=get_vol_time)[0]
if dst_vol.id != oldest.id:
raise ReplicationCollisionError(
'Stepping over {snap} - it\'s already replicating to '
'{vol} in {vol.region}'.format(snap=src_snap,
vol=oldest))
if len(hunged_vols) > 1:
logger.warn(
'Replication to temporary {vols} created during '
'transmitting {snap} to {reg} qualified as hunged up. '
'Starting new replication process.'.format(
snap=src_snap, vols=hunged_vols, reg=dst_vol.region))
update_snap(src_vol, src_mnt, dst_vol, dst_mnt, encr)
if vol_snaps:
dst_snap = sorted(vol_snaps, key=get_snap_time)[-1]
with nested(
attach_snapshot(src_snap, inst=src_inst, encr=encr),
attach_snapshot(dst_snap, inst=dst_inst, encr=encr)) as (
(src_vol, src_mnt), (dst_vol, dst_mnt)):
sync_mountpoints(src_snap, src_vol, src_mnt, dst_vol, dst_mnt)
else:
with nested(
attach_snapshot(src_snap, inst=src_inst, encr=encr),
create_tmp_volume(dst_conn.region, src_snap.volume_size)) as (
(src_vol, src_mnt), (dst_vol, dst_mnt)):
sync_mountpoints(src_snap, src_vol, src_mnt, dst_vol, dst_mnt)
def create_ami(region,
snap_id,
force=None,
root_dev='/dev/sda1',
zone_name=None,
default_arch=None,
default_type='t1.micro',
security_groups=''):
"""
Creates AMI image from given snapshot.
Force option removes prompt request and creates new instance from
created ami image.
region, snap_id
specify snapshot to be processed. Snapshot description in json
format will be used to restore instance with same parameters.
Will automaticaly process snapshots for same instance with near
time (10 minutes or shorter), but for other devices (/dev/sdb,
/dev/sdc, etc);
force
Run instance from ami after creation without confirmation. To
enable set value to "RUN";
default_arch
architecture to use if not mentioned in snapshot description;
default_type
instance type to use if not mentioned in snapshot description.
Used only if ``force`` is "RUN";
security_groups
list of AWS Security Groups names formatted as string separated
with semicolon ';'. Used only if ``force`` is "RUN".
"""
conn = get_region_conn(region)
snap = conn.get_all_snapshots(snapshot_ids=[
snap_id,
])[0]
instance_id = get_snap_instance(snap)
_device = get_snap_device(snap)
snaps = conn.get_all_snapshots(owner='self')
snapshots = [
snp for snp in snaps if get_snap_instance(snp) == instance_id
and get_snap_device(snp) != _device and
abs(get_snap_time(snap) - get_snap_time(snp)) <= timedelta(minutes=10)
]
snapshot = sorted(snapshots, key=get_snap_time,
reverse=True) if snapshots else None
# setup for building an EBS boot snapshot
default_arch = default_arch or config.get('DEFAULT', 'ARCHITECTURE')
arch = get_descr_attr(snap, 'Arch') or default_arch
kernel = config.get(conn.region.name, 'KERNEL' + arch.upper())
dev = re.match(r'^/dev/sda$', _device) # if our instance encrypted
if dev:
kernel = config.get(conn.region.name, 'KERNEL_ENCR_' + arch.upper())
ebs = EBSBlockDeviceType()
ebs.snapshot_id = snap_id
ebs.delete_on_termination = True
block_map = BlockDeviceMapping()
block_map[_device] = ebs
sdb = BlockDeviceType()
sdb.ephemeral_name = 'ephemeral0'
block_map['/dev/sdb'] = sdb
if snapshot:
for s in snapshot:
s_dev = get_snap_device(s)
s_ebs = EBSBlockDeviceType()
s_ebs.delete_on_termination = True
s_ebs.snapshot_id = s.id
block_map[s_dev] = s_ebs
name = 'Created {0} using access key {1}'.format(timestamp(),
conn.access_key)
name = name.replace(":", ".").replace(" ", "_")
# create the new AMI all options from snap JSON description:
wait_for(snap, '100%', limit=SNAP_TIME)
result = conn.register_image(
name=name,
description=snap.description,
architecture=get_descr_attr(snap, 'Arch') or default_arch,
root_device_name=get_descr_attr(snap, 'Root_dev_name') or root_dev,
block_device_map=block_map,
kernel_id=kernel)
sleep(2)
image = conn.get_all_images(image_ids=[
result,
])[0]
wait_for(image, 'available', limit=10 * 60)
add_tags(image, snap.tags)
logger.info('The new AMI ID = {0}'.format(result))
info = ('\nEnter RUN if you want to launch instance using '
'just created {0}: '.format(image))
new_instance = None
if force == 'RUN' or raw_input(info).strip() == 'RUN':
instance_type = get_descr_attr(snap, 'Type') or default_type
new_instance = launch_instance_from_ami(
region,
image.id,
inst_type=instance_type,
security_groups=security_groups,
zone_name=zone_name)
return image, new_instance