def rsync_snapshot(src_region_name, snapshot_id, dst_region_name,
src_inst=None, dst_inst=None):
"""Duplicate the snapshot into dst_region.
src_region_name, dst_region_name
Amazon region names. Allowed to be contracted, e.g.
`ap-southeast-1` will be recognized in `ap-south` or even
`ap-s`;
snapshot_id
snapshot to duplicate;
src_inst, dst_inst
will be used instead of creating new for temporary.
You'll need to open port 60000 for encrypted instances replication."""
src_conn = get_region_conn(src_region_name)
src_snap = src_conn.get_all_snapshots([snapshot_id])[0]
dst_conn = get_region_conn(dst_region_name)
_src_device = get_snap_device(src_snap)
_src_dev = re.match(r'^/dev/sda$', _src_device) # check for encryption
if _src_dev:
encr = True
logger.info('Found traces of encryption')
else:
encr = None
info = 'Going to transmit {snap.volume_size} GiB {snap} {snap.description}'
if src_snap.tags.get('Name'):
info += ' of {name}'
info += ' from {snap.region} to {dst}'
logger.info(info.format(snap=src_snap, dst=dst_conn.region,
name=src_snap.tags.get('Name')))
dst_snaps = dst_conn.get_all_snapshots(owner='self')
dst_snaps = [snp for snp in dst_snaps if not snp.status == 'error']
src_vol = get_snap_vol(src_snap)
vol_snaps = [snp for snp in dst_snaps if get_snap_vol(snp) == src_vol]
if vol_snaps:
dst_snap = sorted(vol_snaps, key=get_snap_time)[-1]
if get_snap_time(dst_snap) >= get_snap_time(src_snap):
kwargs = dict(src=src_snap, dst=dst_snap, dst_reg=dst_conn.region)
logger.info('Stepping over {src} - it\'s not newer than {dst} '
'{dst.description} in {dst_reg}'.format(**kwargs))
return
else:
dst_snap = create_empty_snapshot(dst_conn.region, src_snap.volume_size)
with nested(attach_snapshot(src_snap, inst=src_inst, encr=encr),
attach_snapshot(dst_snap, inst=dst_inst, encr=encr)) as (
(src_vol, src_mnt), (dst_vol, dst_mnt)):
update_snap(src_vol, src_mnt, dst_vol, dst_mnt, encr,
delete_old=not vol_snaps) # Delete only empty snapshots.
def rsync_snapshot(src_region_name, snapshot_id, dst_region_name,
src_inst=None, dst_inst=None, force=False):
"""Duplicate the snapshot into dst_region.
src_region_name, dst_region_name
Amazon region names. Allowed to be contracted, e.g.
`ap-southeast-1` will be recognized in `ap-south` or even
`ap-s`;
snapshot_id
snapshot to duplicate;
src_inst, dst_inst
will be used instead of creating new for temporary;
force
rsync snapshot even if newer version exist.
You'll need to open port 60000 for encrypted instances replication."""
src_conn = get_region_conn(src_region_name)
src_snap = src_conn.get_all_snapshots([snapshot_id])[0]
dst_conn = get_region_conn(dst_region_name)
_src_device = get_snap_device(src_snap)
_src_dev = re.match(r'^/dev/sda$', _src_device) # check for encryption
if _src_dev:
encr = True
logger.info('Found traces of encryption')
else:
encr = None
info = 'Going to transmit {snap.volume_size} GiB {snap} {snap.description}'
if src_snap.tags.get('Name'):
info += ' of {name}'
info += ' from {snap.region} to {dst}'
logger.info(info.format(snap=src_snap, dst=dst_conn.region,
name=src_snap.tags.get('Name')))
src_vol = get_snap_vol(src_snap)
dst_snaps = get_relevant_snapshots(dst_conn, native_only=False)
vol_snaps = [snp for snp in dst_snaps if get_snap_vol(snp) == src_vol]
def sync_mountpoints(src_snap, src_vol, src_mnt, dst_vol, dst_mnt):
# Marking temporary volume with snapshot's description.
dst_vol.add_tag(DESCRIPTION_TAG, src_snap.description)
snaps, vols = get_replicas(src_snap.description, dst_vol.connection)
if not force and snaps:
raise ReplicationCollisionError(
'Stepping over {snap} - it\'s already replicated as {snaps} '
'in {snaps[0].region}'.format(snap=src_snap, snaps=snaps))
if not force and len(vols) > 1:
timeout = src_snap.volume_size / REPLICATION_SPEED
get_vol_time = lambda vol: parse(vol.create_time)
def not_outdated(vol, now):
age = now - get_vol_time(vol)
return age.days * 24 * 60 * 60 + age.seconds < timeout
now = datetime.utcnow().replace(tzinfo=tzutc())
actual_vols = [vol for vol in vols if not_outdated(vol, now)]
hunged_vols = set(vols) - set(actual_vols)
if len(actual_vols) > 1:
oldest = sorted(actual_vols, key=get_vol_time)[0]
if dst_vol.id != oldest.id:
raise ReplicationCollisionError(
'Stepping over {snap} - it\'s already replicating to '
'{vol} in {vol.region}'.format(snap=src_snap,
vol=oldest))
if len(hunged_vols) > 1:
logger.warn(
'Replication to temporary {vols} created during '
'transmitting {snap} to {reg} qualified as hunged up. '
'Starting new replication process.'.format(
snap=src_snap, vols=hunged_vols, reg=dst_vol.region))
update_snap(src_vol, src_mnt, dst_vol, dst_mnt, encr)
if vol_snaps:
dst_snap = sorted(vol_snaps, key=get_snap_time)[-1]
with nested(
attach_snapshot(src_snap, inst=src_inst, encr=encr),
attach_snapshot(dst_snap, inst=dst_inst, encr=encr)) as (
(src_vol, src_mnt), (dst_vol, dst_mnt)):
sync_mountpoints(src_snap, src_vol, src_mnt, dst_vol, dst_mnt)
else:
with nested(
attach_snapshot(src_snap, inst=src_inst, encr=encr),
create_tmp_volume(dst_conn.region, src_snap.volume_size)) as (
(src_vol, src_mnt), (dst_vol, dst_mnt)):
sync_mountpoints(src_snap, src_vol, src_mnt, dst_vol, dst_mnt)
def rsync_snapshot(src_region_name, snapshot_id, dst_region_name,
src_inst=None, dst_inst=None, force=False):
"""Duplicate the snapshot into dst_region.
src_region_name, dst_region_name
Amazon region names. Allowed to be contracted, e.g.
`ap-southeast-1` will be recognized in `ap-south` or even
`ap-s`;
snapshot_id
snapshot to duplicate;
src_inst, dst_inst
will be used instead of creating new for temporary;
force
rsync snapshot even if newer version exist.
You'll need to open port 60000 for encrypted instances replication."""
src_conn = get_region_conn(src_region_name)
src_snap = src_conn.get_all_snapshots([snapshot_id])[0]
dst_conn = get_region_conn(dst_region_name)
_src_device = get_snap_device(src_snap)
_src_dev = re.match(r'^/dev/sda$', _src_device) # check for encryption
if _src_dev:
encr = True
logger.info('Found traces of encryption')
else:
encr = None
info = 'Going to transmit {snap.volume_size} GiB {snap} {snap.description}'
if src_snap.tags.get('Name'):
info += ' of {name}'
info += ' from {snap.region} to {dst}'
logger.info(info.format(snap=src_snap, dst=dst_conn.region,
name=src_snap.tags.get('Name')))
src_vol = get_snap_vol(src_snap)
dst_snaps = get_relevant_snapshots(dst_conn, native_only=False)
vol_snaps = [snp for snp in dst_snaps if get_snap_vol(snp) == src_vol]
def sync_mountpoints(src_snap, src_vol, src_mnt, dst_vol, dst_mnt):
# Marking temporary volume with snapshot's description.
dst_vol.add_tag(DESCRIPTION_TAG, src_snap.description)
snaps, vols = get_replicas(src_snap.description, dst_vol.connection)
if not force and snaps:
raise ReplicationCollisionError(
'Stepping over {snap} - it\'s already replicated as {snaps} '
'in {snaps[0].region}'.format(snap=src_snap, snaps=snaps))
if not force and len(vols) > 1:
timeout = src_snap.volume_size / REPLICATION_SPEED
get_vol_time = lambda vol: parse(vol.create_time)
def not_outdated(vol, now):
age = now - get_vol_time(vol)
return age.days * 24 * 60 * 60 + age.seconds < timeout
now = datetime.utcnow().replace(tzinfo=tzutc())
actual_vols = [vol for vol in vols if not_outdated(vol, now)]
hunged_vols = set(vols) - set(actual_vols)
if len(actual_vols) > 1:
oldest = sorted(actual_vols, key=get_vol_time)[0]
if dst_vol.id != oldest.id:
raise ReplicationCollisionError(
'Stepping over {snap} - it\'s already replicating to '
'{vol} in {vol.region}'.format(snap=src_snap,
vol=oldest))
if len(hunged_vols) > 1:
logger.warn(
'Replication to temporary {vols} created during '
'transmitting {snap} to {reg} qualified as hunged up. '
'Starting new replication process.'.format(
snap=src_snap, vols=hunged_vols, reg=dst_vol.region))
update_snap(src_vol, src_mnt, dst_vol, dst_mnt, encr)
if vol_snaps:
dst_snap = sorted(vol_snaps, key=get_snap_time)[-1]
with nested(
attach_snapshot(src_snap, inst=src_inst, encr=encr),
attach_snapshot(dst_snap, inst=dst_inst, encr=encr)) as (
(src_vol, src_mnt), (dst_vol, dst_mnt)):
sync_mountpoints(src_snap, src_vol, src_mnt, dst_vol, dst_mnt)
else:
with nested(
attach_snapshot(src_snap, inst=src_inst, encr=encr),
create_tmp_volume(dst_conn.region, src_snap.volume_size)) as (
(src_vol, src_mnt), (dst_vol, dst_mnt)):
sync_mountpoints(src_snap, src_vol, src_mnt, dst_vol, dst_mnt)
