def rsync_snapshot(src_region_name, snapshot_id, dst_region_name, src_inst=None, dst_inst=None): """Duplicate the snapshot into dst_region. src_region_name, dst_region_name Amazon region names. Allowed to be contracted, e.g. `ap-southeast-1` will be recognized in `ap-south` or even `ap-s`; snapshot_id snapshot to duplicate; src_inst, dst_inst will be used instead of creating new for temporary. You'll need to open port 60000 for encrypted instances replication.""" src_conn = get_region_conn(src_region_name) src_snap = src_conn.get_all_snapshots([snapshot_id])[0] dst_conn = get_region_conn(dst_region_name) _src_device = get_snap_device(src_snap) _src_dev = re.match(r'^/dev/sda$', _src_device) # check for encryption if _src_dev: encr = True logger.info('Found traces of encryption') else: encr = None info = 'Going to transmit {snap.volume_size} GiB {snap} {snap.description}' if src_snap.tags.get('Name'): info += ' of {name}' info += ' from {snap.region} to {dst}' logger.info(info.format(snap=src_snap, dst=dst_conn.region, name=src_snap.tags.get('Name'))) dst_snaps = dst_conn.get_all_snapshots(owner='self') dst_snaps = [snp for snp in dst_snaps if not snp.status == 'error'] src_vol = get_snap_vol(src_snap) vol_snaps = [snp for snp in dst_snaps if get_snap_vol(snp) == src_vol] if vol_snaps: dst_snap = sorted(vol_snaps, key=get_snap_time)[-1] if get_snap_time(dst_snap) >= get_snap_time(src_snap): kwargs = dict(src=src_snap, dst=dst_snap, dst_reg=dst_conn.region) logger.info('Stepping over {src} - it\'s not newer than {dst} ' '{dst.description} in {dst_reg}'.format(**kwargs)) return else: dst_snap = create_empty_snapshot(dst_conn.region, src_snap.volume_size) with nested(attach_snapshot(src_snap, inst=src_inst, encr=encr), attach_snapshot(dst_snap, inst=dst_inst, encr=encr)) as ( (src_vol, src_mnt), (dst_vol, dst_mnt)): update_snap(src_vol, src_mnt, dst_vol, dst_mnt, encr, delete_old=not vol_snaps) # Delete only empty snapshots.
def rsync_snapshot(src_region_name, snapshot_id, dst_region_name, src_inst=None, dst_inst=None, force=False): """Duplicate the snapshot into dst_region. src_region_name, dst_region_name Amazon region names. Allowed to be contracted, e.g. `ap-southeast-1` will be recognized in `ap-south` or even `ap-s`; snapshot_id snapshot to duplicate; src_inst, dst_inst will be used instead of creating new for temporary; force rsync snapshot even if newer version exist. You'll need to open port 60000 for encrypted instances replication.""" src_conn = get_region_conn(src_region_name) src_snap = src_conn.get_all_snapshots([snapshot_id])[0] dst_conn = get_region_conn(dst_region_name) _src_device = get_snap_device(src_snap) _src_dev = re.match(r'^/dev/sda$', _src_device) # check for encryption if _src_dev: encr = True logger.info('Found traces of encryption') else: encr = None info = 'Going to transmit {snap.volume_size} GiB {snap} {snap.description}' if src_snap.tags.get('Name'): info += ' of {name}' info += ' from {snap.region} to {dst}' logger.info(info.format(snap=src_snap, dst=dst_conn.region, name=src_snap.tags.get('Name'))) src_vol = get_snap_vol(src_snap) dst_snaps = get_relevant_snapshots(dst_conn, native_only=False) vol_snaps = [snp for snp in dst_snaps if get_snap_vol(snp) == src_vol] def sync_mountpoints(src_snap, src_vol, src_mnt, dst_vol, dst_mnt): # Marking temporary volume with snapshot's description. dst_vol.add_tag(DESCRIPTION_TAG, src_snap.description) snaps, vols = get_replicas(src_snap.description, dst_vol.connection) if not force and snaps: raise ReplicationCollisionError( 'Stepping over {snap} - it\'s already replicated as {snaps} ' 'in {snaps[0].region}'.format(snap=src_snap, snaps=snaps)) if not force and len(vols) > 1: timeout = src_snap.volume_size / REPLICATION_SPEED get_vol_time = lambda vol: parse(vol.create_time) def not_outdated(vol, now): age = now - get_vol_time(vol) return age.days * 24 * 60 * 60 + age.seconds < timeout now = datetime.utcnow().replace(tzinfo=tzutc()) actual_vols = [vol for vol in vols if not_outdated(vol, now)] hunged_vols = set(vols) - set(actual_vols) if len(actual_vols) > 1: oldest = sorted(actual_vols, key=get_vol_time)[0] if dst_vol.id != oldest.id: raise ReplicationCollisionError( 'Stepping over {snap} - it\'s already replicating to ' '{vol} in {vol.region}'.format(snap=src_snap, vol=oldest)) if len(hunged_vols) > 1: logger.warn( 'Replication to temporary {vols} created during ' 'transmitting {snap} to {reg} qualified as hunged up. ' 'Starting new replication process.'.format( snap=src_snap, vols=hunged_vols, reg=dst_vol.region)) update_snap(src_vol, src_mnt, dst_vol, dst_mnt, encr) if vol_snaps: dst_snap = sorted(vol_snaps, key=get_snap_time)[-1] with nested( attach_snapshot(src_snap, inst=src_inst, encr=encr), attach_snapshot(dst_snap, inst=dst_inst, encr=encr)) as ( (src_vol, src_mnt), (dst_vol, dst_mnt)): sync_mountpoints(src_snap, src_vol, src_mnt, dst_vol, dst_mnt) else: with nested( attach_snapshot(src_snap, inst=src_inst, encr=encr), create_tmp_volume(dst_conn.region, src_snap.volume_size)) as ( (src_vol, src_mnt), (dst_vol, dst_mnt)): sync_mountpoints(src_snap, src_vol, src_mnt, dst_vol, dst_mnt)
def create_ami(region, snap_id, force=None, root_dev='/dev/sda1', zone_name=None, default_arch=None, default_type='t1.micro', security_groups=''): """ Creates AMI image from given snapshot. Force option removes prompt request and creates new instance from created ami image. region, snap_id specify snapshot to be processed. Snapshot description in json format will be used to restore instance with same parameters. Will automaticaly process snapshots for same instance with near time (10 minutes or shorter), but for other devices (/dev/sdb, /dev/sdc, etc); force Run instance from ami after creation without confirmation. To enable set value to "RUN"; default_arch architecture to use if not mentioned in snapshot description; default_type instance type to use if not mentioned in snapshot description. Used only if ``force`` is "RUN"; security_groups list of AWS Security Groups names formatted as string separated with semicolon ';'. Used only if ``force`` is "RUN". """ conn = get_region_conn(region) snap = conn.get_all_snapshots(snapshot_ids=[snap_id, ])[0] instance_id = get_snap_instance(snap) _device = get_snap_device(snap) snaps = conn.get_all_snapshots(owner='self') snapshots = [snp for snp in snaps if get_snap_instance(snp) == instance_id and get_snap_device(snp) != _device and abs(get_snap_time(snap) - get_snap_time(snp)) <= timedelta(minutes=10)] snapshot = sorted(snapshots, key=get_snap_time, reverse=True) if snapshots else None # setup for building an EBS boot snapshot default_arch = default_arch or config.get('DEFAULT', 'ARCHITECTURE') arch = get_descr_attr(snap, 'Arch') or default_arch kernel = config.get(conn.region.name, 'KERNEL' + arch.upper()) dev = re.match(r'^/dev/sda$', _device) # if our instance encrypted if dev: kernel = config.get(conn.region.name, 'KERNEL_ENCR_' + arch.upper()) ebs = EBSBlockDeviceType() ebs.snapshot_id = snap_id ebs.delete_on_termination = True block_map = BlockDeviceMapping() block_map[_device] = ebs sdb = BlockDeviceType() sdb.ephemeral_name = 'ephemeral0' block_map['/dev/sdb'] = sdb if snapshot: for s in snapshot: s_dev = get_snap_device(s) s_ebs = EBSBlockDeviceType() s_ebs.delete_on_termination = True s_ebs.snapshot_id = s.id block_map[s_dev] = s_ebs name = 'Created {0} using access key {1}'.format(timestamp(), conn.access_key) name = name.replace(":", ".").replace(" ", "_") # create the new AMI all options from snap JSON description: wait_for(snap, '100%', limit=SNAP_TIME) result = conn.register_image( name=name, description=snap.description, architecture=get_descr_attr(snap, 'Arch') or default_arch, root_device_name=get_descr_attr(snap, 'Root_dev_name') or root_dev, block_device_map=block_map, kernel_id=kernel) sleep(2) image = conn.get_all_images(image_ids=[result, ])[0] wait_for(image, 'available', limit=10 * 60) add_tags(image, snap.tags) logger.info('The new AMI ID = {0}'.format(result)) new_instance = None if force == 'RUN': instance_type = get_descr_attr(snap, 'Type') or default_type new_instance = launch_instance_from_ami( region, image.id, inst_type=instance_type, security_groups=security_groups, zone_name=zone_name) return image, new_instance
def create_ami(region, snap_id, force=None, root_dev='/dev/sda1', zone_name=None, default_arch=None, default_type='t1.micro', security_groups=''): """ Creates AMI image from given snapshot. Force option removes prompt request and creates new instance from created ami image. region, snap_id specify snapshot to be processed. Snapshot description in json format will be used to restore instance with same parameters. Will automaticaly process snapshots for same instance with near time (10 minutes or shorter), but for other devices (/dev/sdb, /dev/sdc, etc); force Run instance from ami after creation without confirmation. To enable set value to "RUN"; default_arch architecture to use if not mentioned in snapshot description; default_type instance type to use if not mentioned in snapshot description. Used only if ``force`` is "RUN"; security_groups list of AWS Security Groups names formatted as string separated with semicolon ';'. Used only if ``force`` is "RUN". """ conn = get_region_conn(region) snap = conn.get_all_snapshots(snapshot_ids=[ snap_id, ])[0] instance_id = get_snap_instance(snap) _device = get_snap_device(snap) snaps = conn.get_all_snapshots(owner='self') snapshots = [ snp for snp in snaps if get_snap_instance(snp) == instance_id and get_snap_device(snp) != _device and abs(get_snap_time(snap) - get_snap_time(snp)) <= timedelta(minutes=10) ] snapshot = sorted(snapshots, key=get_snap_time, reverse=True) if snapshots else None # setup for building an EBS boot snapshot default_arch = default_arch or config.get('DEFAULT', 'ARCHITECTURE') arch = get_descr_attr(snap, 'Arch') or default_arch kernel = config.get(conn.region.name, 'KERNEL' + arch.upper()) dev = re.match(r'^/dev/sda$', _device) # if our instance encrypted if dev: kernel = config.get(conn.region.name, 'KERNEL_ENCR_' + arch.upper()) ebs = EBSBlockDeviceType() ebs.snapshot_id = snap_id ebs.delete_on_termination = True block_map = BlockDeviceMapping() block_map[_device] = ebs sdb = BlockDeviceType() sdb.ephemeral_name = 'ephemeral0' block_map['/dev/sdb'] = sdb if snapshot: for s in snapshot: s_dev = get_snap_device(s) s_ebs = EBSBlockDeviceType() s_ebs.delete_on_termination = True s_ebs.snapshot_id = s.id block_map[s_dev] = s_ebs name = 'Created {0} using access key {1}'.format(timestamp(), conn.access_key) name = name.replace(":", ".").replace(" ", "_") # create the new AMI all options from snap JSON description: wait_for(snap, '100%', limit=SNAP_TIME) result = conn.register_image( name=name, description=snap.description, architecture=get_descr_attr(snap, 'Arch') or default_arch, root_device_name=get_descr_attr(snap, 'Root_dev_name') or root_dev, block_device_map=block_map, kernel_id=kernel) sleep(2) image = conn.get_all_images(image_ids=[ result, ])[0] wait_for(image, 'available', limit=10 * 60) add_tags(image, snap.tags) logger.info('The new AMI ID = {0}'.format(result)) info = ('\nEnter RUN if you want to launch instance using ' 'just created {0}: '.format(image)) new_instance = None if force == 'RUN' or raw_input(info).strip() == 'RUN': instance_type = get_descr_attr(snap, 'Type') or default_type new_instance = launch_instance_from_ami( region, image.id, inst_type=instance_type, security_groups=security_groups, zone_name=zone_name) return image, new_instance