def enable_root(self, guestaddr, sshprivkey, user, prefix):
for cmd in (
"mkdir -p /root/.ssh",
"chmod 600 /root/.ssh",
"cp -f /home/%s/.ssh/authorized_keys /root/.ssh" % user,
"chmod 600 /root/.ssh/authorized_keys",
):
process_utils.ssh_execute_command(guestaddr, sshprivkey, cmd, user=user, prefix=prefix)
stdout, stderr, retcode = process_utils.ssh_execute_command(guestaddr, sshprivkey, "/bin/id")
if not re.search("uid=0", stdout):
raise Exception("Running /bin/id on %s as root: %s" % (guestaddr, stdout))
def wait_for_ec2_ssh_access(self, guestaddr, sshprivkey):
self.log.debug("Waiting for SSH access to EC2 instance (User: %s)" % self.user)
for i in range(300):
if i % 10 == 0:
self.log.debug("Waiting for EC2 ssh access: %d/300" % (i))
try:
process_utils.ssh_execute_command(guestaddr, sshprivkey, "/bin/true", user=self.user)
self.log.debug("reached the instance as %s using %s" % (self.user, sshprivkey))
sleep(5)
break
except:
pass
sleep(1)
if i == 299:
raise Exception("Unable to gain ssh access after 300 seconds - aborting")
def enable_root(self,guestaddr, sshprivkey, user, prefix):
for cmd in ('mkdir /root/.ssh',
'chmod 600 /root/.ssh',
'cp -f /home/%s/.ssh/authorized_keys /root/.ssh' % user,
'chmod 600 /root/.ssh/authorized_keys'):
try:
process_utils.ssh_execute_command(guestaddr, sshprivkey, cmd, user=user, prefix=prefix)
except Exception as e:
pass
try:
stdout, stderr, retcode = process_utils.ssh_execute_command(guestaddr, sshprivkey, '/bin/id')
if not re.search('uid=0', stdout):
raise Exception('Running /bin/id on %s as root: %s' % (guestaddr, stdout))
except Exception as e:
raise Exception('Transfer of authorized_keys to root from %s must have failed - Aborting - %s' % (user, e))
def file_to_snapshot(self, filename, compress=True):
if not self.instance:
raise Exception("You must start the utility instance first!")
if not os.path.isfile(filename):
raise Exception("Filename (%s) is not a file" % filename)
filesize = os.path.getsize(filename)
# Gigabytes, rounded up
volume_size = int((filesize / (1024 ** 3)) + 1)
self.log.debug("Creating %d GiB volume in (%s) to hold new image" % (volume_size, self.instance.placement))
volume = self.conn.create_volume(volume_size, self.instance.placement)
# Volumes can sometimes take a very long time to create
# Wait up to 10 minutes for now (plus the time taken for the upload
# above)
self.log.debug("Waiting up to 600 seconds for volume (%s) to become available" % volume.id)
for i in range(60):
safe_call(volume.update, (), self.log, die=True)
if volume.status == "available":
volume.add_tag("Name", resource_tag)
break
self.log.debug("Volume status (%s) - waiting for 'available': %d/600" % (volume.status, i * 10))
sleep(10)
# Volume is now available, attach it
safe_call(self.conn.attach_volume, (volume.id, self.instance.id, "/dev/sdh"), self.log, die=True)
self.log.debug("Waiting up to 120 seconds for volume (%s) to become in-use" % volume.id)
for i in range(12):
safe_call(volume.update, (), self.log, die=True)
vs = volume.attachment_state()
if vs == "attached":
break
self.log.debug("Volume status (%s) - waiting for 'attached': %d/120" % (vs, i * 10))
sleep(10)
# TODO: This may not be necessary but it helped with some funnies
# observed during testing. At some point run a bunch of builds without
# the delay to see if it breaks anything.
self.log.debug("Waiting 20 seconds for EBS attachment to stabilize")
sleep(20)
# Decompress image into new EBS volume
self.log.debug("Copying file into volume")
# This is big and hairy - it also works, and avoids temporary storage
# on the local and remote side of this activity
command = "gzip -c %s | " % filename
command += (
"ssh -i %s -F /dev/null -o ServerAliveInterval=30 -o StrictHostKeyChecking=no " % self.key_file_object.name
)
command += "-o ConnectTimeout=30 -o UserKnownHostsFile=/dev/null -o PasswordAuthentication=no "
command += 'root@%s "gzip -d -c | dd of=/dev/xvdh bs=4k"' % self.instance.public_dns_name
self.log.debug("Command will be:\n%s\n" % command)
self.log.debug("Running. This may take some time.")
process_utils.subprocess_check_output([command], shell=True)
# Sync before snapshot
process_utils.ssh_execute_command(self.instance.public_dns_name, self.key_file_object.name, "sync")
# Snapshot EBS volume
self.log.debug("Taking snapshot of volume (%s)" % volume.id)
snapshot = self.conn.create_snapshot(volume.id, 'EBSHelper snapshot of file "%s"' % filename)
# This can take a _long_ time - wait up to 20 minutes
self.log.debug("Waiting up to 1200 seconds for snapshot (%s) to become completed" % snapshot.id)
for i in range(120):
safe_call(snapshot.update, (), self.log, die=True)
if snapshot.status == "completed":
snapshot.add_tag("Name", resource_tag)
break
self.log.debug(
"Snapshot progress(%s) - status (%s) is not 'completed': %d/1200"
% (str(snapshot.progress), snapshot.status, i * 10)
)
sleep(10)
self.log.debug("Successful creation of snapshot (%s)" % snapshot.id)
self.log.debug("Detaching volume (%s)" % volume.id)
safe_call(volume.detach, (), self.log)
self.log.debug("Waiting up to 120 seconds for %s to become detached (available)" % volume.id)
for i in range(12):
safe_call(volume.update, (), self.log, die=True)
if volume.status == "available":
break
self.log.debug("Volume status (%s) - is not 'available': %d/120" % (volume.status, i * 10))
sleep(10)
self.log.debug("Deleting volume")
safe_call(volume.delete, (), self.log, die=True)
return snapshot.id
def file_to_snapshot(self, filename, compress=True):
# TODO: Add a conservative exception handler over the top of this to delete all remote artifacts on
# an exception
if not self.instance:
raise Exception("You must start the utility instance with start_ami() before uploading files to volumes")
if not os.path.isfile(filename):
raise Exception("Filename (%s) is not a file" % filename)
filesize = os.path.getsize(filename)
# Gigabytes, rounded up
volume_size = int( (filesize/(1024 ** 3)) + 1 )
self.log.debug("Creating %d GiB volume in (%s) to hold new image" % (volume_size, self.instance.placement))
volume = self.conn.create_volume(volume_size, self.instance.placement)
# Volumes can sometimes take a very long time to create
# Wait up to 10 minutes for now (plus the time taken for the upload above)
self.log.debug("Waiting up to 600 seconds for volume (%s) to become available" % (volume.id))
retcode = 1
for i in range(60):
volume.update()
if volume.status == "available":
retcode = 0
break
self.log.debug("Volume status (%s) - waiting for 'available': %d/600" % (volume.status, i*10))
sleep(10)
if retcode:
raise Exception("Unable to create target volume for EBS AMI - aborting")
# Volume is now available
# Attach it
self.conn.attach_volume(volume.id, self.instance.id, "/dev/sdh")
self.log.debug("Waiting up to 120 seconds for volume (%s) to become in-use" % (volume.id))
retcode = 1
for i in range(12):
volume.update()
vs = volume.attachment_state()
if vs == "attached":
retcode = 0
break
self.log.debug("Volume status (%s) - waiting for 'attached': %d/120" % (vs, i*10))
sleep(10)
if retcode:
raise Exception("Unable to attach volume (%s) to instance (%s) aborting" % (volume.id, self.instance.id))
# TODO: This may not be necessary but it helped with some funnies observed during testing
# At some point run a bunch of builds without the delay to see if it breaks anything
self.log.debug("Waiting 20 seconds for EBS attachment to stabilize")
sleep(20)
# Decompress image into new EBS volume
self.log.debug("Copying file into volume")
# This is big and hairy - it also works, and avoids temporary storage on the local and remote
# side of this activity
command = 'gzip -c %s | ' % filename
command += 'ssh -i %s -F /dev/null -o ServerAliveInterval=30 -o StrictHostKeyChecking=no ' % self.key_file_object.name
command += '-o ConnectTimeout=30 -o UserKnownHostsFile=/dev/null -o PasswordAuthentication=no '
command += 'root@%s "gzip -d -c | dd of=/dev/xvdh bs=4k"' % self.instance.public_dns_name
self.log.debug("Command will be:\n\n%s\n\n" % command)
self.log.debug("Running. This may take some time.")
process_utils.subprocess_check_output([ command ], shell=True)
# Sync before snapshot
process_utils.ssh_execute_command(self.instance.public_dns_name, self.key_file_object.name, "sync")
# Snapshot EBS volume
self.log.debug("Taking snapshot of volume (%s)" % (volume.id))
snapshot = self.conn.create_snapshot(volume.id, 'EBSHelper snapshot of file "%s"' % filename)
# This can take a _long_ time - wait up to 20 minutes
self.log.debug("Waiting up to 1200 seconds for snapshot (%s) to become completed" % (snapshot.id))
retcode = 1
for i in range(120):
snapshot.update()
if snapshot.status == "completed":
retcode = 0
break
self.log.debug("Snapshot progress(%s) - status (%s) - waiting for 'completed': %d/1200" % (str(snapshot.progress), snapshot.status, i*10))
sleep(10)
if retcode:
raise Exception("Unable to snapshot volume (%s) - aborting" % (volume.id))
self.log.debug("Successful creation of snapshot (%s)" % (snapshot.id))
self.log.debug("Detaching volume (%s)" % volume.id)
volume.detach()
self.log.debug("Waiting up to 120 seconds for volume (%s) to become detached (available)" % (volume.id))
retcode = 1
for i in range(12):
volume.update()
if volume.status == "available":
retcode = 0
break
self.log.debug("Volume status (%s) - waiting for 'available': %d/120" % (volume.status, i*10))
sleep(10)
if retcode:
raise Exception("Unable to detach volume - WARNING - volume may persist and cost money!")
self.log.debug("Deleting volume")
volume.delete()
# TODO: Verify delete
return snapshot.id
