def on_init(self, *args, **kwargs):
bus.on("before_hello", self.on_before_hello)
bus.on("before_host_init", self.on_before_host_init)
bus.on("before_restart", self.on_before_restart)
bus.on("before_reboot_finish", self.on_before_reboot_finish)
try:
system(('ntpdate', '-u', '0.amazon.pool.ntp.org'))
except:
pass
msg_service = bus.messaging_service
producer = msg_service.get_producer()
producer.on("before_send", self.on_before_message_send)
if not os_dist.windows_family and not __node__.get('hostname'):
# Set the hostname to this instance's public hostname
try:
hostname_as_pubdns = int(__ec2__['hostname_as_pubdns'])
except:
hostname_as_pubdns = True
if hostname_as_pubdns:
pub_hostname = self._platform.get_public_hostname()
self._logger.debug('Setting hostname to %s' % pub_hostname)
system2("hostname " + pub_hostname, shell=True)
if disttool.is_ubuntu():
# Ubuntu cloud-init scripts may disable root ssh login
for path in ('/etc/ec2-init/ec2-config.cfg', '/etc/cloud/cloud.cfg'):
if os.path.exists(path):
c = None
with open(path, 'r') as fp:
c = fp.read()
c = re.sub(re.compile(r'^disable_root[^:=]*([:=]).*', re.M), r'disable_root\1 0', c)
with open(path, 'w') as fp:
fp.write(c)
if not linux.os.windows_family:
# Add server ssh public key to authorized_keys
ssh_key = self._platform.get_ssh_pub_key()
if ssh_key:
add_authorized_key(ssh_key)
# Mount ephemeral devices
# Seen on eucalyptus:
# - fstab contains invalid fstype and `mount -a` fails
if self._platform.name == 'eucalyptus':
mtab = mount.mounts()
fstab = mount.fstab()
for device in self._platform.instance_store_devices:
if os.path.exists(device) and device in fstab and device not in mtab:
entry = fstab[device]
try:
mount.mount(device, entry.mpoint, '-o', entry.options)
except:
self._logger.warn(sys.exc_info()[1])
else:
if not os_dist.windows_family:
system2('mount -a', shell=True, raise_exc=False)
def mount(self):
self._check(mpoint=True)
mounted_to = self.mounted_to()
if mounted_to == self.mpoint:
return
elif mounted_to:
LOG.debug('Umounting %s from %s', self.id, mounted_to)
self.umount()
if not os.path.exists(self.mpoint):
os.makedirs(self.mpoint)
short_args = []
if self.mount_options:
short_args += ['-o', ','.join(self.mount_options)]
LOG.debug('Mounting %s to %s', self.id, self.mpoint)
try:
mod_mount.mount(self.device, self.mpoint, *short_args)
except linux.LinuxError as e:
# XXX: SCALARIZR-1974 is for complete solution
if linux.os.ubuntu and \
re.search(r'already mounted or /mnt/(dbstorage|pgstorage|redisstorage) busy', str(e)):
LOG.debug(
'upstart already mounted database storage volume {} to {}'.
format(self.device, self.mpoint))
else:
raise
bus.fire("block_device_mounted", volume=self)
def _create(self, volume, snapshot, snap_lv, tranzit_path, complete_cb):
try:
chunk_prefix = '%s.data' % snapshot.id
snapshot.path = None
snap_mpoint = mkdtemp()
try:
opts = []
if volume.fstype == 'xfs':
opts += ['-o', 'nouuid,ro']
mount.mount(snap_lv, snap_mpoint, *opts)
tar_cmd = ['tar', 'cp', '-C', snap_mpoint, '.']
pigz_bins = whereis('pigz')
compress_cmd = [pigz_bins[0] if pigz_bins else 'gzip', '-5']
self._logger.debug("Creating and compressing snapshot data.")
tar = subprocess.Popen(tar_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
compress = subprocess.Popen(compress_cmd, stdin=tar.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
tar.stdout.close() # Allow tar to receive a SIGPIPE if compress exits.
split = threading.Thread(target=self._split, name='split',
args=(compress.stdout, tranzit_path, chunk_prefix, snapshot))
split.start()
uploaders = []
for i in range(2):
uploader = threading.Thread(name="Uploader-%s" % i, target=self._uploader,
args=(volume.snap_backend['path'], snapshot))
self._logger.debug("Starting uploader '%s'", uploader.getName())
uploader.start()
uploaders.append(uploader)
self._logger.debug('uploaders started. waiting compress')
compress.wait()
self._logger.debug('compress completed (code: %s). waiting split', compress.returncode)
if compress.returncode:
raise StorageError('Compress process terminated with exit code %s. <err>: %s' % (compress.returncode, compress.stderr.read()))
split.join()
self._logger.debug('split completed. waiting uploaders')
for uploader in uploaders:
uploader.join()
self._logger.debug('uploaders completed')
if self._inner_exc_info:
t, e, s = self._inner_exc_info
raise t, e, s
finally:
self._return_ev.set()
mount.umount(snap_mpoint)
os.rmdir(snap_mpoint)
self._lvm.remove_lv(snap_lv)
self._inner_exc_info = None
self._state_map[snapshot.id] = Snapshot.COMPLETED
except (Exception, BaseException), e:
self._state_map[snapshot.id] = Snapshot.FAILED
self._logger.exception('Snapshot creation failed. %s' % e)
def _ensure(self):
expr = isinstance(self.size, int) and self.size
assert expr, self.error_messages["invalid_size"]
self._check_attr("mpoint")
if not self.device:
if not os.path.exists(self.mpoint):
os.makedirs(self.mpoint)
mount.mount("tmpfs", self.mpoint, "-t", "tmpfs", "-o", "size=%sM" % self.size)
self.device = self.mpoint
def mount(self):
self._check(mpoint=True)
mounted_to = self.mounted_to()
if mounted_to == self.mpoint:
return
elif mounted_to:
self.umount()
if not os.path.exists(self.mpoint):
os.makedirs(self.mpoint)
mod_mount.mount(self.device, self.mpoint)
def _ensure(self):
expr = isinstance(self.size, int) and self.size
assert expr, self.error_messages['invalid_size']
self._check_attr('mpoint')
if not self.device:
if not os.path.exists(self.mpoint):
os.makedirs(self.mpoint)
mount.mount('tmpfs', self.mpoint, '-t', 'tmpfs', '-o',
'size=%sM' % self.size)
self.device = self.mpoint
def mount(self): self._check(mpoint=True) mounted_to = self.mounted_to() if mounted_to == self.mpoint: return elif mounted_to: self.umount() if not os.path.exists(self.mpoint): os.makedirs(self.mpoint) mod_mount.mount(self.device, self.mpoint)
def mount(self):
self._check(mpoint=True)
mounted_to = self.mounted_to()
if mounted_to == self.mpoint:
return
elif mounted_to:
LOG.debug('Umounting %s from %s', self.id, mounted_to)
self.umount()
if not os.path.exists(self.mpoint):
os.makedirs(self.mpoint)
LOG.debug('Mounting %s to %s', self.id, self.mpoint)
mod_mount.mount(self.device, self.mpoint)
bus.fire("block_device_mounted", volume=self)
def mount(self):
self._check(mpoint=True)
mounted_to = self.mounted_to()
if mounted_to == self.mpoint:
return
elif mounted_to:
LOG.debug('Umounting %s from %s', self.id, mounted_to)
self.umount()
if not os.path.exists(self.mpoint):
os.makedirs(self.mpoint)
LOG.debug('Mounting %s to %s', self.id, self.mpoint)
mod_mount.mount(self.device, self.mpoint)
bus.fire("block_device_mounted", volume=self)
def upload_lvm_snapshot(self, lvm_snap, tags, path):
"""
Method which uploads data from lvm snapshot to cloud storage and
updates snapshot status.
EphVolume runs this method in separate thread
"""
try:
self._snap_status = self.QUEUED
mpoint = tempfile.mkdtemp()
opts = []
if coreutils.blkid(lvm_snap.device).get('type') == 'xfs':
opts += ['-o', 'nouuid,ro']
mount.mount(lvm_snap.device, mpoint, *opts)
self.data_size = coreutils.statvfs(mpoint)['used']
try:
cmd = ['/bin/tar', 'cp', mpoint]
stream = subprocess.Popen(cmd, stdout=subprocess.PIPE, close_fds=True).stdout
src = cloudfs.NamedStream(stream, 'lvm_snapshot', streamer='tar', extension='tar')
dst = path
transfer = largetransfer.Upload(src, dst, tags=tags, transfer_id=self.id)
self._snap_status = self.IN_PROGRESS
transfer.apply_async()
transfer.join()
manifesto = transfer.manifest
self.path = manifesto.cloudfs_path
self._snap_status = self.COMPLETED
finally:
mount.umount(mpoint)
os.rmdir(mpoint)
except:
self._snap_status = self.FAILED
LOG.exception('Caught error while uploading LVM snapshot')
finally:
lvm_snap.destroy()
def upload_lvm_snapshot(self, lvm_snap, tags, path):
"""
Method which uploads data from lvm snapshot to cloud storage and
updates snapshot status.
EphVolume runs this method in separate thread
"""
try:
self._snap_status = self.QUEUED
mpoint = tempfile.mkdtemp()
opts = []
if coreutils.blkid(lvm_snap.device).get('type') == 'xfs':
opts += ['-o', 'nouuid,ro']
mount.mount(lvm_snap.device, mpoint, *opts)
self.data_size = coreutils.statvfs(mpoint)['used']
try:
transfer = cloudfs.LargeTransfer(
src=mpoint + '/',
dst=path,
tar_it=True,
gzip_it=True,
tags=tags,
transfer_id=self.id)
self._snap_status = self.IN_PROGRESS
manifesto = transfer.run()
self.path = manifesto.cloudfs_path
self._snap_status = self.COMPLETED
finally:
mount.umount(mpoint)
os.rmdir(mpoint)
except:
self._snap_status = self.FAILED
LOG.exception('Caught error while uploading LVM snapshot')
finally:
lvm_snap.destroy()
def upload_lvm_snapshot(self, lvm_snap, tags, path):
"""
Method which uploads data from lvm snapshot to cloud storage and
updates snapshot status.
EphVolume runs this method in separate thread
"""
try:
self._snap_status = self.QUEUED
mpoint = tempfile.mkdtemp()
opts = []
if coreutils.blkid(lvm_snap.device).get('type') == 'xfs':
opts += ['-o', 'nouuid,ro']
mount.mount(lvm_snap.device, mpoint, *opts)
self.data_size = coreutils.statvfs(mpoint)['used']
try:
transfer = cloudfs.LargeTransfer(src=mpoint + '/',
dst=path,
tar_it=True,
gzip_it=True,
tags=tags,
transfer_id=self.id)
self._snap_status = self.IN_PROGRESS
manifesto = transfer.run()
self.path = manifesto.cloudfs_path
self._snap_status = self.COMPLETED
finally:
mount.umount(mpoint)
os.rmdir(mpoint)
except:
self._snap_status = self.FAILED
LOG.exception('Caught error while uploading LVM snapshot')
finally:
lvm_snap.destroy()
def rebundle(self):
rebundle_dir = tempfile.mkdtemp()
try:
pl = bus.platform
proj_id = pl.get_numeric_project_id()
proj_name = pl.get_project_id()
cloudstorage = pl.new_storage_client()
tmp_mount_dir = os.path.join(rebundle_dir, 'root')
os.makedirs(tmp_mount_dir)
image_name = 'disk.raw'
image_path = os.path.join(rebundle_dir, image_name)
root_size = coreutils.statvfs('/')['size']
LOG.debug('Creating image file %s' % image_path)
with open(image_path, 'w') as f:
f.truncate(root_size + 1 * 1024)
try:
LOG.debug('Creating partition table on image')
system(('parted', image_path, 'mklabel', 'msdos'))
system(('parted', image_path, 'mkpart', 'primary', 'ext2', 1,
str(root_size / (1024 * 1024))))
# Map disk image
out = system(('kpartx', '-av', image_path))[0]
try:
loop = re.search('(/dev/loop\d+)', out).group(1)
root_dev_name = '/dev/mapper/%sp1' % loop.split('/')[-1]
LOG.info('Creating filesystem')
storage2.filesystem('ext4').mkfs(root_dev_name)
dev_uuid = uuid.uuid4()
system(('tune2fs', '-U', str(dev_uuid), root_dev_name))
mount.mount(root_dev_name, tmp_mount_dir)
try:
lines = system(('/bin/mount', '-l'))[0].splitlines()
exclude_dirs = set()
for line in lines:
mpoint = line.split()[2]
if mpoint != '/':
exclude_dirs.add(mpoint)
exclude_dirs.update(self.exclude_dirs)
excludes = [
os.path.join(ex, '**') for ex in exclude_dirs
]
excludes.extend(self.exclude_files)
excludes.extend(self._excludes)
LOG.info('Copying root filesystem to image')
rsync('/',
tmp_mount_dir,
archive=True,
hard_links=True,
times=True,
sparse=True,
exclude=excludes)
LOG.info('Cleanup image')
self._create_spec_devices(tmp_mount_dir)
LOG.debug('Removing roles-builder user')
sh = pexpect.spawn('/bin/sh')
try:
sh.sendline('chroot %s' % tmp_mount_dir)
sh.expect('#')
sh.sendline('userdel -rf %s' % ROLEBUILDER_USER)
sh.expect('#')
finally:
sh.close()
""" Patch fstab"""
fstab_path = os.path.join(tmp_mount_dir, 'etc/fstab')
if os.path.exists(fstab_path):
with open(fstab_path) as f:
fstab = f.read()
new_fstab = re.sub('UUID=\S+\s+/\s+(.*)',
'UUID=%s / \\1' % dev_uuid,
fstab)
with open(fstab_path, 'w') as f:
f.write(new_fstab)
finally:
mount.umount(root_dev_name)
finally:
system(('kpartx', '-d', image_path))
LOG.info('Compressing image.')
arch_name = '%s.tar.gz' % self._role_name.lower()
arch_path = os.path.join(rebundle_dir, arch_name)
tar = Tar()
tar.create().gzip().sparse()
tar.archive(arch_path)
tar.add(image_name, rebundle_dir)
system(str(tar), shell=True)
finally:
os.unlink(image_path)
try:
LOG.info('Uploading compressed image to cloud storage')
uploader = transfer.Transfer(logger=LOG)
tmp_bucket_name = 'scalr-images-%s-%s' % (random.randint(
1, 1000000), int(time.time()))
try:
remote_path = 'gcs://%s/' % tmp_bucket_name
uploader.upload((arch_path, ), remote_path)
except:
try:
objs = cloudstorage.objects()
objs.delete(bucket=tmp_bucket_name,
object=arch_name).execute()
except:
pass
cloudstorage.buckets().delete(
bucket=tmp_bucket_name).execute()
raise
finally:
os.unlink(arch_path)
finally:
shutil.rmtree(rebundle_dir)
try:
goog_image_name = self._role_name.lower().replace('_', '-')
LOG.info('Registering new image %s' % goog_image_name)
# TODO: check duplicate names
compute = pl.new_compute_client()
current_image_fq = pl.get_image().split('/')
current_img_project = current_image_fq[1]
current_img_name = current_image_fq[3]
current_img_obj = compute.images().get(
project=current_img_project, image=current_img_name).execute()
kernel = current_img_obj['preferredKernel']
image_url = 'http://storage.googleapis.com/%s/%s' % (
tmp_bucket_name, arch_name)
req_body = dict(name=goog_image_name,
sourceType='RAW',
preferredKernel=kernel,
rawDisk=dict(containerType='TAR',
source=image_url))
req = compute.images().insert(project=proj_id, body=req_body)
operation = req.execute()['name']
LOG.info('Waiting for image to register')
def image_is_ready():
req = compute.globalOperations().get(project=proj_id,
operation=operation)
res = req.execute()
if res['status'] == 'DONE':
if res.get('error'):
errors = []
for e in res['error']['errors']:
err_text = '%s: %s' % (e['code'], e['message'])
errors.append(err_text)
raise Exception('\n'.join(errors))
return True
return False
wait_until(image_is_ready, logger=LOG, timeout=600)
finally:
objs = cloudstorage.objects()
objs.delete(bucket=tmp_bucket_name, object=arch_name).execute()
cloudstorage.buckets().delete(bucket=tmp_bucket_name).execute()
return '%s/images/%s' % (proj_name, goog_image_name)
def _mount_image(self, options=None):
LOG.info("Mounting image")
if self.mpoint in self._mtab:
raise HandlerError("Image already mounted")
options = options or []
mount.mount(self.devname, self.mpoint, *options)
def _create(self, volume, snapshot, snap_lv, tranzit_path, complete_cb):
try:
chunk_prefix = '%s.data' % snapshot.id
snapshot.path = None
snap_mpoint = mkdtemp()
try:
opts = []
if volume.fstype == 'xfs':
opts += ['-o', 'nouuid,ro']
mount.mount(snap_lv, snap_mpoint, *opts)
tar_cmd = ['tar', 'cp', '-C', snap_mpoint, '.']
if which('pigz'):
compress_cmd = [which('pigz'), '-5']
else:
compress_cmd = ['gzip', '-5']
self._logger.debug("Creating and compressing snapshot data.")
tar = subprocess.Popen(tar_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True)
compress = subprocess.Popen(compress_cmd,
stdin=tar.stdout,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True)
tar.stdout.close(
) # Allow tar to receive a SIGPIPE if compress exits.
split = threading.Thread(target=self._split,
name='split',
args=(compress.stdout, tranzit_path,
chunk_prefix, snapshot))
split.start()
uploaders = []
for i in range(2):
uploader = threading.Thread(
name="Uploader-%s" % i,
target=self._uploader,
args=(volume.snap_backend['path'], snapshot))
self._logger.debug("Starting uploader '%s'",
uploader.getName())
uploader.start()
uploaders.append(uploader)
self._logger.debug('uploaders started. waiting compress')
compress.wait()
self._logger.debug(
'compress completed (code: %s). waiting split',
compress.returncode)
if compress.returncode:
raise StorageError(
'Compress process terminated with exit code %s. <err>: %s'
% (compress.returncode, compress.stderr.read()))
split.join()
self._logger.debug('split completed. waiting uploaders')
for uploader in uploaders:
uploader.join()
self._logger.debug('uploaders completed')
if self._inner_exc_info:
t, e, s = self._inner_exc_info
raise t, e, s
finally:
self._return_ev.set()
mount.umount(snap_mpoint)
os.rmdir(snap_mpoint)
self._lvm.remove_lv(snap_lv)
self._inner_exc_info = None
self._state_map[snapshot.id] = Snapshot.COMPLETED
except (Exception, BaseException), e:
self._state_map[snapshot.id] = Snapshot.FAILED
self._logger.exception('Snapshot creation failed. %s' % e)
def post_handle_request(self):
if self.umounted:
mount.mount(self.device_name, cassandra.storage_path)
cassandra.start_service()
def test_mount():
with mock.patch('scalarizr.linux.system') as m:
mount.mount('/dev/sdb', '/mnt')
assert m.called
def on_init(self, *args, **kwargs):
bus.on("before_hello", self.on_before_hello)
bus.on("before_host_init", self.on_before_host_init)
bus.on("before_restart", self.on_before_restart)
msg_service = bus.messaging_service
producer = msg_service.get_producer()
producer.on("before_send", self.on_before_message_send)
# Set the hostname to this instance's public hostname
try:
hostname_as_pubdns = int(__ec2__['hostname_as_pubdns'])
except:
hostname_as_pubdns = True
if hostname_as_pubdns:
pub_hostname = self._platform.get_public_hostname()
self._logger.debug('Setting hostname to %s' % pub_hostname)
system2("hostname " + pub_hostname, shell=True)
if disttool.is_ubuntu():
# Ubuntu cloud-init scripts may disable root ssh login
for path in ('/etc/ec2-init/ec2-config.cfg',
'/etc/cloud/cloud.cfg'):
if os.path.exists(path):
c = None
with open(path, 'r') as fp:
c = fp.read()
c = re.sub(
re.compile(r'^disable_root[^:=]*([:=]).*', re.M),
r'disable_root\1 0', c)
with open(path, 'w') as fp:
fp.write(c)
# Add server ssh public key to authorized_keys
authorized_keys_path = "/root/.ssh/authorized_keys"
if os.path.exists(authorized_keys_path):
c = None
with open(authorized_keys_path, 'r') as fp:
c = fp.read()
ssh_key = self._platform.get_ssh_pub_key()
idx = c.find(ssh_key)
if idx == -1:
if c and c[-1] != '\n':
c += '\n'
c += ssh_key + "\n"
self._logger.debug(
"Add server ssh public key to authorized_keys")
elif idx > 0 and c[idx - 1] != '\n':
c = c[0:idx] + '\n' + c[idx:]
self._logger.warn(
'Adding new-line character before server SSH key in authorized_keys file'
)
with open(authorized_keys_path, 'w') as fp:
fp.write(c)
# Mount ephemeral devices
# Seen on eucalyptus:
# - fstab contains invalid fstype and `mount -a` fails
if self._platform.name == 'eucalyptus':
mtab = mount.mounts()
fstab = mount.fstab()
for device in self._platform.instance_store_devices:
if os.path.exists(
device) and device in fstab and device not in mtab:
entry = fstab[device]
try:
mount.mount(device, entry.mpoint, '-o', entry.options)
except:
self._logger.warn(sys.exc_info()[1])
else:
system2('mount -a', shell=True, raise_exc=False)
def test_mount_no_filesystem():
m = mock.Mock(side_effect=linux.LinuxError('', '', 'mount: you must specify the filesystem type\n', 32, ()))
with mock.patch('scalarizr.linux.system', m):
mount.mount('/dev/sdb', '/mnt')
def test_mount():
with mock.patch('scalarizr.linux.system') as m:
mount.mount('/dev/sdb', '/mnt')
assert m.called
def rebundle(self):
rebundle_dir = tempfile.mkdtemp()
try:
pl = bus.platform
proj_id = pl.get_numeric_project_id()
proj_name = pl.get_project_id()
cloudstorage = pl.new_storage_client()
tmp_mount_dir = os.path.join(rebundle_dir, 'root')
os.makedirs(tmp_mount_dir)
image_name = 'disk.raw'
image_path = os.path.join(rebundle_dir, image_name)
root_size = coreutils.statvfs('/')['size']
LOG.debug('Creating image file %s' % image_path)
with open(image_path, 'w') as f:
f.truncate(root_size + 1*1024)
try:
LOG.debug('Creating partition table on image')
system(('parted', image_path, 'mklabel', 'msdos'))
system(('parted', image_path, 'mkpart', 'primary', 'ext2', 1, str(root_size/(1024*1024))))
# Map disk image
out = system(('kpartx', '-av', image_path))[0]
try:
loop = re.search('(/dev/loop\d+)', out).group(1)
root_dev_name = '/dev/mapper/%sp1' % loop.split('/')[-1]
LOG.info('Creating filesystem')
storage2.filesystem('ext4').mkfs(root_dev_name)
dev_uuid = uuid.uuid4()
system(('tune2fs', '-U', str(dev_uuid), root_dev_name))
mount.mount(root_dev_name, tmp_mount_dir)
try:
lines = system(('/bin/mount', '-l'))[0].splitlines()
exclude_dirs = set()
for line in lines:
mpoint = line.split()[2]
if mpoint != '/':
exclude_dirs.add(mpoint)
exclude_dirs.update(self.exclude_dirs)
excludes = [os.path.join(ex, '**') for ex in exclude_dirs]
excludes.extend(self.exclude_files)
excludes.extend(self._excludes)
LOG.info('Copying root filesystem to image')
rsync('/', tmp_mount_dir, archive=True,
hard_links=True,
times=True,
sparse=True,
exclude=excludes)
LOG.info('Cleanup image')
self._create_spec_devices(tmp_mount_dir)
LOG.debug('Removing roles-builder user')
sh = pexpect.spawn('/bin/sh')
try:
sh.sendline('chroot %s' % tmp_mount_dir)
sh.expect('#')
sh.sendline('userdel -rf %s' % ROLEBUILDER_USER)
sh.expect('#')
finally:
sh.close()
""" Patch fstab"""
fstab_path = os.path.join(tmp_mount_dir, 'etc/fstab')
if os.path.exists(fstab_path):
with open(fstab_path) as f:
fstab = f.read()
new_fstab = re.sub('UUID=\S+\s+/\s+(.*)', 'UUID=%s / \\1' % dev_uuid, fstab)
with open(fstab_path, 'w') as f:
f.write(new_fstab)
finally:
mount.umount(root_dev_name)
finally:
system(('kpartx', '-d', image_path))
LOG.info('Compressing image.')
arch_name = '%s.tar.gz' % self._role_name.lower()
arch_path = os.path.join(rebundle_dir, arch_name)
tar = Tar()
tar.create().gzip().sparse()
tar.archive(arch_path)
tar.add(image_name, rebundle_dir)
system(str(tar), shell=True)
finally:
os.unlink(image_path)
try:
LOG.info('Uploading compressed image to cloud storage')
uploader = transfer.Transfer(logger=LOG)
tmp_bucket_name = 'scalr-images-%s-%s' % (
random.randint(1,1000000), int(time.time()))
try:
remote_path = 'gcs://%s/' % tmp_bucket_name
uploader.upload((arch_path,), remote_path)
except:
try:
objs = cloudstorage.objects()
objs.delete(bucket=tmp_bucket_name, object=arch_name).execute()
except:
pass
cloudstorage.buckets().delete(bucket=tmp_bucket_name).execute()
raise
finally:
os.unlink(arch_path)
finally:
shutil.rmtree(rebundle_dir)
try:
goog_image_name = self._role_name.lower().replace('_', '-')
LOG.info('Registering new image %s' % goog_image_name)
# TODO: check duplicate names
compute = pl.new_compute_client()
current_image_fq = pl.get_image().split('/')
current_img_project = current_image_fq[1]
current_img_name = current_image_fq[3]
current_img_obj = compute.images().get(project=current_img_project,
image=current_img_name).execute()
kernel = current_img_obj['preferredKernel']
image_url = 'http://storage.googleapis.com/%s/%s' % (
tmp_bucket_name, arch_name)
req_body = dict(
name=goog_image_name,
sourceType='RAW',
preferredKernel=kernel,
rawDisk=dict(
containerType='TAR',
source=image_url
)
)
req = compute.images().insert(project=proj_id, body=req_body)
operation = req.execute()['name']
LOG.info('Waiting for image to register')
def image_is_ready():
req = compute.globalOperations().get(project=proj_id, operation=operation)
res = req.execute()
if res['status'] == 'DONE':
if res.get('error'):
errors = []
for e in res['error']['errors']:
err_text = '%s: %s' % (e['code'], e['message'])
errors.append(err_text)
raise Exception('\n'.join(errors))
return True
return False
wait_until(image_is_ready, logger=LOG, timeout=600)
finally:
objs = cloudstorage.objects()
objs.delete(bucket=tmp_bucket_name, object=arch_name).execute()
cloudstorage.buckets().delete(bucket=tmp_bucket_name).execute()
return '%s/images/%s' % (proj_name, goog_image_name)
def on_init(self, *args, **kwargs):
bus.on("before_hello", self.on_before_hello)
bus.on("before_host_init", self.on_before_host_init)
bus.on("before_restart", self.on_before_restart)
msg_service = bus.messaging_service
producer = msg_service.get_producer()
producer.on("before_send", self.on_before_message_send)
# Set the hostname to this instance's public hostname
try:
hostname_as_pubdns = int(__ec2__['hostname_as_pubdns'])
except:
hostname_as_pubdns = True
if hostname_as_pubdns:
pub_hostname = self._platform.get_public_hostname()
self._logger.debug('Setting hostname to %s' % pub_hostname)
system2("hostname " + pub_hostname, shell=True)
if disttool.is_ubuntu():
# Ubuntu cloud-init scripts may disable root ssh login
for path in ('/etc/ec2-init/ec2-config.cfg', '/etc/cloud/cloud.cfg'):
if os.path.exists(path):
c = None
with open(path, 'r') as fp:
c = fp.read()
c = re.sub(re.compile(r'^disable_root[^:=]*([:=]).*', re.M), r'disable_root\1 0', c)
with open(path, 'w') as fp:
fp.write(c)
# Add server ssh public key to authorized_keys
authorized_keys_path = "/root/.ssh/authorized_keys"
if os.path.exists(authorized_keys_path):
c = None
with open(authorized_keys_path, 'r') as fp:
c = fp.read()
ssh_key = self._platform.get_ssh_pub_key()
idx = c.find(ssh_key)
if idx == -1:
if c and c[-1] != '\n':
c += '\n'
c += ssh_key + "\n"
self._logger.debug("Add server ssh public key to authorized_keys")
elif idx > 0 and c[idx-1] != '\n':
c = c[0:idx] + '\n' + c[idx:]
self._logger.warn('Adding new-line character before server SSH key in authorized_keys file')
with open(authorized_keys_path, 'w') as fp:
fp.write(c)
# Mount ephemeral devices
# Seen on eucalyptus:
# - fstab contains invalid fstype and `mount -a` fails
if self._platform.name == 'eucalyptus':
mtab = mount.mounts()
fstab = mount.fstab()
for device in self._platform.instance_store_devices:
if os.path.exists(device) and device in fstab and device not in mtab:
entry = fstab[device]
try:
mount.mount(device, entry.mpoint, '-o', entry.options)
except:
self._logger.warn(sys.exc_info()[1])
else:
system2('mount -a', shell=True, raise_exc=False)
def test_mount_no_filesystem():
m = mock.Mock(side_effect=linux.LinuxError(
'', '', 'mount: you must specify the filesystem type\n', 32, ()))
with mock.patch('scalarizr.linux.system', m):
mount.mount('/dev/sdb', '/mnt')
def on_init(self, *args, **kwargs):
bus.on("before_hello", self.on_before_hello)
bus.on("before_host_init", self.on_before_host_init)
bus.on("before_restart", self.on_before_restart)
bus.on("before_reboot_finish", self.on_before_reboot_finish)
try:
system(('ntpdate', '-u', '0.amazon.pool.ntp.org'))
except:
pass
msg_service = bus.messaging_service
producer = msg_service.get_producer()
producer.on("before_send", self.on_before_message_send)
if not os_dist.windows_family and not __node__.get('hostname'):
# Set the hostname to this instance's public hostname
try:
hostname_as_pubdns = int(__ec2__['hostname_as_pubdns'])
except:
hostname_as_pubdns = True
if hostname_as_pubdns:
pub_hostname = self._platform.get_public_hostname()
self._logger.debug('Setting hostname to %s' % pub_hostname)
system2("hostname " + pub_hostname, shell=True)
if disttool.is_ubuntu():
# Ubuntu cloud-init scripts may disable root ssh login
for path in ('/etc/ec2-init/ec2-config.cfg',
'/etc/cloud/cloud.cfg'):
if os.path.exists(path):
c = None
with open(path, 'r') as fp:
c = fp.read()
c = re.sub(
re.compile(r'^disable_root[^:=]*([:=]).*', re.M),
r'disable_root\1 0', c)
with open(path, 'w') as fp:
fp.write(c)
if not linux.os.windows_family:
# Add server ssh public key to authorized_keys
ssh_key = self._platform.get_ssh_pub_key()
if ssh_key:
add_authorized_key(ssh_key)
# Mount ephemeral devices
# Seen on eucalyptus:
# - fstab contains invalid fstype and `mount -a` fails
if self._platform.name == 'eucalyptus':
mtab = mount.mounts()
fstab = mount.fstab()
for device in self._platform.instance_store_devices:
if os.path.exists(
device) and device in fstab and device not in mtab:
entry = fstab[device]
try:
mount.mount(device, entry.mpoint, '-o', entry.options)
except:
self._logger.warn(sys.exc_info()[1])
else:
if not os_dist.windows_family:
system2('mount -a', shell=True, raise_exc=False)
def _mount_image(self, options=None):
LOG.info("Mounting image")
if self.mpoint in self._mtab:
raise HandlerError("Image already mounted")
options = options or []
mount.mount(self.devname, self.mpoint, *options)