def enabled():
# amazon linux doesn't have iptables service installed by default,
# which makes "chkconfig --list iptables" fail
# update: amzn >= 6.4 doesn't allow installing iptables-services;
# however, the latest version of iptables itself suits all our needs
if linux.os["name"] == "Amazon":
if linux.os["release"] == "6.3":
pkgmgr.installed("iptables-services")
else: # 6.4 and higher
# Upgrading iptables-1.4.18-1.16 to iptables-1.4.18-1.19 did
# the job
# Reverted: cannot reproduce the error
# # We need to remove iptables first because it can be i686 version
# # installed and installing x86_64 over it causes error
# pkgmgr.remove("iptables")
pkgmgr.latest("iptables")
if linux.os['family'] in ('RedHat', 'Oracle'):
try:
out = redhat.chkconfig(list="iptables")[0]
return bool(re.search(r"iptables.*?\s\d:on", out))
except linux.LinuxError, e:
if 'not referenced in any runlevel' in str(e):
return False
else:
raise
def enabled():
# amazon linux doesn't have iptables service installed by default,
# which makes "chkconfig --list iptables" fail
# update: amzn >= 6.4 doesn't allow installing iptables-services;
# however, the latest version of iptables itself suits all our needs
if linux.os["name"] == "Amazon":
if linux.os["release"] == "6.3":
pkgmgr.installed("iptables-services")
else: # 6.4 and higher
# Upgrading iptables-1.4.18-1.16 to iptables-1.4.18-1.19 did
# the job
# Reverted: cannot reproduce the error
# # We need to remove iptables first because it can be i686 version
# # installed and installing x86_64 over it causes error
# pkgmgr.remove("iptables")
pkgmgr.latest("iptables")
if linux.os['family'] in ('RedHat', 'Oracle'):
try:
out = redhat.chkconfig(list="iptables")[0]
return bool(re.search(r"iptables.*?\s\d:on", out))
except linux.LinuxError, e:
if 'not referenced in any runlevel' in str(e):
return False
else:
raise
def test_latest(mgr):
mgr().info.return_value = {"candidate": None, "installed": None}
pkgmgr.latest("thing", True)
mgr().install.assert_called_once_with("thing", None)
mgr().info.return_value = {"candidate": "2.0", "installed": None}
pkgmgr.latest("thing", False)
mgr().install.assert_called_with("thing", "2.0")
def test_latest(mgr):
mgr().info.return_value = {'candidate': None,
'installed': None}
pkgmgr.latest('thing', True)
mgr().install.assert_called_once_with('thing', None)
mgr().info.return_value = {'candidate': '2.0',
'installed': None}
pkgmgr.latest('thing', False)
mgr().install.assert_called_with('thing', '2.0')
def test_latest(mgr):
mgr().info.return_value = {'candidate': None,
'installed': None}
pkgmgr.latest('thing', True)
mgr().install.assert_called_once_with('thing', None)
mgr().info.return_value = {'candidate': '2.0',
'installed': None}
pkgmgr.latest('thing', False)
mgr().install.assert_called_with('thing', '2.0')
def enabled():
# amazon linux doesn't have iptables service installed by default,
# which makes "chkconfig --list iptables" fail
# update: amzn >= 6.4 doesn't allow installing iptables-services;
# however, the latest version of iptables itself suits all or needs
if linux.os["name"] == "Amazon":
if linux.os["release"] == "6.3":
pkgmgr.installed("iptables-services")
else: # 6.4 and higher
# Upgrading iptables-1.4.18-1.16 to iptables-1.4.18-1.19 did
# the job
pkgmgr.latest("iptables")
if linux.os['family'] in ('RedHat', 'Oracle'):
out = redhat.chkconfig(list="iptables")[0]
return bool(re.search(r"iptables.*?\s\d:on", out))
else:
return os.access(IPTABLES_BIN, os.X_OK)
def enabled():
# amazon linux doesn't have iptables service installed by default,
# which makes "chkconfig --list iptables" fail
# update: amzn >= 6.4 doesn't allow installing iptables-services;
# however, the latest version of iptables itself suits all or needs
if linux.os["name"] == "Amazon":
if linux.os["release"] == "6.3":
pkgmgr.installed("iptables-services")
else: # 6.4 and higher
# Upgrading iptables-1.4.18-1.16 to iptables-1.4.18-1.19 did
# the job
pkgmgr.latest("iptables")
if linux.os['family'] in ('RedHat', 'Oracle'):
out = redhat.chkconfig(list="iptables")[0]
return bool(re.search(r"iptables.*?\s\d:on", out))
else:
return os.access(IPTABLES_BIN, os.X_OK)
def rebundle(self):
rebundle_dir = tempfile.mkdtemp()
try:
pl = bus.platform
proj_id = pl.get_numeric_project_id()
proj_name = pl.get_project_id()
cloudstorage = pl.new_storage_client()
root_part_path = os.path.realpath('/dev/root')
root_part_sysblock_path = glob.glob('/sys/block/*/%s' % os.path.basename(root_part_path))[0]
root_device = '/dev/%s' % os.path.basename(os.path.dirname(root_part_sysblock_path))
arch_name = '%s.tar.gz' % self._role_name.lower()
arch_path = os.path.join(rebundle_dir, arch_name)
# update gcimagebundle
try:
pkgmgr.latest(self.gcimagebundle_pkg_name)
except:
e = sys.exc_info()[1]
LOG.warn('Gcimagebundle update failed: %s' % e)
if os_dist.redhat_family:
semanage = software.which('semanage')
if not semanage:
pkgmgr.installed('policycoreutils-python')
semanage = software.which('semanage')
util.system2((semanage, 'permissive', '-a', 'rsync_t'))
gc_img_bundle_bin = software.which('gcimagebundle')
o, e, p = util.system2((gc_img_bundle_bin,
'-d', root_device,
'-e', ','.join(self.exclude_dirs),
'-o', rebundle_dir,
'--output_file_name', arch_name), raise_exc=False)
if p:
raise HandlerError('Gcimagebundle util returned non-zero code %s. Stderr: %s' % (p, e))
try:
LOG.info('Uploading compressed image to cloud storage')
tmp_bucket_name = 'scalr-images-%s-%s' % (random.randint(1, 1000000), int(time.time()))
remote_path = 'gcs://%s/%s' % (tmp_bucket_name, arch_name)
arch_size = os.stat(arch_path).st_size
uploader = FileTransfer(src=arch_path, dst=remote_path)
try:
upload_result = uploader.run()
if upload_result['failed']:
errors = [str(failed['exc_info'][1]) for failed in upload_result['failed']]
raise HandlerError('Image upload failed. Errors:\n%s' % '\n'.join(errors))
assert arch_size == upload_result['completed'][0]['size']
except:
with util.capture_exception(LOG):
objs = cloudstorage.objects()
objs.delete(bucket=tmp_bucket_name, object=arch_name).execute()
cloudstorage.buckets().delete(bucket=tmp_bucket_name).execute()
finally:
os.unlink(arch_path)
finally:
shutil.rmtree(rebundle_dir)
goog_image_name = self._role_name.lower().replace('_', '-') + '-' + str(int(time.time()))
try:
LOG.info('Registering new image %s' % goog_image_name)
compute = pl.new_compute_client()
image_url = 'http://storage.googleapis.com/%s/%s' % (tmp_bucket_name, arch_name)
req_body = dict(
name=goog_image_name,
sourceType='RAW',
rawDisk=dict(
source=image_url
)
)
req = compute.images().insert(project=proj_id, body=req_body)
operation = req.execute()['name']
LOG.info('Waiting for image to register')
def image_is_ready():
req = compute.globalOperations().get(project=proj_id, operation=operation)
res = req.execute()
if res['status'] == 'DONE':
if res.get('error'):
errors = []
for e in res['error']['errors']:
err_text = '%s: %s' % (e['code'], e['message'])
errors.append(err_text)
raise Exception('\n'.join(errors))
return True
return False
util.wait_until(image_is_ready, logger=LOG, timeout=600)
finally:
try:
objs = cloudstorage.objects()
objs.delete(bucket=tmp_bucket_name, object=arch_name).execute()
cloudstorage.buckets().delete(bucket=tmp_bucket_name).execute()
except:
e = sys.exc_info()[1]
LOG.error('Faled to remove image compressed source: %s' % e)
return '%s/images/%s' % (proj_name, goog_image_name)
def rebundle(self):
rebundle_dir = tempfile.mkdtemp()
try:
pl = bus.platform
proj_id = pl.get_numeric_project_id()
proj_name = pl.get_project_id()
cloudstorage = pl.new_storage_client()
root_part_path = os.path.realpath('/dev/root')
root_part_sysblock_path = glob.glob(
'/sys/block/*/%s' % os.path.basename(root_part_path))[0]
root_device = '/dev/%s' % os.path.basename(
os.path.dirname(root_part_sysblock_path))
arch_name = '%s.tar.gz' % self._role_name.lower()
arch_path = os.path.join(rebundle_dir, arch_name)
# update gcimagebundle
try:
pkgmgr.latest(self.gcimagebundle_pkg_name)
except:
e = sys.exc_info()[1]
LOG.warn('Gcimagebundle update failed: %s' % e)
if os_dist.redhat_family:
semanage = software.which('semanage')
if not semanage:
pkgmgr.installed('policycoreutils-python')
semanage = software.which('semanage')
util.system2((semanage, 'permissive', '-a', 'rsync_t'))
gc_img_bundle_bin = software.which('gcimagebundle')
o, e, p = util.system2(
(gc_img_bundle_bin, '-d', root_device, '-e', ','.join(
self.exclude_dirs), '-o', rebundle_dir,
'--output_file_name', arch_name),
raise_exc=False)
if p:
raise HandlerError(
'Gcimagebundle util returned non-zero code %s. Stderr: %s'
% (p, e))
try:
LOG.info('Uploading compressed image to cloud storage')
tmp_bucket_name = 'scalr-images-%s-%s' % (random.randint(
1, 1000000), int(time.time()))
remote_path = 'gcs://%s/%s' % (tmp_bucket_name, arch_name)
arch_size = os.stat(arch_path).st_size
uploader = FileTransfer(src=arch_path, dst=remote_path)
try:
upload_result = uploader.run()
if upload_result['failed']:
errors = [
str(failed['exc_info'][1])
for failed in upload_result['failed']
]
raise HandlerError('Image upload failed. Errors:\n%s' %
'\n'.join(errors))
assert arch_size == upload_result['completed'][0]['size']
except:
with util.capture_exception(LOG):
objs = cloudstorage.objects()
objs.delete(bucket=tmp_bucket_name,
object=arch_name).execute()
cloudstorage.buckets().delete(
bucket=tmp_bucket_name).execute()
finally:
os.unlink(arch_path)
finally:
shutil.rmtree(rebundle_dir)
goog_image_name = self._role_name.lower().replace(
'_', '-') + '-' + str(int(time.time()))
try:
LOG.info('Registering new image %s' % goog_image_name)
compute = pl.new_compute_client()
image_url = 'http://storage.googleapis.com/%s/%s' % (
tmp_bucket_name, arch_name)
req_body = dict(name=goog_image_name,
sourceType='RAW',
rawDisk=dict(source=image_url))
req = compute.images().insert(project=proj_id, body=req_body)
operation = req.execute()['name']
LOG.info('Waiting for image to register')
def image_is_ready():
req = compute.globalOperations().get(project=proj_id,
operation=operation)
res = req.execute()
if res['status'] == 'DONE':
if res.get('error'):
errors = []
for e in res['error']['errors']:
err_text = '%s: %s' % (e['code'], e['message'])
errors.append(err_text)
raise Exception('\n'.join(errors))
return True
return False
util.wait_until(image_is_ready, logger=LOG, timeout=600)
finally:
try:
objs = cloudstorage.objects()
objs.delete(bucket=tmp_bucket_name, object=arch_name).execute()
cloudstorage.buckets().delete(bucket=tmp_bucket_name).execute()
except:
e = sys.exc_info()[1]
LOG.error('Faled to remove image compressed source: %s' % e)
return '%s/images/%s' % (proj_name, goog_image_name)
def _prepare_software(self):
try:
pkgmgr.latest(self.gcimagebundle_pkg_name)
except (Exception, BaseException), e:
LOG.warn('Gcimagebundle update failed: %s' % e)
def _prepare_software(self):
try:
pkgmgr.latest(self.gcimagebundle_pkg_name)
except (Exception, BaseException), e:
LOG.warn('Gcimagebundle update failed: %s' % e)
def rebundle(self):
rebundle_dir = tempfile.mkdtemp()
try:
pl = bus.platform
proj_id = pl.get_numeric_project_id()
proj_name = pl.get_project_id()
cloudstorage = pl.get_storage_conn()
# Determine the root filesystem size
devices = coreutils.df()
root_disk = firstmatched(lambda x: x.mpoint == '/', devices)
if not root_disk:
raise HandlerError("Can't find root device")
# in bytes adjusted to 512 block device size
fssize = (root_disk.size * 1000 / 512) * 512
# Old code. Should be reworked
if os.path.exists('/dev/root'):
root_part_path = os.path.realpath('/dev/root')
else:
rootfs_stat = os.stat('/')
root_device_minor = os.minor(rootfs_stat.st_dev)
root_device_major = os.major(rootfs_stat.st_dev)
root_part_path = os.path.realpath('/dev/block/{0}:{1}'.format(
root_device_major, root_device_minor))
root_part_sysblock_path = glob.glob(
'/sys/block/*/%s' % os.path.basename(root_part_path))[0]
root_device = '/dev/%s' % os.path.basename(
os.path.dirname(root_part_sysblock_path))
arch_name = '%s.tar.gz' % self._role_name.lower()
arch_path = os.path.join(rebundle_dir, arch_name)
# update gcimagebundle
try:
pkgmgr.latest(self.gcimagebundle_pkg_name)
except:
e = sys.exc_info()[1]
LOG.warn('Gcimagebundle update failed: %s' % e)
if os_dist.redhat_family:
semanage = software.which('semanage')
if not semanage:
pkgmgr.installed('policycoreutils-python')
semanage = software.which('semanage')
util.system2((semanage, 'permissive', '-a', 'rsync_t'))
gc_img_bundle_bin = software.which('gcimagebundle')
o, e, p = util.system2(
(gc_img_bundle_bin, '-d', root_device, '-e', ','.join(
self.exclude_dirs), '--fssize', str(fssize), '-o',
rebundle_dir, '--output_file_name', arch_name),
raise_exc=False)
if p:
raise HandlerError(
'Gcimagebundle util returned non-zero code %s. Stderr: %s'
% (p, e))
try:
LOG.info('Uploading compressed image to cloud storage')
tmp_bucket_name = 'scalr-images-%s-%s' % (random.randint(
1, 1000000), int(time.time()))
remote_dir = 'gcs://%s' % tmp_bucket_name
def progress_cb(progress):
LOG.debug('Uploading {perc}%'.format(
perc=progress / os.path.getsize(arch_path)))
uploader = largetransfer.Upload(arch_path,
remote_dir,
simple=True,
progress_cb=progress_cb)
uploader.apply_async()
try:
try:
uploader.join()
except:
if uploader.error:
error = uploader.error[1]
else:
error = sys.exc_info()[1]
msg = 'Image upload failed. Error:\n{error}'
msg = msg.format(error=error)
raise HandlerError(msg)
except:
with util.capture_exception(LOG):
objs = cloudstorage.objects()
objs.delete(bucket=tmp_bucket_name,
object=arch_name).execute()
cloudstorage.buckets().delete(
bucket=tmp_bucket_name).execute()
finally:
os.unlink(arch_path)
finally:
shutil.rmtree(rebundle_dir)
goog_image_name = self._role_name.lower().replace(
'_', '-') + '-' + str(int(time.time()))
try:
LOG.info('Registering new image %s' % goog_image_name)
compute = pl.get_compute_conn()
image_url = 'http://storage.googleapis.com/%s/%s' % (
tmp_bucket_name, arch_name)
req_body = dict(name=goog_image_name,
sourceType='RAW',
rawDisk=dict(source=image_url))
req = compute.images().insert(project=proj_id, body=req_body)
operation = req.execute()['name']
LOG.info('Waiting for image to register')
def image_is_ready():
req = compute.globalOperations().get(project=proj_id,
operation=operation)
res = req.execute()
if res['status'] == 'DONE':
if res.get('error'):
errors = []
for e in res['error']['errors']:
err_text = '%s: %s' % (e['code'], e['message'])
errors.append(err_text)
raise Exception('\n'.join(errors))
return True
return False
util.wait_until(image_is_ready, logger=LOG, timeout=600)
finally:
try:
objs = cloudstorage.objects()
objs.delete(bucket=tmp_bucket_name, object=arch_name).execute()
cloudstorage.buckets().delete(bucket=tmp_bucket_name).execute()
except:
e = sys.exc_info()[1]
LOG.error('Failed to remove image compressed source: %s' % e)
return '%s/images/%s' % (proj_name, goog_image_name)