def build_agent(container_id=None):
logger.info('Rebuilding agent package')
container_id = container_id or work.last_container_id
quiet_docker('exec', container_id, 'tar', 'czf',
configuration.agent_package_path, '-C',
path(constants.AGENT_TEMPLATE_DIR).dirname(),
path(constants.AGENT_TEMPLATE_DIR).basename())
def _serve():
host = get_host()
port = 9797
local_resources_url = 'http://{}:{}/{}'.format(
host, port, work.cached_resources_tar_path.basename())
process = serve(work.dir, host=host, port=port, _bg=True)
logger.info('Resources tar available at {}'.format(local_resources_url))
return process, local_resources_url
def prepare(config_output=None, label=None, details_path=None, tag=None):
config_output = config_output or constants.CONFIG_YAML
container_id, container_ip = _create_base_container(
label=label, details_path=details_path, tag=tag)
logger.info('Container {0} started on ip {1}'.format(
container_id, container_ip))
_ssh_setup(container_id=container_id, container_ip=container_ip)
_write_config(container_ip=container_ip, config_path=config_output)
return container_id, container_ip
def _serve():
host = get_host()
port = 9797
local_rpm_url = 'http://{}:{}/{}'.format(
host, port, work.cached_install_rpm_path.basename())
process = serve(work.dir, host=host, port=port, _bg=True)
logger.info('Install RPM available at {}'.format(local_rpm_url))
_wait_for_file_server(local_rpm_url)
return process, local_rpm_url
def _download_install_rpm(no_progress):
rpm_local_path = work.cached_install_rpm_path
if rpm_local_path.exists():
rpm_local_path.unlink()
rpm_url = get_rpm_url()
logger.info('Downloading install RPM from {}. This might take a '
'while'.format(rpm_url))
files.download(url=rpm_url,
output_path=rpm_local_path,
no_progress=no_progress)
def prepare(inputs_output=None, label=None, details_path=None, tag=None):
inputs_output = inputs_output or constants.INPUTS_YAML
container_id, container_ip = _create_base_container(
label=label,
details_path=details_path,
tag=tag)
logger.info('Container {0} started on ip {1}'
.format(container_id, container_ip))
_ssh_setup(container_id=container_id, container_ip=container_ip)
_write_inputs(container_ip=container_ip, inputs_path=inputs_output)
def _download_resources_tar(no_progress):
resources_local_path = work.cached_resources_tar_path
if resources_local_path.exists():
resources_local_path.unlink()
resources_url = _get_resources_url()
logger.info('Downloading resources tar from {}. This might take a '
'while'.format(resources_url))
files.download(url=resources_url,
output_path=resources_local_path,
no_progress=no_progress)
def install_docker(version=None, container_id=None):
container_id = container_id or work.last_container_id
try:
quiet_docker('exec', container_id, *'which docker'.split(' '))
logger.info('Docker already installed on container. Doing nothing')
return
except sh.ErrorReturnCode:
pass
version = version or _get_docker_version(container_id)
install_docker_command = 'yum install -y -q {}'.format(version)
docker('exec', container_id, *install_docker_command.split(' '))
def watch(container_id=None, rebuild_agent=False, interval=2):
container_id = container_id or work.last_container_id
services_to_restart = set()
services_to_restart_lock = threading.Lock()
class Handler(events.FileSystemEventHandler):
def __init__(self, services):
self.services = set(services)
def on_modified(self, event):
if event.is_directory:
with services_to_restart_lock:
services_to_restart.update(self.services)
observer = observers.Observer()
for package, services in configuration.package_services.items():
src = '{}/{}/{}'.format(configuration.source_root,
configuration.package_dir[package], package)
observer.schedule(Handler(services), path=src, recursive=True)
observer.start()
scheduler = sched.scheduler(time.time, time.sleep)
def restart_changed_services():
with services_to_restart_lock:
current_services_to_restart = services_to_restart.copy()
services_to_restart.clear()
for service in current_services_to_restart:
if service == constants.AGENT_STUB_SERVICE and rebuild_agent:
build_agent(container_id)
else:
_restart_service(container_id, service)
scheduler.enter(interval, 1, restart_changed_services, ())
restart_changed_services()
message = 'Filesystem watch started.'
if rebuild_agent:
message = (
'{} Relevant services will be restarted and CentOS agent '
'package (dockercompute) rebuilt on code changes.'.format(message))
else:
message = ('{} Relevant services will be restarted on code changes. '
'Use --rebuild-agent to also rebuild Centos agent package '
'(dockercompute) on code changes.'.format(message))
logger.info(message)
try:
scheduler.run()
except KeyboardInterrupt:
pass
def clean(label=None):
label = label or []
docker_tag1 = configuration.clean_image_docker_tag
docker_tag2 = configuration.manager_image_docker_tag
ps_command = [
'-aq', '--filter', 'ancestor={}'.format(docker_tag1), '--filter',
'ancestor={}'.format(docker_tag2)
]
for l in label:
ps_command += ['--filter', 'label={}'.format(l)]
containers = quiet_docker.ps(ps_command).split('\n')
containers = [c.strip() for c in containers if c.strip()]
logger.info('Removing containers')
for container in containers:
docker.rm('-f', container)
def _update_container(container_id, container_ip):
logger.info('Updating files on the container')
with tempfile.NamedTemporaryFile() as f:
json.dump(
{
'ip': container_ip,
'is_debug_on': bool(os.environ.get('DEBUG_MODE')),
'host': _get_debug_ip(),
'services': constants.ALL_IP_SERVICES
}, f)
f.flush()
cp(source=f.name,
target=':{}'.format(constants.DATA_JSON_TARGET_PATH),
container_id=container_id)
def _install_manager(container_id, container_ip, config_path, rpm_url=None):
rpm_url = rpm_url or install_rpm_server.get_rpm_url()
logger.info('Downloading install RPM from: {0}'.format(rpm_url))
exc('curl {0} -o {1}'.format(rpm_url, constants.INSTALL_RPM_PATH),
container_id)
logger.info('Installing RPM...')
exc('yum install -y {0}'.format(constants.INSTALL_RPM_PATH), container_id)
logger.info('Removing install RPM...')
exc('rm {0}'.format(constants.INSTALL_RPM_PATH))
logger.info('Copying configuration...')
cp(config_path, ':/etc/cloudify/config.yaml')
logger.info('Installing Cloudify Manager...')
install_cmd = 'cfy_manager install --private-ip {0} --public-ip ' \
'{0}'.format(container_ip)
exc(install_cmd, container_id)
def install_docker(version=None, container_id=None):
container_id = container_id or work.last_container_id
try:
quiet_docker('exec', container_id, *'which docker'.split(' '))
logger.info('Docker already installed on container. Doing nothing')
return
except sh.ErrorReturnCode:
pass
cp(resources.DIR / 'docker.repo', ':/etc/yum.repos.d/docker.repo',
container_id=container_id)
if not version:
try:
version = quiet_docker.version('-f', '{{.Client.Version}}').strip()
except sh.ErrorReturnCode as e:
version = e.stdout.strip()
install_docker_command = 'yum install -y -q docker-engine-{}'.format(
version)
docker('exec', container_id, *install_docker_command.split(' '))
def pull_image(no_progress=False):
# try contacting the docker daemon first, to break early if it's not
# reachable - before the long download
quiet_docker.version()
online_sha1 = requests.get(
configuration.manager_image_commit_sha_url).text.strip()
local_sha1 = work.last_pulled_image_commit_sha1
if online_sha1 == local_sha1:
logger.info(
'Current image is the latest image. It is based on the '
'following commit in the manager blueprints repo: {}'.format(
local_sha1))
return
logger.info('Download manager image from {} to {}'.format(
configuration.manager_image_url, work.pulled_image_path))
if os.path.exists(work.pulled_image_path):
os.remove(work.pulled_image_path)
files.download(url=configuration.manager_image_url,
output_path=work.pulled_image_path,
no_progress=no_progress)
logger.info('Loading image into docker (may take a while)')
quiet_docker.load(gzip('-dc',
work.pulled_image_path,
_piped=True,
_out_bufsize=constants.BUFFER_SIZE),
_in_bufsize=constants.BUFFER_SIZE)
work.last_pulled_image_commit_sha1 = online_sha1
def save_image(container_id=None,
tag=None,
output_file=None,
skip_agent_prepare=False):
container_id = container_id or work.last_container_id
docker_tag = tag or configuration.manager_image_docker_tag
logger.info('Preparing manager container before saving as docker image')
_run_container_preparation_scripts(container_id, skip_agent_prepare)
logger.info('Saving manager container to image {}'.format(docker_tag))
quiet_docker.stop(container_id)
quiet_docker.commit(container_id, docker_tag)
logger.info("Removing container. Run 'docl run' to start it again")
quiet_docker.rm('-f', container_id)
if output_file:
logger.info('Saving manager image to {}. This may take a while'.format(
output_file))
gzip(quiet_docker.save(docker_tag,
_piped=True,
_tty_out=False,
_out_bufsize=constants.BUFFER_SIZE),
_in_bufsize=constants.BUFFER_SIZE,
_out=output_file)
def save_image(container_id=None,
tag=None,
output_file=None,
skip_agent_prepare=False):
container_id = container_id or work.last_container_id
docker_tag = tag or configuration.manager_image_docker_tag
logger.info('Preparing manager container before saving as docker image')
cp(source=resources.DIR / 'update_running_system.py',
target=':{}'.format(constants.PY_SCRIPT_TARGET_PATH),
container_id=container_id)
cp(source=resources.DIR / 'prepare_save_image.py',
target=':{}'.format(constants.PREPARE_SAVE_IMAGE_TARGET_PATH),
container_id=container_id)
params = {
'data_json_path': constants.DATA_JSON_TARGET_PATH,
'pydevd_egg_url': configuration.pydevd_egg_url,
'skip_agent_prepare': skip_agent_prepare,
'agent_template_dir': constants.AGENT_TEMPLATE_DIR,
'agent_package_path': configuration.agent_package_path
}
docker('exec', container_id, 'python',
constants.PREPARE_SAVE_IMAGE_TARGET_PATH,
base64.b64encode(json.dumps(params)))
logger.info('Saving manager container to image {}'.format(docker_tag))
quiet_docker.stop(container_id)
quiet_docker.commit(container_id, docker_tag)
logger.info("Removing container. Run 'docl run' to start it again")
quiet_docker.rm('-f', container_id)
if output_file:
logger.info('Saving manager image to {}. This may take a while'
.format(output_file))
gzip(quiet_docker.save(docker_tag,
_piped=True,
_tty_out=False,
_out_bufsize=constants.BUFFER_SIZE),
_in_bufsize=constants.BUFFER_SIZE,
_out=output_file)
def pull_image(no_progress=False):
online_sha1 = requests.get(
configuration.manager_image_commit_sha_url).text.strip()
local_sha1 = work.last_pulled_image_commit_sha1
if online_sha1 == local_sha1:
logger.info('Current image is the latest image. It is based on the '
'following commit in the manager blueprints repo: {}'
.format(local_sha1))
return
logger.info('Download manager image from {} to {}'
.format(configuration.manager_image_url,
work.pulled_image_path))
if os.path.exists(work.pulled_image_path):
os.remove(work.pulled_image_path)
files.download(url=configuration.manager_image_url,
output_path=work.pulled_image_path,
no_progress=no_progress)
logger.info('Loading image into docker')
docker.load(gzip('-dc', work.pulled_image_path,
_piped=True,
_out_bufsize=constants.BUFFER_SIZE),
_in_bufsize=constants.BUFFER_SIZE)
work.last_pulled_image_commit_sha1 = online_sha1
def remove_image(tag=None):
logger.info('Removing image {}'.format(tag))
quiet_docker.rmi(tag)
def _restart_service(container_id, service):
logger.info('Restarting {}'.format(service))
quiet_docker('exec', container_id, 'systemctl', 'restart', service)
