def build_agent(container_id=None): logger.info('Rebuilding agent package') container_id = container_id or work.last_container_id quiet_docker('exec', container_id, 'tar', 'czf', configuration.agent_package_path, '-C', path(constants.AGENT_TEMPLATE_DIR).dirname(), path(constants.AGENT_TEMPLATE_DIR).basename())
def _serve(): host = get_host() port = 9797 local_resources_url = 'http://{}:{}/{}'.format( host, port, work.cached_resources_tar_path.basename()) process = serve(work.dir, host=host, port=port, _bg=True) logger.info('Resources tar available at {}'.format(local_resources_url)) return process, local_resources_url
def prepare(config_output=None, label=None, details_path=None, tag=None): config_output = config_output or constants.CONFIG_YAML container_id, container_ip = _create_base_container( label=label, details_path=details_path, tag=tag) logger.info('Container {0} started on ip {1}'.format( container_id, container_ip)) _ssh_setup(container_id=container_id, container_ip=container_ip) _write_config(container_ip=container_ip, config_path=config_output) return container_id, container_ip
def _serve(): host = get_host() port = 9797 local_rpm_url = 'http://{}:{}/{}'.format( host, port, work.cached_install_rpm_path.basename()) process = serve(work.dir, host=host, port=port, _bg=True) logger.info('Install RPM available at {}'.format(local_rpm_url)) _wait_for_file_server(local_rpm_url) return process, local_rpm_url
def _download_install_rpm(no_progress): rpm_local_path = work.cached_install_rpm_path if rpm_local_path.exists(): rpm_local_path.unlink() rpm_url = get_rpm_url() logger.info('Downloading install RPM from {}. This might take a ' 'while'.format(rpm_url)) files.download(url=rpm_url, output_path=rpm_local_path, no_progress=no_progress)
def prepare(inputs_output=None, label=None, details_path=None, tag=None): inputs_output = inputs_output or constants.INPUTS_YAML container_id, container_ip = _create_base_container( label=label, details_path=details_path, tag=tag) logger.info('Container {0} started on ip {1}' .format(container_id, container_ip)) _ssh_setup(container_id=container_id, container_ip=container_ip) _write_inputs(container_ip=container_ip, inputs_path=inputs_output)
def _download_resources_tar(no_progress): resources_local_path = work.cached_resources_tar_path if resources_local_path.exists(): resources_local_path.unlink() resources_url = _get_resources_url() logger.info('Downloading resources tar from {}. This might take a ' 'while'.format(resources_url)) files.download(url=resources_url, output_path=resources_local_path, no_progress=no_progress)
def install_docker(version=None, container_id=None): container_id = container_id or work.last_container_id try: quiet_docker('exec', container_id, *'which docker'.split(' ')) logger.info('Docker already installed on container. Doing nothing') return except sh.ErrorReturnCode: pass version = version or _get_docker_version(container_id) install_docker_command = 'yum install -y -q {}'.format(version) docker('exec', container_id, *install_docker_command.split(' '))
def watch(container_id=None, rebuild_agent=False, interval=2): container_id = container_id or work.last_container_id services_to_restart = set() services_to_restart_lock = threading.Lock() class Handler(events.FileSystemEventHandler): def __init__(self, services): self.services = set(services) def on_modified(self, event): if event.is_directory: with services_to_restart_lock: services_to_restart.update(self.services) observer = observers.Observer() for package, services in configuration.package_services.items(): src = '{}/{}/{}'.format(configuration.source_root, configuration.package_dir[package], package) observer.schedule(Handler(services), path=src, recursive=True) observer.start() scheduler = sched.scheduler(time.time, time.sleep) def restart_changed_services(): with services_to_restart_lock: current_services_to_restart = services_to_restart.copy() services_to_restart.clear() for service in current_services_to_restart: if service == constants.AGENT_STUB_SERVICE and rebuild_agent: build_agent(container_id) else: _restart_service(container_id, service) scheduler.enter(interval, 1, restart_changed_services, ()) restart_changed_services() message = 'Filesystem watch started.' if rebuild_agent: message = ( '{} Relevant services will be restarted and CentOS agent ' 'package (dockercompute) rebuilt on code changes.'.format(message)) else: message = ('{} Relevant services will be restarted on code changes. ' 'Use --rebuild-agent to also rebuild Centos agent package ' '(dockercompute) on code changes.'.format(message)) logger.info(message) try: scheduler.run() except KeyboardInterrupt: pass
def clean(label=None): label = label or [] docker_tag1 = configuration.clean_image_docker_tag docker_tag2 = configuration.manager_image_docker_tag ps_command = [ '-aq', '--filter', 'ancestor={}'.format(docker_tag1), '--filter', 'ancestor={}'.format(docker_tag2) ] for l in label: ps_command += ['--filter', 'label={}'.format(l)] containers = quiet_docker.ps(ps_command).split('\n') containers = [c.strip() for c in containers if c.strip()] logger.info('Removing containers') for container in containers: docker.rm('-f', container)
def _update_container(container_id, container_ip): logger.info('Updating files on the container') with tempfile.NamedTemporaryFile() as f: json.dump( { 'ip': container_ip, 'is_debug_on': bool(os.environ.get('DEBUG_MODE')), 'host': _get_debug_ip(), 'services': constants.ALL_IP_SERVICES }, f) f.flush() cp(source=f.name, target=':{}'.format(constants.DATA_JSON_TARGET_PATH), container_id=container_id)
def _install_manager(container_id, container_ip, config_path, rpm_url=None): rpm_url = rpm_url or install_rpm_server.get_rpm_url() logger.info('Downloading install RPM from: {0}'.format(rpm_url)) exc('curl {0} -o {1}'.format(rpm_url, constants.INSTALL_RPM_PATH), container_id) logger.info('Installing RPM...') exc('yum install -y {0}'.format(constants.INSTALL_RPM_PATH), container_id) logger.info('Removing install RPM...') exc('rm {0}'.format(constants.INSTALL_RPM_PATH)) logger.info('Copying configuration...') cp(config_path, ':/etc/cloudify/config.yaml') logger.info('Installing Cloudify Manager...') install_cmd = 'cfy_manager install --private-ip {0} --public-ip ' \ '{0}'.format(container_ip) exc(install_cmd, container_id)
def install_docker(version=None, container_id=None): container_id = container_id or work.last_container_id try: quiet_docker('exec', container_id, *'which docker'.split(' ')) logger.info('Docker already installed on container. Doing nothing') return except sh.ErrorReturnCode: pass cp(resources.DIR / 'docker.repo', ':/etc/yum.repos.d/docker.repo', container_id=container_id) if not version: try: version = quiet_docker.version('-f', '{{.Client.Version}}').strip() except sh.ErrorReturnCode as e: version = e.stdout.strip() install_docker_command = 'yum install -y -q docker-engine-{}'.format( version) docker('exec', container_id, *install_docker_command.split(' '))
def pull_image(no_progress=False): # try contacting the docker daemon first, to break early if it's not # reachable - before the long download quiet_docker.version() online_sha1 = requests.get( configuration.manager_image_commit_sha_url).text.strip() local_sha1 = work.last_pulled_image_commit_sha1 if online_sha1 == local_sha1: logger.info( 'Current image is the latest image. It is based on the ' 'following commit in the manager blueprints repo: {}'.format( local_sha1)) return logger.info('Download manager image from {} to {}'.format( configuration.manager_image_url, work.pulled_image_path)) if os.path.exists(work.pulled_image_path): os.remove(work.pulled_image_path) files.download(url=configuration.manager_image_url, output_path=work.pulled_image_path, no_progress=no_progress) logger.info('Loading image into docker (may take a while)') quiet_docker.load(gzip('-dc', work.pulled_image_path, _piped=True, _out_bufsize=constants.BUFFER_SIZE), _in_bufsize=constants.BUFFER_SIZE) work.last_pulled_image_commit_sha1 = online_sha1
def save_image(container_id=None, tag=None, output_file=None, skip_agent_prepare=False): container_id = container_id or work.last_container_id docker_tag = tag or configuration.manager_image_docker_tag logger.info('Preparing manager container before saving as docker image') _run_container_preparation_scripts(container_id, skip_agent_prepare) logger.info('Saving manager container to image {}'.format(docker_tag)) quiet_docker.stop(container_id) quiet_docker.commit(container_id, docker_tag) logger.info("Removing container. Run 'docl run' to start it again") quiet_docker.rm('-f', container_id) if output_file: logger.info('Saving manager image to {}. This may take a while'.format( output_file)) gzip(quiet_docker.save(docker_tag, _piped=True, _tty_out=False, _out_bufsize=constants.BUFFER_SIZE), _in_bufsize=constants.BUFFER_SIZE, _out=output_file)
def save_image(container_id=None, tag=None, output_file=None, skip_agent_prepare=False): container_id = container_id or work.last_container_id docker_tag = tag or configuration.manager_image_docker_tag logger.info('Preparing manager container before saving as docker image') cp(source=resources.DIR / 'update_running_system.py', target=':{}'.format(constants.PY_SCRIPT_TARGET_PATH), container_id=container_id) cp(source=resources.DIR / 'prepare_save_image.py', target=':{}'.format(constants.PREPARE_SAVE_IMAGE_TARGET_PATH), container_id=container_id) params = { 'data_json_path': constants.DATA_JSON_TARGET_PATH, 'pydevd_egg_url': configuration.pydevd_egg_url, 'skip_agent_prepare': skip_agent_prepare, 'agent_template_dir': constants.AGENT_TEMPLATE_DIR, 'agent_package_path': configuration.agent_package_path } docker('exec', container_id, 'python', constants.PREPARE_SAVE_IMAGE_TARGET_PATH, base64.b64encode(json.dumps(params))) logger.info('Saving manager container to image {}'.format(docker_tag)) quiet_docker.stop(container_id) quiet_docker.commit(container_id, docker_tag) logger.info("Removing container. Run 'docl run' to start it again") quiet_docker.rm('-f', container_id) if output_file: logger.info('Saving manager image to {}. This may take a while' .format(output_file)) gzip(quiet_docker.save(docker_tag, _piped=True, _tty_out=False, _out_bufsize=constants.BUFFER_SIZE), _in_bufsize=constants.BUFFER_SIZE, _out=output_file)
def pull_image(no_progress=False): online_sha1 = requests.get( configuration.manager_image_commit_sha_url).text.strip() local_sha1 = work.last_pulled_image_commit_sha1 if online_sha1 == local_sha1: logger.info('Current image is the latest image. It is based on the ' 'following commit in the manager blueprints repo: {}' .format(local_sha1)) return logger.info('Download manager image from {} to {}' .format(configuration.manager_image_url, work.pulled_image_path)) if os.path.exists(work.pulled_image_path): os.remove(work.pulled_image_path) files.download(url=configuration.manager_image_url, output_path=work.pulled_image_path, no_progress=no_progress) logger.info('Loading image into docker') docker.load(gzip('-dc', work.pulled_image_path, _piped=True, _out_bufsize=constants.BUFFER_SIZE), _in_bufsize=constants.BUFFER_SIZE) work.last_pulled_image_commit_sha1 = online_sha1
def remove_image(tag=None): logger.info('Removing image {}'.format(tag)) quiet_docker.rmi(tag)
def _restart_service(container_id, service): logger.info('Restarting {}'.format(service)) quiet_docker('exec', container_id, 'systemctl', 'restart', service)