Exemplo n.º 1
0
def clean_dependencies_cache(verbose=True):
    images_by_projects = collections.defaultdict(list)
    limit = flask.current_app.config['KOZMIC_CACHED_IMAGES_LIMIT']

    for image_data in docker.images():
        created_at = image_data['Created']

        for repo_tag in image_data['RepoTags']:
            if not repo_tag.startswith('kozmic-cache/'):
                continue

            try:
                repository, tag = repo_tag.split(':')
            except ValueError:
                continue

            try:
                project_id = int(tag)
            except ValueError:
                continue

            images_by_projects[project_id].append((created_at, image_data['Id']))
            break

    for project_id, timestamped_images in images_by_projects.iteritems():
        images_to_remove = [image for _, image in
                            sorted(timestamped_images)[:-limit]]
        for image in images_to_remove:
            docker.remove_image(image)
            logger.info('Removed %s', image)
Exemplo n.º 2
0
def clean_dependencies_cache(verbose=True):
    images_by_projects = collections.defaultdict(list)
    limit = flask.current_app.config['KOZMIC_CACHED_IMAGES_LIMIT']

    for image_data in docker.images():
        created_at = image_data['Created']

        for repo_tag in image_data['RepoTags']:
            if not repo_tag.startswith('kozmic-cache/'):
                continue

            try:
                repository, tag = repo_tag.split(':')
            except ValueError:
                continue

            try:
                project_id = int(tag)
            except ValueError:
                continue

            images_by_projects[project_id].append(
                (created_at, image_data['Id']))
            break

    for project_id, timestamped_images in images_by_projects.iteritems():
        images_to_remove = [
            image for _, image in sorted(timestamped_images)[:-limit]
        ]
        for image in images_to_remove:
            docker.remove_image(image)
            logger.info('Removed %s', image)
Exemplo n.º 3
0
    def test_private_project(self):
        cache_id = 'qwerty'
        cached_image = 'kozmic-cache/{}'.format(cache_id)

        try:
            for image_data in docker.images(cached_image):
                for repo_tag in image_data['RepoTags']:
                    if repo_tag.startswith(cached_image):
                        docker.remove_image(image_data['Id'])
                        break
        except:
            pass
        assert not docker.images(cached_image)

        build = factories.BuildFactory.create(
            project=self.project,
            gh_commit_sha=self.prev_head_sha)
        hook_call = factories.HookCallFactory.create(
            hook=self.hook,
            build=build)

        job = self._do_job(hook_call)
        assert job.return_code == 0
        assert job.stdout == (
            'Pulling "{}" Docker image...\n'
             'installed!\nit works\nYEAH\n'.format(self.hook.docker_image))
        assert docker.images(cached_image)

        build = factories.BuildFactory.create(
            project=self.project,
            gh_commit_sha=self.head_sha)
        hook_call = factories.HookCallFactory.create(
            hook=self.hook,
            build=build)

        job = self._do_job(hook_call)
        assert job.return_code == 0
        assert job.stdout == (
            'Pulling "{}" Docker image...\n'
            'Skipping install script as tracked files did not change...\n'
            'it works\n'
            'YEAH\n'.format(self.hook.docker_image))
Exemplo n.º 4
0
    def test_private_project(self):
        cache_id = 'qwerty'
        cached_image = 'kozmic-cache/{}'.format(cache_id)

        try:
            for image_data in docker.images(cached_image):
                for repo_tag in image_data['RepoTags']:
                    if repo_tag.startswith(cached_image):
                        docker.remove_image(image_data['Id'])
                        break
        except:
            pass
        assert not docker.images(cached_image)

        build = factories.BuildFactory.create(
            project=self.project,
            gh_commit_sha=self.prev_head_sha)
        hook_call = factories.HookCallFactory.create(
            hook=self.hook,
            build=build)

        job = self._do_job(hook_call)
        assert job.return_code == 0
        assert job.stdout == (
            'Pulling "{}" Docker image...\n'
             'installed!\nit works\nYEAH\n'.format(self.hook.docker_image))
        assert docker.images(cached_image)

        build = factories.BuildFactory.create(
            project=self.project,
            gh_commit_sha=self.head_sha)
        hook_call = factories.HookCallFactory.create(
            hook=self.hook,
            build=build)

        job = self._do_job(hook_call)
        assert job.return_code == 0
        assert job.stdout == (
            'Pulling "{}" Docker image...\n'
            'Skipping install script as tracked files did not change...\n'
            'it works\n'
            'YEAH\n'.format(self.hook.docker_image))
Exemplo n.º 5
0
def do_job(hook_call_id):
    """A Celery task that does a job specified by a hook call.

    Creates a :class:`Job` instance and executes a build script prescribed
    by a triggered :class:`Hook`. Also sends job output to :attr:`Job.task_uuid`
    Redis pub-sub channel and updates build status.

    :param hook_call_id: int, :class:`HookCall` identifier
    """
    hook_call = HookCall.query.get(hook_call_id)
    assert hook_call, 'HookCall#{} does not exist.'.format(hook_call_id)

    job = Job(
        build=hook_call.build,
        hook_call=hook_call,
        task_uuid=do_job.request.id)
    db.session.add(job)
    job.started()
    db.session.commit()

    logger.info('start sleeping for some time...')
    while(True):
      sleep(5)
      logger.info('zzzzzz...')
    logger.info('done sleeping')

    hook = hook_call.hook
    project = hook.project
    config = current_app.config

    redis_client = redis.StrictRedis(host=config['KOZMIC_REDIS_HOST'],
                                     port=config['KOZMIC_REDIS_PORT'],
                                     db=config['KOZMIC_REDIS_DATABASE'])
    publisher = Publisher(redis_client=redis_client, channel=job.task_uuid)

    stdout = ''
    try:
        kwargs = dict(
            publisher=publisher,
            stall_timeout=config['KOZMIC_STALL_TIMEOUT'],
            clone_url=(project.gh_https_clone_url if project.is_public else
                       project.gh_ssh_clone_url),
            commit_sha=hook_call.build.gh_commit_sha)

        message = 'Pulling "{}" Docker image...'.format(hook.docker_image)
        logger.info(message)
        publisher.publish(message)
        stdout = message + '\n'

        try:
            logger.info(docker.base_url)
            docker.pull(hook.docker_image)
            # Make sure that image has been successfully pulled by calling
            # `inspect_image` on it:
            docker.inspect_image(hook.docker_image)
        except DockerAPIError as e:
            logger.info('Failed to pull %s: %s.', hook.docker_image, e)
            job.finished(1)
            job.stdout = str(e)
            db.session.commit()
            return
        else:
            message = '"{}" image has been pulled.'.format(hook.docker_image)
            logger.info(message)
            publisher.publish(message)

        #if not project.is_public:
        #    project.deploy_key.ensure()

        #if not project.is_public:
        #    kwargs['deploy_key'] = (
        #        project.deploy_key.rsa_private_key,
        #        project.passphrase)

        if job.hook_call.hook.install_script:
            cached_image = 'kozmic-cache/{}'.format(job.get_cache_id())
            cached_image_tag = str(project.id)
            if does_docker_image_exist(cached_image, cached_image_tag):
                install_stdout = ('Skipping install script as tracked files '
                                  'did not change...')
                publisher.publish(install_stdout)
                stdout += install_stdout + '\n'
            else:
                with _run(docker_image=hook.docker_image,
                          script=hook.install_script,
                          remove_container=False,
                          **kwargs) as (return_code, install_stdout, container):
                    stdout += install_stdout
                    if return_code == 0:
                        # Install script has finished successfully. So we
                        # promote the resulting container to an image that
                        # we will use for running the build script in
                        # this and consequent jobs
                        docker.commit(container['Id'], repository=cached_image,
                                      tag=cached_image_tag)
                        docker.remove_container(container)
                    else:
                        job.finished(return_code)
                        job.stdout = stdout
                        db.session.commit()
                        return
                assert docker.images(cached_image)
            docker_image = cached_image + ':' + cached_image_tag
        else:
            docker_image = job.hook_call.hook.docker_image

        with _run(docker_image=docker_image,
                  script=hook.build_script,
                  remove_container=True,
                  **kwargs) as (return_code, docker_stdout, container):
            job.finished(return_code)
            if len(docker_stdout) > 1000000:
                build_stdout = docker_stdout[:200000] + docker_stdout[-800000:]
            else:
                build_stdout = docker_stdout

            job.stdout = stdout + build_stdout
            db.session.commit()
            return
    finally:
        publisher.finish()