Пример #1
0
    def post(self, build_id):
        args = self.parser.parse_args()

        build = Build.query.options(
            joinedload('project', innerjoin=True),
            joinedload('author'),
            joinedload('source').joinedload('revision'),
        ).get(build_id)
        if build is None:
            return '', 404

        selective_testing_policy = SelectiveTestingPolicy.disabled
        build_message = None
        if args.selective_testing and project_lib.contains_active_autogenerated_plan(
                build.project):
            if not build.source.patch:
                selective_testing_policy, reasons = get_selective_testing_policy(
                    build.project, build.source.revision_sha, None)
                if reasons:
                    if selective_testing_policy is SelectiveTestingPolicy.disabled:
                        reasons = [
                            "Selective testing was requested but not done because:"
                        ] + ['    ' + m for m in reasons]
                    build_message = '\n'.join(reasons)
            else:
                # NOTE: for diff builds, it makes sense to just do selective testing,
                # since it will never become a parent build and will never be used to
                # calculate revision results.
                selective_testing_policy = SelectiveTestingPolicy.enabled

        collection_id = uuid.uuid4()
        new_build = create_build(
            project=build.project,
            collection_id=collection_id,
            label=build.label,
            target=build.target,
            message=build.message,
            author=build.author,
            source=build.source,
            cause=Cause.retry,
            selective_testing_policy=selective_testing_policy,
        )

        if build_message:
            message = BuildMessage(
                build=new_build,
                text=build_message,
            )
            db.session.add(message)
            db.session.commit()

        return '', 302, {
            'Location': '/api/0/builds/{0}/'.format(new_build.id.hex)
        }
Пример #2
0
    def post(self, build_id):
        args = self.parser.parse_args()

        build = Build.query.options(
            joinedload('project', innerjoin=True),
            joinedload('author'),
            joinedload('source').joinedload('revision'),
        ).get(build_id)
        if build is None:
            return '', 404

        selective_testing_policy = SelectiveTestingPolicy.disabled
        build_message = None
        if args.selective_testing and project_lib.contains_active_autogenerated_plan(build.project):
            if not build.source.patch:
                selective_testing_policy, reasons = get_selective_testing_policy(build.project, build.source.revision_sha, None)
                if reasons:
                    if selective_testing_policy is SelectiveTestingPolicy.disabled:
                        reasons = ["Selective testing was requested but not done because:"] + ['    ' + m for m in reasons]
                    build_message = '\n'.join(reasons)
            else:
                # NOTE: for diff builds, it makes sense to just do selective testing,
                # since it will never become a parent build and will never be used to
                # calculate revision results.
                selective_testing_policy = SelectiveTestingPolicy.enabled

        collection_id = uuid.uuid4()
        new_build = create_build(
            project=build.project,
            collection_id=collection_id,
            label=build.label,
            target=build.target,
            message=build.message,
            author=build.author,
            source=build.source,
            cause=Cause.retry,
            selective_testing_policy=selective_testing_policy,
        )

        if build_message:
            message = BuildMessage(
                build=new_build,
                text=build_message,
            )
            db.session.add(message)
            db.session.commit()

        return '', 302, {'Location': '/api/0/builds/{0}/'.format(new_build.id.hex)}
Пример #3
0
    def post(self, diff_id):
        """
        Ask Changes to restart all builds for this diff. The response will be
        the list of all builds.
        """
        diff = self._get_diff_by_id(diff_id)
        if not diff:
            return error("Diff with ID %s does not exist." % (diff_id, ))
        diff_parser = DiffParser(diff.source.patch.diff)
        files_changed = diff_parser.get_changed_files()
        try:
            projects = self._get_projects_for_diff(diff, files_changed)
        except InvalidDiffError:
            return error('Patch does not apply')
        except ProjectConfigError:
            return error('Project config is not in a valid format.')
        collection_id = uuid.uuid4()

        builds = self._get_builds_for_diff(diff)
        new_builds = []
        for project in projects:
            builds_for_project = [
                x for x in builds if x.project_id == project.id
            ]
            if not builds_for_project:
                logging.warning('Project with id %s does not have a build.',
                                project.id)
                continue
            build = max(builds_for_project, key=lambda x: x.number)
            if build.status is not Status.finished:
                continue
            if build.result is Result.passed:
                continue
            new_build = create_build(
                project=project,
                collection_id=collection_id,
                label=build.label,
                target=build.target,
                message=build.message,
                author=build.author,
                source=diff.source,
                cause=Cause.retry,
                selective_testing_policy=build.selective_testing_policy,
            )
            new_builds.append(new_build)
        return self.respond(new_builds)
Пример #4
0
    def post(self, diff_id):
        """
        Ask Changes to restart all builds for this diff. The response will be
        the list of all builds.
        """
        diff = self._get_diff_by_id(diff_id)
        if not diff:
            return error("Diff with ID %s does not exist." % (diff_id,))
        diff_parser = DiffParser(diff.source.patch.diff)
        files_changed = diff_parser.get_changed_files()
        try:
            projects = self._get_projects_for_diff(diff, files_changed)
        except InvalidDiffError:
            return error('Patch does not apply')
        except ProjectConfigError:
            return error('Project config is not in a valid format.')
        collection_id = uuid.uuid4()

        builds = self._get_builds_for_diff(diff)
        new_builds = []
        for project in projects:
            builds_for_project = [x for x in builds if x.project_id == project.id]
            if not builds_for_project:
                logging.warning('Project with id %s does not have a build.', project.id)
                continue
            build = max(builds_for_project, key=lambda x: x.number)
            if build.status is not Status.finished:
                continue
            if build.result is Result.passed:
                continue
            new_build = create_build(
                project=project,
                collection_id=collection_id,
                label=build.label,
                target=build.target,
                message=build.message,
                author=build.author,
                source=diff.source,
                cause=Cause.retry,
                selective_testing_policy=build.selective_testing_policy,
            )
            new_builds.append(new_build)
        return self.respond(new_builds)
Пример #5
0
    def post(self, build_id):
        build = Build.query.options(
            joinedload('project', innerjoin=True),
            joinedload('author'),
            joinedload('source').joinedload('revision'),
        ).get(build_id)
        if build is None:
            return '', 404

        new_build = create_build(
            project=build.project,
            label=build.label,
            target=build.target,
            message=build.message,
            author=build.author,
            source=build.source,
            cause=Cause.retry,
        )

        return '', 302, {'Location': '/api/0/builds/{0}/'.format(new_build.id.hex)}
Пример #6
0
    def post(self, build_id):
        build = Build.query.options(
            joinedload('project', innerjoin=True),
            joinedload('author'),
            joinedload('source').joinedload('revision'),
        ).get(build_id)
        if build is None:
            return '', 404

        new_build = create_build(
            project=build.project,
            label=build.label,
            target=build.target,
            message=build.message,
            author=build.author,
            source=build.source,
            cause=Cause.retry,
        )

        return '', 302, {
            'Location': '/api/0/builds/{0}/'.format(new_build.id.hex)
        }
Пример #7
0
    def post(self):
        """
        Notify Changes of a newly created diff.

        Depending on system configuration, this may create 0 or more new builds,
        and the resulting response will be a list of those build objects.
        """
        args = self.parser.parse_args()

        repository = args.repository
        if not args.repository:
            return error("Repository not found")

        projects = list(
            Project.query.options(subqueryload_all('plans'), ).filter(
                Project.status == ProjectStatus.active,
                Project.repository_id == repository.id,
            ))

        # no projects bound to repository
        if not projects:
            return self.respond([])

        options = dict(
            db.session.query(
                ProjectOption.project_id, ProjectOption.value).filter(
                    ProjectOption.project_id.in_([p.id for p in projects]),
                    ProjectOption.name.in_([
                        'phabricator.diff-trigger',
                    ])))

        projects = [p for p in projects if options.get(p.id, '1') == '1']

        if not projects:
            return self.respond([])

        label = args.label[:128]
        author = args.author
        message = args.message
        sha = args.sha
        target = 'D{}'.format(args['phabricator.revisionID'])

        try:
            identify_revision(repository, sha)
        except MissingRevision:
            return error("Unable to find commit %s in %s." %
                         (sha, repository.url),
                         problems=['sha', 'repository'])

        source_data = {
            'phabricator.buildTargetPHID': args['phabricator.buildTargetPHID'],
            'phabricator.diffID': args['phabricator.diffID'],
            'phabricator.revisionID': args['phabricator.revisionID'],
            'phabricator.revisionURL': args['phabricator.revisionURL'],
        }

        patch = Patch(
            repository=repository,
            parent_revision_sha=sha,
            diff=''.join(args.patch_file),
        )
        db.session.add(patch)

        source = Source(
            patch=patch,
            repository=repository,
            revision_sha=sha,
            data=source_data,
        )
        db.session.add(source)

        phabricatordiff = try_create(
            PhabricatorDiff, {
                'diff_id': args['phabricator.diffID'],
                'revision_id': args['phabricator.revisionID'],
                'url': args['phabricator.revisionURL'],
                'source': source,
            })
        if phabricatordiff is None:
            logging.error("Diff %s, Revision %s already exists",
                          args['phabricator.diffID'],
                          args['phabricator.revisionID'])
            return error("Diff already exists within Changes")

        project_options = ProjectOptionsHelper.get_options(
            projects, ['build.file-whitelist'])
        diff_parser = DiffParser(patch.diff)
        files_changed = diff_parser.get_changed_files()

        collection_id = uuid.uuid4()
        builds = []
        for project in projects:
            plan_list = get_build_plans(project)
            if not plan_list:
                logging.warning('No plans defined for project %s',
                                project.slug)
                continue

            if not in_project_files_whitelist(project_options[project.id],
                                              files_changed):
                logging.info(
                    'No changed files matched build.file-whitelist for project %s',
                    project.slug)
                continue

            builds.append(
                create_build(
                    project=project,
                    collection_id=collection_id,
                    sha=sha,
                    target=target,
                    label=label,
                    message=message,
                    author=author,
                    patch=patch,
                ))

        return self.respond(builds)
Пример #8
0
    def post(self):
        """
        Notify Changes of a newly created diff.

        Depending on system configuration, this may create 0 or more new builds,
        and the resulting response will be a list of those build objects.
        """
        args = self.parser.parse_args()

        repository = args.repository
        if not args.repository:
            return error("Repository not found")

        projects = list(Project.query.options(
            subqueryload_all('plans'),
        ).filter(
            Project.status == ProjectStatus.active,
            Project.repository_id == repository.id,
        ))

        # no projects bound to repository
        if not projects:
            return self.respond([])

        options = dict(
            db.session.query(
                ProjectOption.project_id, ProjectOption.value
            ).filter(
                ProjectOption.project_id.in_([p.id for p in projects]),
                ProjectOption.name.in_([
                    'phabricator.diff-trigger',
                ])
            )
        )

        projects = [
            p for p in projects
            if options.get(p.id, '1') == '1'
        ]

        if not projects:
            return self.respond([])

        label = args.label[:128]
        author = args.author
        message = args.message
        sha = args.sha
        target = 'D{}'.format(args['phabricator.revisionID'])

        try:
            identify_revision(repository, sha)
        except MissingRevision:
            return error("Unable to find commit %s in %s." % (
                sha, repository.url), problems=['sha', 'repository'])

        source_data = {
            'phabricator.buildTargetPHID': args['phabricator.buildTargetPHID'],
            'phabricator.diffID': args['phabricator.diffID'],
            'phabricator.revisionID': args['phabricator.revisionID'],
            'phabricator.revisionURL': args['phabricator.revisionURL'],
        }

        patch = Patch(
            repository=repository,
            parent_revision_sha=sha,
            diff=''.join(args.patch_file),
        )
        db.session.add(patch)

        source = Source(
            patch=patch,
            repository=repository,
            revision_sha=sha,
            data=source_data,
        )
        db.session.add(source)

        phabricatordiff = try_create(PhabricatorDiff, {
            'diff_id': args['phabricator.diffID'],
            'revision_id': args['phabricator.revisionID'],
            'url': args['phabricator.revisionURL'],
            'source': source,
        })
        if phabricatordiff is None:
            logging.error("Diff %s, Revision %s already exists", args['phabricator.diffID'], args['phabricator.revisionID'])
            return error("Diff already exists within Changes")

        project_options = ProjectOptionsHelper.get_options(projects, ['build.file-whitelist'])
        diff_parser = DiffParser(patch.diff)
        files_changed = diff_parser.get_changed_files()

        collection_id = uuid.uuid4()
        builds = []
        for project in projects:
            plan_list = get_build_plans(project)
            if not plan_list:
                logging.warning('No plans defined for project %s', project.slug)
                continue

            if not in_project_files_whitelist(project_options[project.id], files_changed):
                logging.info('No changed files matched build.file-whitelist for project %s', project.slug)
                continue

            builds.append(create_build(
                project=project,
                collection_id=collection_id,
                sha=sha,
                target=target,
                label=label,
                message=message,
                author=author,
                patch=patch,
            ))

        return self.respond(builds)
Пример #9
0
    def post(self):
        """
        Notify Changes of a newly created diff.

        Depending on system configuration, this may create 0 or more new builds,
        and the resulting response will be a list of those build objects.
        """
        args = self.parser.parse_args()

        repository = args.repository
        if not args.repository:
            return error("Repository not found")

        projects = list(
            Project.query.options(subqueryload_all('plans'), ).filter(
                Project.status == ProjectStatus.active,
                Project.repository_id == repository.id,
            ))

        # no projects bound to repository
        if not projects:
            return self.respond([])

        options = dict(
            db.session.query(
                ProjectOption.project_id, ProjectOption.value).filter(
                    ProjectOption.project_id.in_([p.id for p in projects]),
                    ProjectOption.name.in_([
                        'phabricator.diff-trigger',
                    ])))

        projects = [p for p in projects if options.get(p.id, '1') == '1']

        if not projects:
            return self.respond([])

        statsreporter.stats().incr('diffs_posted_from_phabricator')

        label = args.label[:128]
        author = args.author
        message = args.message
        sha = args.sha
        target = 'D{}'.format(args['phabricator.revisionID'])

        try:
            identify_revision(repository, sha)
        except MissingRevision:
            # This may just be a broken request (which is why we respond with a 400) but
            # it also might indicate Phabricator and Changes being out of sync somehow,
            # so we err on the side of caution and log it as an error.
            logging.error(
                "Diff %s was posted for an unknown revision (%s, %s)", target,
                sha, repository.url)
            return error("Unable to find commit %s in %s." %
                         (sha, repository.url),
                         problems=['sha', 'repository'])

        source_data = {
            'phabricator.buildTargetPHID': args['phabricator.buildTargetPHID'],
            'phabricator.diffID': args['phabricator.diffID'],
            'phabricator.revisionID': args['phabricator.revisionID'],
            'phabricator.revisionURL': args['phabricator.revisionURL'],
        }

        patch = Patch(
            repository=repository,
            parent_revision_sha=sha,
            diff=''.join(args.patch_file),
        )
        db.session.add(patch)

        source = Source(
            patch=patch,
            repository=repository,
            revision_sha=sha,
            data=source_data,
        )
        db.session.add(source)

        phabricatordiff = try_create(
            PhabricatorDiff, {
                'diff_id': args['phabricator.diffID'],
                'revision_id': args['phabricator.revisionID'],
                'url': args['phabricator.revisionURL'],
                'source': source,
            })
        if phabricatordiff is None:
            logging.error("Diff %s, Revision %s already exists",
                          args['phabricator.diffID'],
                          args['phabricator.revisionID'])
            return error("Diff already exists within Changes")

        project_options = ProjectOptionsHelper.get_options(
            projects, ['build.file-whitelist'])
        diff_parser = DiffParser(patch.diff)
        files_changed = diff_parser.get_changed_files()

        collection_id = uuid.uuid4()
        builds = []
        for project in projects:
            plan_list = get_build_plans(project)
            if not plan_list:
                logging.warning('No plans defined for project %s',
                                project.slug)
                continue

            try:
                if not files_changed_should_trigger_project(
                        files_changed,
                        project,
                        project_options[project.id],
                        sha,
                        diff=patch.diff):
                    logging.info(
                        'No changed files matched project trigger for project %s',
                        project.slug)
                    continue
            except InvalidDiffError:
                # ok, the build will fail and the user will be notified
                pass
            except ProjectConfigError:
                logging.error(
                    'Project config for project %s is not in a valid format. Author is %s.',
                    project.slug,
                    author.name,
                    exc_info=True)

            builds.append(
                create_build(
                    project=project,
                    collection_id=collection_id,
                    sha=sha,
                    target=target,
                    label=label,
                    message=message,
                    author=author,
                    patch=patch,
                    tag="phabricator",
                ))
        # This is the counterpoint to the above 'diffs_posted_from_phabricator';
        # at this point we've successfully processed the diff, so comparing this
        # stat to the above should give us the phabricator diff failure rate.
        statsreporter.stats().incr(
            'diffs_successfully_processed_from_phabricator')

        return self.respond(builds)
Пример #10
0
    def post_impl(self):
        """
        Notify Changes of a newly created diff.

        Depending on system configuration, this may create 0 or more new builds,
        and the resulting response will be a list of those build objects.
        """
        args = self.parser.parse_args()

        repository = args.repository
        if not args.repository:
            return error("Repository not found")

        projects = list(Project.query.options(
            subqueryload_all('plans'),
        ).filter(
            Project.status == ProjectStatus.active,
            Project.repository_id == repository.id,
        ))

        # no projects bound to repository
        if not projects:
            return self.respond([])

        options = dict(
            db.session.query(
                ProjectOption.project_id, ProjectOption.value
            ).filter(
                ProjectOption.project_id.in_([p.id for p in projects]),
                ProjectOption.name.in_([
                    'phabricator.diff-trigger',
                ])
            )
        )

        projects = [
            p for p in projects
            if options.get(p.id, '1') == '1'
        ]

        if not projects:
            return self.respond([])

        statsreporter.stats().incr('diffs_posted_from_phabricator')

        label = args.label[:128]
        author = args.author
        message = args.message
        sha = args.sha
        target = 'D{}'.format(args['phabricator.revisionID'])

        try:
            identify_revision(repository, sha)
        except MissingRevision:
            # This may just be a broken request (which is why we respond with a 400) but
            # it also might indicate Phabricator and Changes being out of sync somehow,
            # so we err on the side of caution and log it as an error.
            logging.error("Diff %s was posted for an unknown revision (%s, %s)",
                          target, sha, repository.url)
            return error("Unable to find commit %s in %s." % (
                sha, repository.url), problems=['sha', 'repository'])

        source_data = {
            'phabricator.buildTargetPHID': args['phabricator.buildTargetPHID'],
            'phabricator.diffID': args['phabricator.diffID'],
            'phabricator.revisionID': args['phabricator.revisionID'],
            'phabricator.revisionURL': args['phabricator.revisionURL'],
        }

        patch = Patch(
            repository=repository,
            parent_revision_sha=sha,
            diff=''.join(line.decode('utf-8') for line in args.patch_file),
        )
        db.session.add(patch)

        source = Source(
            patch=patch,
            repository=repository,
            revision_sha=sha,
            data=source_data,
        )
        db.session.add(source)

        phabricatordiff = try_create(PhabricatorDiff, {
            'diff_id': args['phabricator.diffID'],
            'revision_id': args['phabricator.revisionID'],
            'url': args['phabricator.revisionURL'],
            'source': source,
        })
        if phabricatordiff is None:
            logging.warning("Diff %s, Revision %s already exists",
                            args['phabricator.diffID'], args['phabricator.revisionID'])
            return error("Diff already exists within Changes")

        project_options = ProjectOptionsHelper.get_options(projects, ['build.file-whitelist'])
        diff_parser = DiffParser(patch.diff)
        files_changed = diff_parser.get_changed_files()

        collection_id = uuid.uuid4()
        builds = []
        for project in projects:
            plan_list = get_build_plans(project)
            if not plan_list:
                logging.warning('No plans defined for project %s', project.slug)
                continue

            try:
                if not files_changed_should_trigger_project(files_changed, project, project_options[project.id], sha, diff=patch.diff):
                    logging.info('No changed files matched project trigger for project %s', project.slug)
                    continue
            except InvalidDiffError:
                # ok, the build will fail and the user will be notified
                pass
            except ProjectConfigError:
                logging.error('Project config for project %s is not in a valid format. Author is %s.', project.slug, author.name, exc_info=True)

            builds.append(create_build(
                project=project,
                collection_id=collection_id,
                sha=sha,
                target=target,
                label=label,
                message=message,
                author=author,
                patch=patch,
                tag="phabricator",
            ))

        # This is the counterpoint to the above 'diffs_posted_from_phabricator';
        # at this point we've successfully processed the diff, so comparing this
        # stat to the above should give us the phabricator diff failure rate.
        statsreporter.stats().incr('diffs_successfully_processed_from_phabricator')

        return self.respond(builds)
Пример #11
0
    def post_impl(self):
        """
        Notify Changes of a newly created diff.

        Depending on system configuration, this may create 0 or more new builds,
        and the resulting response will be a list of those build objects.
        """

        # we manually check for arg presence here so we can send a more specific
        # error message to the user (rather than a plain 400)
        args = self.parser.parse_args()
        if not args.repository:
            # No need to postback a comment for this
            statsreporter.stats().incr("diffs_repository_not_found")
            return error("Repository not found")

        repository = args.repository

        projects = list(
            Project.query.options(subqueryload_all('plans'), ).filter(
                Project.status == ProjectStatus.active,
                Project.repository_id == repository.id,
            ))

        # no projects bound to repository
        if not projects:
            return self.respond([])

        options = dict(
            db.session.query(
                ProjectOption.project_id, ProjectOption.value).filter(
                    ProjectOption.project_id.in_([p.id for p in projects]),
                    ProjectOption.name.in_([
                        'phabricator.diff-trigger',
                    ])))

        # Filter out projects that aren't configured to run builds off of diffs
        # - Diff trigger disabled
        # - No build plans
        projects = [
            p for p in projects
            if options.get(p.id, '1') == '1' and get_build_plans(p)
        ]

        if not projects:
            return self.respond([])

        statsreporter.stats().incr('diffs_posted_from_phabricator')

        label = args.label[:128]
        author = args.author
        message = args.message
        sha = args.sha
        target = 'D%s' % args['phabricator.revisionID']

        try:
            identify_revision(repository, sha)
        except MissingRevision:
            # This may just be a broken request (which is why we respond with a 400) but
            # it also might indicate Phabricator and Changes being out of sync somehow,
            # so we err on the side of caution and log it as an error.
            logging.error(
                "Diff %s was posted for an unknown revision (%s, %s)", target,
                sha, repository.url)
            # We should postback since this can happen if a user diffs dependent revisions
            statsreporter.stats().incr("diffs_missing_base_revision")
            return self.postback_error(
                "Unable to find base revision {revision} in {repo} on Changes. Some possible reasons:\n"
                " - You may be working on multiple stacked diffs in your local repository.\n"
                "   {revision} only exists in your local copy. Changes thus cannot apply your patch\n"
                " - If you are sure that's not the case, it's possible you applied your patch to an extremely\n"
                "   recent revision which Changes hasn't picked up yet. Retry in a minute\n"
                .format(
                    revision=sha,
                    repo=repository.url,
                ),
                target,
                problems=['sha', 'repository'])

        source_data = {
            'phabricator.buildTargetPHID': args['phabricator.buildTargetPHID'],
            'phabricator.diffID': args['phabricator.diffID'],
            'phabricator.revisionID': args['phabricator.revisionID'],
            'phabricator.revisionURL': args['phabricator.revisionURL'],
        }

        patch = Patch(
            repository=repository,
            parent_revision_sha=sha,
            diff=''.join(line.decode('utf-8') for line in args.patch_file),
        )
        db.session.add(patch)

        source = Source(
            patch=patch,
            repository=repository,
            revision_sha=sha,
            data=source_data,
        )
        db.session.add(source)

        phabricatordiff = try_create(
            PhabricatorDiff, {
                'diff_id': args['phabricator.diffID'],
                'revision_id': args['phabricator.revisionID'],
                'url': args['phabricator.revisionURL'],
                'source': source,
            })
        if phabricatordiff is None:
            logging.warning("Diff %s, Revision %s already exists",
                            args['phabricator.diffID'],
                            args['phabricator.revisionID'])
            # No need to inform user about this explicitly
            statsreporter.stats().incr("diffs_already_exists")
            return error("Diff already exists within Changes")

        project_options = ProjectOptionsHelper.get_options(
            projects, ['build.file-whitelist'])
        diff_parser = DiffParser(patch.diff)
        files_changed = diff_parser.get_changed_files()

        collection_id = uuid.uuid4()
        builds = []
        for project in projects:
            plan_list = get_build_plans(project)
            # We already filtered out empty build plans
            assert plan_list, ('No plans defined for project {}'.format(
                project.slug))

            try:
                if not files_changed_should_trigger_project(
                        files_changed,
                        project,
                        project_options[project.id],
                        sha,
                        diff=patch.diff):
                    logging.info(
                        'No changed files matched project trigger for project %s',
                        project.slug)
                    continue
            except InvalidDiffError:
                # ok, the build will fail and the user will be notified
                pass
            except ProjectConfigError:
                logging.error(
                    'Project config for project %s is not in a valid format. Author is %s.',
                    project.slug,
                    author.name,
                    exc_info=True)

            builds.append(
                create_build(
                    project=project,
                    collection_id=collection_id,
                    sha=sha,
                    target=target,
                    label=label,
                    message=message,
                    author=author,
                    patch=patch,
                    tag="phabricator",
                ))

        # This is the counterpoint to the above 'diffs_posted_from_phabricator';
        # at this point we've successfully processed the diff, so comparing this
        # stat to the above should give us the phabricator diff failure rate.
        statsreporter.stats().incr(
            'diffs_successfully_processed_from_phabricator')

        return self.respond(builds)
Пример #12
0
    def post_impl(self):
        """
        Notify Changes of a newly created diff.

        Depending on system configuration, this may create 0 or more new builds,
        and the resulting response will be a list of those build objects.
        """

        # we manually check for arg presence here so we can send a more specific
        # error message to the user (rather than a plain 400)
        args = self.parser.parse_args()
        if not args.repository:
            # No need to postback a comment for this
            statsreporter.stats().incr("diffs_repository_not_found")
            return error("Repository not found")

        repository = args.repository

        projects = list(Project.query.options(
            subqueryload_all('plans'),
        ).filter(
            Project.status == ProjectStatus.active,
            Project.repository_id == repository.id,
        ))

        # no projects bound to repository
        if not projects:
            return self.respond([])

        options = dict(
            db.session.query(
                ProjectOption.project_id, ProjectOption.value
            ).filter(
                ProjectOption.project_id.in_([p.id for p in projects]),
                ProjectOption.name.in_([
                    'phabricator.diff-trigger',
                ])
            )
        )

        # Filter out projects that aren't configured to run builds off of diffs
        # - Diff trigger disabled
        # - No build plans
        projects = [
            p for p in projects
            if options.get(p.id, '1') == '1' and get_build_plans(p)
        ]

        if not projects:
            return self.respond([])

        statsreporter.stats().incr('diffs_posted_from_phabricator')

        label = args.label[:128]
        author = args.author
        message = args.message
        sha = args.sha
        target = 'D%s' % args['phabricator.revisionID']

        try:
            identify_revision(repository, sha)
        except MissingRevision:
            # This may just be a broken request (which is why we respond with a 400) but
            # it also might indicate Phabricator and Changes being out of sync somehow,
            # so we err on the side of caution and log it as an error.
            logging.error("Diff %s was posted for an unknown revision (%s, %s)",
                          target, sha, repository.url)
            # We should postback since this can happen if a user diffs dependent revisions
            statsreporter.stats().incr("diff_missing_base_revision")
            return self.postback_error(
                "Unable to find base revision {revision} in {repo} on Changes. Some possible reasons:\n"
                " - Changes hasn't picked up {revision} yet. Retry in a minute\n"
                " - {revision} only exists in your local copy. Changes cannot apply your patch\n".format(
                    revision=sha,
                    repo=repository.url,
                ),
                target,
                problems=['sha', 'repository'])

        source_data = {
            'phabricator.buildTargetPHID': args['phabricator.buildTargetPHID'],
            'phabricator.diffID': args['phabricator.diffID'],
            'phabricator.revisionID': args['phabricator.revisionID'],
            'phabricator.revisionURL': args['phabricator.revisionURL'],
        }

        patch = Patch(
            repository=repository,
            parent_revision_sha=sha,
            diff=''.join(line.decode('utf-8') for line in args.patch_file),
        )
        db.session.add(patch)

        source = Source(
            patch=patch,
            repository=repository,
            revision_sha=sha,
            data=source_data,
        )
        db.session.add(source)

        phabricatordiff = try_create(PhabricatorDiff, {
            'diff_id': args['phabricator.diffID'],
            'revision_id': args['phabricator.revisionID'],
            'url': args['phabricator.revisionURL'],
            'source': source,
        })
        if phabricatordiff is None:
            logging.warning("Diff %s, Revision %s already exists",
                            args['phabricator.diffID'], args['phabricator.revisionID'])
            # No need to inform user about this explicitly
            statsreporter.stats().incr("diffs_already_exists")
            return error("Diff already exists within Changes")

        project_options = ProjectOptionsHelper.get_options(projects, ['build.file-whitelist'])
        diff_parser = DiffParser(patch.diff)
        files_changed = diff_parser.get_changed_files()

        collection_id = uuid.uuid4()
        builds = []
        for project in projects:
            plan_list = get_build_plans(project)
            # We already filtered out empty build plans
            assert plan_list, ('No plans defined for project {}'.format(project.slug))

            try:
                if not files_changed_should_trigger_project(files_changed, project, project_options[project.id], sha, diff=patch.diff):
                    logging.info('No changed files matched project trigger for project %s', project.slug)
                    continue
            except InvalidDiffError:
                # ok, the build will fail and the user will be notified
                pass
            except ProjectConfigError:
                logging.error('Project config for project %s is not in a valid format. Author is %s.', project.slug, author.name, exc_info=True)

            builds.append(create_build(
                project=project,
                collection_id=collection_id,
                sha=sha,
                target=target,
                label=label,
                message=message,
                author=author,
                patch=patch,
                tag="phabricator",
            ))

        # This is the counterpoint to the above 'diffs_posted_from_phabricator';
        # at this point we've successfully processed the diff, so comparing this
        # stat to the above should give us the phabricator diff failure rate.
        statsreporter.stats().incr('diffs_successfully_processed_from_phabricator')

        return self.respond(builds)
    def post(self):
        """
        Notify Changes of a newly created diff.

        Depending on system configuration, this may create 0 or more new builds,
        and the resulting response will be a list of those build objects.
        """
        args = self.parser.parse_args()

        repository = args.repository
        if not args.repository:
            return error("Repository not found")

        projects = list(Project.query.options(
            subqueryload_all('plans'),
        ).filter(
            Project.status == ProjectStatus.active,
            Project.repository_id == repository.id,
        ))

        # no projects bound to repository
        if not projects:
            return self.respond([])

        options = dict(
            db.session.query(
                ProjectOption.project_id, ProjectOption.value
            ).filter(
                ProjectOption.project_id.in_([p.id for p in projects]),
                ProjectOption.name.in_([
                    'phabricator.diff-trigger',
                ])
            )
        )

        projects = [
            p for p in projects
            if options.get(p.id, '1') == '1'
        ]

        if not projects:
            return self.respond([])

        label = args.label[:128]
        author = args.author
        message = args.message
        sha = args.sha
        target = 'D{}'.format(args['phabricator.revisionID'])

        try:
            identify_revision(repository, sha)
        except MissingRevision:
            return error("Unable to find commit %s in %s." % (
                sha, repository.url), problems=['sha', 'repository'])

        patch = Patch(
            repository=repository,
            parent_revision_sha=sha,
            diff=''.join(args.patch_file),
        )
        db.session.add(patch)

        patch_data = {
            'phabricator.buildTargetPHID': args['phabricator.buildTargetPHID'],
            'phabricator.diffID': args['phabricator.diffID'],
            'phabricator.revisionID': args['phabricator.revisionID'],
            'phabricator.revisionURL': args['phabricator.revisionURL'],
        }

        builds = []
        for project in projects:
            plan_list = get_build_plans(project)
            if not plan_list:
                logging.warning('No plans defined for project %s', project.slug)
                continue

            builds.append(create_build(
                project=project,
                sha=sha,
                target=target,
                label=label,
                message=message,
                author=author,
                patch=patch,
                source_data=patch_data,
            ))

        return self.respond(builds)