def _get_projects_for_diff(self, diff, files_changed): projects = list(Project.query.options( subqueryload_all('plans'), ).filter( Project.status == ProjectStatus.active, Project.repository_id == diff.source.repository_id, )) project_options = ProjectOptionsHelper.get_options(projects, ['build.file-whitelist', 'phabricator.diff-trigger']) projects = [ x for x in projects if get_build_plans(x) and project_options[x.id].get('phabricator.diff-trigger', '1') == '1' and files_changed_should_trigger_project(files_changed, x, project_options[x.id], diff.source.revision_sha, diff=diff.source.patch.diff) ] return projects
def _get_projects_for_diff(self, diff, files_changed): projects = list( Project.query.options(subqueryload_all('plans'), ).filter( Project.status == ProjectStatus.active, Project.repository_id == diff.source.repository_id, )) project_options = ProjectOptionsHelper.get_options( projects, ['build.file-whitelist', 'phabricator.diff-trigger']) projects = [ x for x in projects if get_build_plans(x) and project_options[x.id].get( 'phabricator.diff-trigger', '1') == '1' and files_changed_should_trigger_project(files_changed, x, project_options[x.id], diff.source.revision_sha, diff=diff.source.patch.diff) ] return projects
def post(self): """ Notify Changes of a newly created diff. Depending on system configuration, this may create 0 or more new builds, and the resulting response will be a list of those build objects. """ args = self.parser.parse_args() repository = args.repository if not args.repository: return error("Repository not found") projects = list( Project.query.options(subqueryload_all('plans'), ).filter( Project.status == ProjectStatus.active, Project.repository_id == repository.id, )) # no projects bound to repository if not projects: return self.respond([]) options = dict( db.session.query( ProjectOption.project_id, ProjectOption.value).filter( ProjectOption.project_id.in_([p.id for p in projects]), ProjectOption.name.in_([ 'phabricator.diff-trigger', ]))) projects = [p for p in projects if options.get(p.id, '1') == '1'] if not projects: return self.respond([]) label = args.label[:128] author = args.author message = args.message sha = args.sha target = 'D{}'.format(args['phabricator.revisionID']) try: identify_revision(repository, sha) except MissingRevision: return error("Unable to find commit %s in %s." % (sha, repository.url), problems=['sha', 'repository']) source_data = { 'phabricator.buildTargetPHID': args['phabricator.buildTargetPHID'], 'phabricator.diffID': args['phabricator.diffID'], 'phabricator.revisionID': args['phabricator.revisionID'], 'phabricator.revisionURL': args['phabricator.revisionURL'], } patch = Patch( repository=repository, parent_revision_sha=sha, diff=''.join(args.patch_file), ) db.session.add(patch) source = Source( patch=patch, repository=repository, revision_sha=sha, data=source_data, ) db.session.add(source) phabricatordiff = try_create( PhabricatorDiff, { 'diff_id': args['phabricator.diffID'], 'revision_id': args['phabricator.revisionID'], 'url': args['phabricator.revisionURL'], 'source': source, }) if phabricatordiff is None: logging.error("Diff %s, Revision %s already exists", args['phabricator.diffID'], args['phabricator.revisionID']) return error("Diff already exists within Changes") project_options = ProjectOptionsHelper.get_options( projects, ['build.file-whitelist']) diff_parser = DiffParser(patch.diff) files_changed = diff_parser.get_changed_files() collection_id = uuid.uuid4() builds = [] for project in projects: plan_list = get_build_plans(project) if not plan_list: logging.warning('No plans defined for project %s', project.slug) continue if not in_project_files_whitelist(project_options[project.id], files_changed): logging.info( 'No changed files matched build.file-whitelist for project %s', project.slug) continue builds.append( create_build( project=project, collection_id=collection_id, sha=sha, target=target, label=label, message=message, author=author, patch=patch, )) return self.respond(builds)
def post(self): """ Notify Changes of a newly created diff. Depending on system configuration, this may create 0 or more new builds, and the resulting response will be a list of those build objects. """ args = self.parser.parse_args() repository = args.repository if not args.repository: return error("Repository not found") projects = list(Project.query.options( subqueryload_all('plans'), ).filter( Project.status == ProjectStatus.active, Project.repository_id == repository.id, )) # no projects bound to repository if not projects: return self.respond([]) options = dict( db.session.query( ProjectOption.project_id, ProjectOption.value ).filter( ProjectOption.project_id.in_([p.id for p in projects]), ProjectOption.name.in_([ 'phabricator.diff-trigger', ]) ) ) projects = [ p for p in projects if options.get(p.id, '1') == '1' ] if not projects: return self.respond([]) label = args.label[:128] author = args.author message = args.message sha = args.sha target = 'D{}'.format(args['phabricator.revisionID']) try: identify_revision(repository, sha) except MissingRevision: return error("Unable to find commit %s in %s." % ( sha, repository.url), problems=['sha', 'repository']) source_data = { 'phabricator.buildTargetPHID': args['phabricator.buildTargetPHID'], 'phabricator.diffID': args['phabricator.diffID'], 'phabricator.revisionID': args['phabricator.revisionID'], 'phabricator.revisionURL': args['phabricator.revisionURL'], } patch = Patch( repository=repository, parent_revision_sha=sha, diff=''.join(args.patch_file), ) db.session.add(patch) source = Source( patch=patch, repository=repository, revision_sha=sha, data=source_data, ) db.session.add(source) phabricatordiff = try_create(PhabricatorDiff, { 'diff_id': args['phabricator.diffID'], 'revision_id': args['phabricator.revisionID'], 'url': args['phabricator.revisionURL'], 'source': source, }) if phabricatordiff is None: logging.error("Diff %s, Revision %s already exists", args['phabricator.diffID'], args['phabricator.revisionID']) return error("Diff already exists within Changes") project_options = ProjectOptionsHelper.get_options(projects, ['build.file-whitelist']) diff_parser = DiffParser(patch.diff) files_changed = diff_parser.get_changed_files() collection_id = uuid.uuid4() builds = [] for project in projects: plan_list = get_build_plans(project) if not plan_list: logging.warning('No plans defined for project %s', project.slug) continue if not in_project_files_whitelist(project_options[project.id], files_changed): logging.info('No changed files matched build.file-whitelist for project %s', project.slug) continue builds.append(create_build( project=project, collection_id=collection_id, sha=sha, target=target, label=label, message=message, author=author, patch=patch, )) return self.respond(builds)
def post(self): """ Notify Changes of a newly created diff. Depending on system configuration, this may create 0 or more new builds, and the resulting response will be a list of those build objects. """ args = self.parser.parse_args() repository = args.repository if not args.repository: return error("Repository not found") projects = list( Project.query.options(subqueryload_all('plans'), ).filter( Project.status == ProjectStatus.active, Project.repository_id == repository.id, )) # no projects bound to repository if not projects: return self.respond([]) options = dict( db.session.query( ProjectOption.project_id, ProjectOption.value).filter( ProjectOption.project_id.in_([p.id for p in projects]), ProjectOption.name.in_([ 'phabricator.diff-trigger', ]))) projects = [p for p in projects if options.get(p.id, '1') == '1'] if not projects: return self.respond([]) statsreporter.stats().incr('diffs_posted_from_phabricator') label = args.label[:128] author = args.author message = args.message sha = args.sha target = 'D{}'.format(args['phabricator.revisionID']) try: identify_revision(repository, sha) except MissingRevision: # This may just be a broken request (which is why we respond with a 400) but # it also might indicate Phabricator and Changes being out of sync somehow, # so we err on the side of caution and log it as an error. logging.error( "Diff %s was posted for an unknown revision (%s, %s)", target, sha, repository.url) return error("Unable to find commit %s in %s." % (sha, repository.url), problems=['sha', 'repository']) source_data = { 'phabricator.buildTargetPHID': args['phabricator.buildTargetPHID'], 'phabricator.diffID': args['phabricator.diffID'], 'phabricator.revisionID': args['phabricator.revisionID'], 'phabricator.revisionURL': args['phabricator.revisionURL'], } patch = Patch( repository=repository, parent_revision_sha=sha, diff=''.join(args.patch_file), ) db.session.add(patch) source = Source( patch=patch, repository=repository, revision_sha=sha, data=source_data, ) db.session.add(source) phabricatordiff = try_create( PhabricatorDiff, { 'diff_id': args['phabricator.diffID'], 'revision_id': args['phabricator.revisionID'], 'url': args['phabricator.revisionURL'], 'source': source, }) if phabricatordiff is None: logging.error("Diff %s, Revision %s already exists", args['phabricator.diffID'], args['phabricator.revisionID']) return error("Diff already exists within Changes") project_options = ProjectOptionsHelper.get_options( projects, ['build.file-whitelist']) diff_parser = DiffParser(patch.diff) files_changed = diff_parser.get_changed_files() collection_id = uuid.uuid4() builds = [] for project in projects: plan_list = get_build_plans(project) if not plan_list: logging.warning('No plans defined for project %s', project.slug) continue try: if not files_changed_should_trigger_project( files_changed, project, project_options[project.id], sha, diff=patch.diff): logging.info( 'No changed files matched project trigger for project %s', project.slug) continue except InvalidDiffError: # ok, the build will fail and the user will be notified pass except ProjectConfigError: logging.error( 'Project config for project %s is not in a valid format. Author is %s.', project.slug, author.name, exc_info=True) builds.append( create_build( project=project, collection_id=collection_id, sha=sha, target=target, label=label, message=message, author=author, patch=patch, tag="phabricator", )) # This is the counterpoint to the above 'diffs_posted_from_phabricator'; # at this point we've successfully processed the diff, so comparing this # stat to the above should give us the phabricator diff failure rate. statsreporter.stats().incr( 'diffs_successfully_processed_from_phabricator') return self.respond(builds)
def post_impl(self): """ Notify Changes of a newly created diff. Depending on system configuration, this may create 0 or more new builds, and the resulting response will be a list of those build objects. """ args = self.parser.parse_args() repository = args.repository if not args.repository: return error("Repository not found") projects = list(Project.query.options( subqueryload_all('plans'), ).filter( Project.status == ProjectStatus.active, Project.repository_id == repository.id, )) # no projects bound to repository if not projects: return self.respond([]) options = dict( db.session.query( ProjectOption.project_id, ProjectOption.value ).filter( ProjectOption.project_id.in_([p.id for p in projects]), ProjectOption.name.in_([ 'phabricator.diff-trigger', ]) ) ) projects = [ p for p in projects if options.get(p.id, '1') == '1' ] if not projects: return self.respond([]) statsreporter.stats().incr('diffs_posted_from_phabricator') label = args.label[:128] author = args.author message = args.message sha = args.sha target = 'D{}'.format(args['phabricator.revisionID']) try: identify_revision(repository, sha) except MissingRevision: # This may just be a broken request (which is why we respond with a 400) but # it also might indicate Phabricator and Changes being out of sync somehow, # so we err on the side of caution and log it as an error. logging.error("Diff %s was posted for an unknown revision (%s, %s)", target, sha, repository.url) return error("Unable to find commit %s in %s." % ( sha, repository.url), problems=['sha', 'repository']) source_data = { 'phabricator.buildTargetPHID': args['phabricator.buildTargetPHID'], 'phabricator.diffID': args['phabricator.diffID'], 'phabricator.revisionID': args['phabricator.revisionID'], 'phabricator.revisionURL': args['phabricator.revisionURL'], } patch = Patch( repository=repository, parent_revision_sha=sha, diff=''.join(line.decode('utf-8') for line in args.patch_file), ) db.session.add(patch) source = Source( patch=patch, repository=repository, revision_sha=sha, data=source_data, ) db.session.add(source) phabricatordiff = try_create(PhabricatorDiff, { 'diff_id': args['phabricator.diffID'], 'revision_id': args['phabricator.revisionID'], 'url': args['phabricator.revisionURL'], 'source': source, }) if phabricatordiff is None: logging.warning("Diff %s, Revision %s already exists", args['phabricator.diffID'], args['phabricator.revisionID']) return error("Diff already exists within Changes") project_options = ProjectOptionsHelper.get_options(projects, ['build.file-whitelist']) diff_parser = DiffParser(patch.diff) files_changed = diff_parser.get_changed_files() collection_id = uuid.uuid4() builds = [] for project in projects: plan_list = get_build_plans(project) if not plan_list: logging.warning('No plans defined for project %s', project.slug) continue try: if not files_changed_should_trigger_project(files_changed, project, project_options[project.id], sha, diff=patch.diff): logging.info('No changed files matched project trigger for project %s', project.slug) continue except InvalidDiffError: # ok, the build will fail and the user will be notified pass except ProjectConfigError: logging.error('Project config for project %s is not in a valid format. Author is %s.', project.slug, author.name, exc_info=True) builds.append(create_build( project=project, collection_id=collection_id, sha=sha, target=target, label=label, message=message, author=author, patch=patch, tag="phabricator", )) # This is the counterpoint to the above 'diffs_posted_from_phabricator'; # at this point we've successfully processed the diff, so comparing this # stat to the above should give us the phabricator diff failure rate. statsreporter.stats().incr('diffs_successfully_processed_from_phabricator') return self.respond(builds)
def post_impl(self): """ Notify Changes of a newly created diff. Depending on system configuration, this may create 0 or more new builds, and the resulting response will be a list of those build objects. """ # we manually check for arg presence here so we can send a more specific # error message to the user (rather than a plain 400) args = self.parser.parse_args() if not args.repository: # No need to postback a comment for this statsreporter.stats().incr("diffs_repository_not_found") return error("Repository not found") repository = args.repository projects = list( Project.query.options(subqueryload_all('plans'), ).filter( Project.status == ProjectStatus.active, Project.repository_id == repository.id, )) # no projects bound to repository if not projects: return self.respond([]) options = dict( db.session.query( ProjectOption.project_id, ProjectOption.value).filter( ProjectOption.project_id.in_([p.id for p in projects]), ProjectOption.name.in_([ 'phabricator.diff-trigger', ]))) # Filter out projects that aren't configured to run builds off of diffs # - Diff trigger disabled # - No build plans projects = [ p for p in projects if options.get(p.id, '1') == '1' and get_build_plans(p) ] if not projects: return self.respond([]) statsreporter.stats().incr('diffs_posted_from_phabricator') label = args.label[:128] author = args.author message = args.message sha = args.sha target = 'D%s' % args['phabricator.revisionID'] try: identify_revision(repository, sha) except MissingRevision: # This may just be a broken request (which is why we respond with a 400) but # it also might indicate Phabricator and Changes being out of sync somehow, # so we err on the side of caution and log it as an error. logging.error( "Diff %s was posted for an unknown revision (%s, %s)", target, sha, repository.url) # We should postback since this can happen if a user diffs dependent revisions statsreporter.stats().incr("diffs_missing_base_revision") return self.postback_error( "Unable to find base revision {revision} in {repo} on Changes. Some possible reasons:\n" " - You may be working on multiple stacked diffs in your local repository.\n" " {revision} only exists in your local copy. Changes thus cannot apply your patch\n" " - If you are sure that's not the case, it's possible you applied your patch to an extremely\n" " recent revision which Changes hasn't picked up yet. Retry in a minute\n" .format( revision=sha, repo=repository.url, ), target, problems=['sha', 'repository']) source_data = { 'phabricator.buildTargetPHID': args['phabricator.buildTargetPHID'], 'phabricator.diffID': args['phabricator.diffID'], 'phabricator.revisionID': args['phabricator.revisionID'], 'phabricator.revisionURL': args['phabricator.revisionURL'], } patch = Patch( repository=repository, parent_revision_sha=sha, diff=''.join(line.decode('utf-8') for line in args.patch_file), ) db.session.add(patch) source = Source( patch=patch, repository=repository, revision_sha=sha, data=source_data, ) db.session.add(source) phabricatordiff = try_create( PhabricatorDiff, { 'diff_id': args['phabricator.diffID'], 'revision_id': args['phabricator.revisionID'], 'url': args['phabricator.revisionURL'], 'source': source, }) if phabricatordiff is None: logging.warning("Diff %s, Revision %s already exists", args['phabricator.diffID'], args['phabricator.revisionID']) # No need to inform user about this explicitly statsreporter.stats().incr("diffs_already_exists") return error("Diff already exists within Changes") project_options = ProjectOptionsHelper.get_options( projects, ['build.file-whitelist']) diff_parser = DiffParser(patch.diff) files_changed = diff_parser.get_changed_files() collection_id = uuid.uuid4() builds = [] for project in projects: plan_list = get_build_plans(project) # We already filtered out empty build plans assert plan_list, ('No plans defined for project {}'.format( project.slug)) try: if not files_changed_should_trigger_project( files_changed, project, project_options[project.id], sha, diff=patch.diff): logging.info( 'No changed files matched project trigger for project %s', project.slug) continue except InvalidDiffError: # ok, the build will fail and the user will be notified pass except ProjectConfigError: logging.error( 'Project config for project %s is not in a valid format. Author is %s.', project.slug, author.name, exc_info=True) builds.append( create_build( project=project, collection_id=collection_id, sha=sha, target=target, label=label, message=message, author=author, patch=patch, tag="phabricator", )) # This is the counterpoint to the above 'diffs_posted_from_phabricator'; # at this point we've successfully processed the diff, so comparing this # stat to the above should give us the phabricator diff failure rate. statsreporter.stats().incr( 'diffs_successfully_processed_from_phabricator') return self.respond(builds)
def post_impl(self): """ Notify Changes of a newly created diff. Depending on system configuration, this may create 0 or more new builds, and the resulting response will be a list of those build objects. """ # we manually check for arg presence here so we can send a more specific # error message to the user (rather than a plain 400) args = self.parser.parse_args() if not args.repository: # No need to postback a comment for this statsreporter.stats().incr("diffs_repository_not_found") return error("Repository not found") repository = args.repository projects = list(Project.query.options( subqueryload_all('plans'), ).filter( Project.status == ProjectStatus.active, Project.repository_id == repository.id, )) # no projects bound to repository if not projects: return self.respond([]) options = dict( db.session.query( ProjectOption.project_id, ProjectOption.value ).filter( ProjectOption.project_id.in_([p.id for p in projects]), ProjectOption.name.in_([ 'phabricator.diff-trigger', ]) ) ) # Filter out projects that aren't configured to run builds off of diffs # - Diff trigger disabled # - No build plans projects = [ p for p in projects if options.get(p.id, '1') == '1' and get_build_plans(p) ] if not projects: return self.respond([]) statsreporter.stats().incr('diffs_posted_from_phabricator') label = args.label[:128] author = args.author message = args.message sha = args.sha target = 'D%s' % args['phabricator.revisionID'] try: identify_revision(repository, sha) except MissingRevision: # This may just be a broken request (which is why we respond with a 400) but # it also might indicate Phabricator and Changes being out of sync somehow, # so we err on the side of caution and log it as an error. logging.error("Diff %s was posted for an unknown revision (%s, %s)", target, sha, repository.url) # We should postback since this can happen if a user diffs dependent revisions statsreporter.stats().incr("diff_missing_base_revision") return self.postback_error( "Unable to find base revision {revision} in {repo} on Changes. Some possible reasons:\n" " - Changes hasn't picked up {revision} yet. Retry in a minute\n" " - {revision} only exists in your local copy. Changes cannot apply your patch\n".format( revision=sha, repo=repository.url, ), target, problems=['sha', 'repository']) source_data = { 'phabricator.buildTargetPHID': args['phabricator.buildTargetPHID'], 'phabricator.diffID': args['phabricator.diffID'], 'phabricator.revisionID': args['phabricator.revisionID'], 'phabricator.revisionURL': args['phabricator.revisionURL'], } patch = Patch( repository=repository, parent_revision_sha=sha, diff=''.join(line.decode('utf-8') for line in args.patch_file), ) db.session.add(patch) source = Source( patch=patch, repository=repository, revision_sha=sha, data=source_data, ) db.session.add(source) phabricatordiff = try_create(PhabricatorDiff, { 'diff_id': args['phabricator.diffID'], 'revision_id': args['phabricator.revisionID'], 'url': args['phabricator.revisionURL'], 'source': source, }) if phabricatordiff is None: logging.warning("Diff %s, Revision %s already exists", args['phabricator.diffID'], args['phabricator.revisionID']) # No need to inform user about this explicitly statsreporter.stats().incr("diffs_already_exists") return error("Diff already exists within Changes") project_options = ProjectOptionsHelper.get_options(projects, ['build.file-whitelist']) diff_parser = DiffParser(patch.diff) files_changed = diff_parser.get_changed_files() collection_id = uuid.uuid4() builds = [] for project in projects: plan_list = get_build_plans(project) # We already filtered out empty build plans assert plan_list, ('No plans defined for project {}'.format(project.slug)) try: if not files_changed_should_trigger_project(files_changed, project, project_options[project.id], sha, diff=patch.diff): logging.info('No changed files matched project trigger for project %s', project.slug) continue except InvalidDiffError: # ok, the build will fail and the user will be notified pass except ProjectConfigError: logging.error('Project config for project %s is not in a valid format. Author is %s.', project.slug, author.name, exc_info=True) builds.append(create_build( project=project, collection_id=collection_id, sha=sha, target=target, label=label, message=message, author=author, patch=patch, tag="phabricator", )) # This is the counterpoint to the above 'diffs_posted_from_phabricator'; # at this point we've successfully processed the diff, so comparing this # stat to the above should give us the phabricator diff failure rate. statsreporter.stats().incr('diffs_successfully_processed_from_phabricator') return self.respond(builds)
def post(self): """ Notify Changes of a newly created diff. Depending on system configuration, this may create 0 or more new builds, and the resulting response will be a list of those build objects. """ args = self.parser.parse_args() repository = args.repository if not args.repository: return error("Repository not found") projects = list(Project.query.options( subqueryload_all('plans'), ).filter( Project.status == ProjectStatus.active, Project.repository_id == repository.id, )) # no projects bound to repository if not projects: return self.respond([]) options = dict( db.session.query( ProjectOption.project_id, ProjectOption.value ).filter( ProjectOption.project_id.in_([p.id for p in projects]), ProjectOption.name.in_([ 'phabricator.diff-trigger', ]) ) ) projects = [ p for p in projects if options.get(p.id, '1') == '1' ] if not projects: return self.respond([]) label = args.label[:128] author = args.author message = args.message sha = args.sha target = 'D{}'.format(args['phabricator.revisionID']) try: identify_revision(repository, sha) except MissingRevision: return error("Unable to find commit %s in %s." % ( sha, repository.url), problems=['sha', 'repository']) patch = Patch( repository=repository, parent_revision_sha=sha, diff=''.join(args.patch_file), ) db.session.add(patch) patch_data = { 'phabricator.buildTargetPHID': args['phabricator.buildTargetPHID'], 'phabricator.diffID': args['phabricator.diffID'], 'phabricator.revisionID': args['phabricator.revisionID'], 'phabricator.revisionURL': args['phabricator.revisionURL'], } builds = [] for project in projects: plan_list = get_build_plans(project) if not plan_list: logging.warning('No plans defined for project %s', project.slug) continue builds.append(create_build( project=project, sha=sha, target=target, label=label, message=message, author=author, patch=patch, source_data=patch_data, )) return self.respond(builds)