def test_patch(self, find_job):
responses.add(responses.POST,
'http://jenkins.example.com/job/server/build/api/json/',
body='',
status=201)
find_job.return_value = {
'build_no': '1',
'item_id': None,
'job_name': 'server',
'queued': False,
}
patch = Patch(
repository=self.project.repository,
parent_revision_sha='7ebd1f2d750064652ef5bbff72452cc19e1731e0',
diff=SAMPLE_DIFF,
)
db.session.add(patch)
source = self.create_source(self.project, patch=patch)
build = self.create_build(self.project, source=source)
job = self.create_job(build=build,
id=UUID('81d1596fd4d642f4a6bdf86c45e014e8'))
builder = self.get_builder()
builder.create_job(job)
def patch(project, **kwargs):
kwargs.setdefault('diff', SAMPLE_DIFF)
patch = Patch(repository=project.repository, **kwargs)
db.session.add(patch)
return patch
def test_ensure_match_patch_want_commit(self, get_vcs):
"""This makes sure that the ensure API handles diff builds correctly.
This is the case where we want to ensure a commit build.
"""
get_vcs.return_value = self.get_fake_vcs()
revision = self.create_revision(repository=self.project.repository,
sha='a' * 40)
patch = Patch(
repository=self.project.repository,
parent_revision_sha=revision.sha,
diff=SAMPLE_DIFF,
)
source = self.create_source(self.project, revision=revision)
bad_source = self.create_source(self.project,
revision=revision,
patch=patch)
build = self.create_build(self.project, source=source)
# if diff builds weren't handled properly, this build would be older
# and would be returned
self.create_build(self.project, source=bad_source)
resp = self.client.post(self.path,
data={
'sha': 'a' * 40,
'project': self.project.slug,
'apply_project_files_trigger': '1',
'ensure_only': '1',
})
assert resp.status_code == 200
data = self.unserialize(resp)
assert len(data) == 1
assert data[0]['id'] == build.id.hex
def test_patch(self):
responses.add(
responses.POST,
'https://koality.example.com/api/v/0/repositories/1/changes',
body=self.load_fixture('fixtures/POST/change_index.json'))
revision = '7ebd1f2d750064652ef5bbff72452cc19e1731e0'
patch = Patch(
repository=self.repo,
project=self.project,
parent_revision_sha=revision,
label='D1345',
diff=SAMPLE_DIFF,
)
db.session.add(patch)
source = self.create_source(self.project,
patch=patch,
revision_sha=revision)
build = self.create_build(self.project, source=source)
job = self.create_job(build=build, )
backend = self.get_builder()
backend.create_job(job=job, )
assert job.data == {
'project_id': 1,
'change_id': 1501,
}
assert len(responses.calls) == 1
def create_patch(self, **kwargs):
kwargs.setdefault('diff', SAMPLE_DIFF)
kwargs.setdefault('parent_revision_sha', uuid4().hex)
if not kwargs.get('repository'):
kwargs['repository'] = self.create_repo()
kwargs['repository_id'] = kwargs['repository'].id
patch = Patch(**kwargs)
db.session.add(patch)
db.session.commit()
return patch
def test_with_revision_addressees(self):
db.session.add(
ProjectOption(project=self.project,
name='mail.notify-author',
value='1'))
db.session.add(
ProjectOption(project=self.project,
name='mail.notify-addresses-revisions',
value='[email protected], [email protected]'))
author = self.create_author('*****@*****.**')
patch = Patch(
repository=self.repo,
project=self.project,
label='foo',
diff='',
)
source = self.create_source(self.project, patch=patch)
build = self.create_build(
project=self.project,
source=source,
author=author,
result=Result.failed,
)
job = self.create_job(build=build)
db.session.commit()
handler = MailNotificationHandler()
recipients = handler.get_recipients(job)
assert recipients == ['{0} <*****@*****.**>'.format(author.name)]
build = self.create_build(
project=self.project,
result=Result.failed,
author=author,
)
job = self.create_job(build=build)
job_finished_handler(job)
handler = MailNotificationHandler()
recipients = handler.get_recipients(job)
assert recipients == [
'{0} <*****@*****.**>'.format(author.name),
'*****@*****.**',
'*****@*****.**',
]
def test_simple():
patch = Patch(
id=UUID(hex='33846695b2774b29a71795a009e8168a'),
diff=SAMPLE_DIFF,
parent_revision_sha='1e7958a368f44b0eb5a57372a9910d50',
date_created=datetime(2013, 9, 19, 22, 15, 22),
)
result = serialize(patch)
assert result[
'link'] == 'http://example.com/patches/33846695b2774b29a71795a009e8168a/'
assert result['id'] == '33846695b2774b29a71795a009e8168a'
assert result['parentRevision'] == {
'sha': '1e7958a368f44b0eb5a57372a9910d50',
}
assert result['dateCreated'] == '2013-09-19T22:15:22'
assert result['diff'] == SAMPLE_DIFF
def create_patch(self, project, **kwargs):
kwargs.setdefault('label', 'Test Patch')
kwargs.setdefault('message', 'Hello world!')
kwargs.setdefault('diff', SAMPLE_DIFF)
kwargs.setdefault('parent_revision_sha', uuid4().hex)
if not kwargs.get('repository'):
kwargs['repository'] = self.create_repo()
kwargs['repository_id'] = kwargs['repository'].id
patch = Patch(
project=project,
project_id=project.id,
**kwargs
)
db.session.add(patch)
db.session.commit()
return patch
def post(self):
"""
Notify Changes of a newly created diff.
Depending on system configuration, this may create 0 or more new builds,
and the resulting response will be a list of those build objects.
"""
args = self.parser.parse_args()
repository = args.repository
if not args.repository:
return error("Repository not found")
projects = list(
Project.query.options(subqueryload_all('plans'), ).filter(
Project.status == ProjectStatus.active,
Project.repository_id == repository.id,
))
# no projects bound to repository
if not projects:
return self.respond([])
options = dict(
db.session.query(
ProjectOption.project_id, ProjectOption.value).filter(
ProjectOption.project_id.in_([p.id for p in projects]),
ProjectOption.name.in_([
'phabricator.diff-trigger',
])))
projects = [p for p in projects if options.get(p.id, '1') == '1']
if not projects:
return self.respond([])
label = args.label[:128]
author = args.author
message = args.message
sha = args.sha
target = 'D{}'.format(args['phabricator.revisionID'])
try:
identify_revision(repository, sha)
except MissingRevision:
return error("Unable to find commit %s in %s." %
(sha, repository.url),
problems=['sha', 'repository'])
source_data = {
'phabricator.buildTargetPHID': args['phabricator.buildTargetPHID'],
'phabricator.diffID': args['phabricator.diffID'],
'phabricator.revisionID': args['phabricator.revisionID'],
'phabricator.revisionURL': args['phabricator.revisionURL'],
}
patch = Patch(
repository=repository,
parent_revision_sha=sha,
diff=''.join(args.patch_file),
)
db.session.add(patch)
source = Source(
patch=patch,
repository=repository,
revision_sha=sha,
data=source_data,
)
db.session.add(source)
phabricatordiff = try_create(
PhabricatorDiff, {
'diff_id': args['phabricator.diffID'],
'revision_id': args['phabricator.revisionID'],
'url': args['phabricator.revisionURL'],
'source': source,
})
if phabricatordiff is None:
logging.error("Diff %s, Revision %s already exists",
args['phabricator.diffID'],
args['phabricator.revisionID'])
return error("Diff already exists within Changes")
project_options = ProjectOptionsHelper.get_options(
projects, ['build.file-whitelist'])
diff_parser = DiffParser(patch.diff)
files_changed = diff_parser.get_changed_files()
collection_id = uuid.uuid4()
builds = []
for project in projects:
plan_list = get_build_plans(project)
if not plan_list:
logging.warning('No plans defined for project %s',
project.slug)
continue
if not in_project_files_whitelist(project_options[project.id],
files_changed):
logging.info(
'No changed files matched build.file-whitelist for project %s',
project.slug)
continue
builds.append(
create_build(
project=project,
collection_id=collection_id,
sha=sha,
target=target,
label=label,
message=message,
author=author,
patch=patch,
))
return self.respond(builds)
def post(self):
"""
Create a new commit or diff build. The API roughly goes like this:
1. Identify the project(s) to build for. This can be done by specifying
``project``, ``repository``, or ``repository[callsign]``. If a repository is
specified somehow, then all projects for that repository are considered
for building.
2. Using the ``sha``, find the appropriate revision object. This may
involve updating the repo.
3. If ``patch`` is given, then apply the patch and mark this as a diff build.
Otherwise, this is a commit build.
4. If ``snapshot_id`` is given, verify that the snapshot can be used by all
projects.
5. If provided, apply project_whitelist, filtering out projects not in
this whitelist.
6. Based on the flag ``apply_project_files_trigger`` (see comment on the argument
itself for default values), decide whether or not to filter out projects
by file blacklist and whitelist.
7. Attach metadata and create/ensure existence of a build for each project,
depending on the flag ``ensure_only``.
NOTE: In ensure-only mode, the collection_ids of the returned builds are
not necessarily identical, as we give new builds new collection IDs
and preserve the existing builds' collection IDs.
NOTE: If ``patch`` is specified ``sha`` is assumed to be the original
base revision to apply the patch.
Not relevant until we fix TODO: ``sha`` is **not** guaranteed to be the rev
used to apply the patch. See ``find_green_parent_sha`` for the logic of
identifying the correct revision.
"""
args = self.parser.parse_args()
if args.patch_file and args.ensure_only:
return error(
"Ensure-only mode does not work with a diff build yet.",
problems=["patch", "ensure_only"])
if not (args.project or args.repository
or args['repository[phabricator.callsign]']):
return error("Project or repository must be specified",
problems=[
"project", "repository",
"repository[phabricator.callsign]"
])
# read arguments
if args.patch_data:
try:
patch_data = json.loads(args.patch_data)
except Exception:
return error("Invalid patch data (must be JSON dict)",
problems=["patch[data]"])
if not isinstance(patch_data, dict):
return error("Invalid patch data (must be JSON dict)",
problems=["patch[data]"])
else:
patch_data = None
# 1. identify project(s)
projects, repository = try_get_projects_and_repository(args)
if not projects:
return error("Unable to find project(s).")
# read arguments
label = args.label
author = args.author
message = args.message
tag = args.tag
snapshot_id = args.snapshot_id
no_snapshot = args.no_snapshot
if no_snapshot and snapshot_id:
return error("Cannot specify snapshot with no_snapshot option")
if not tag and args.patch_file:
tag = 'patch'
# 2. validate snapshot
if snapshot_id:
snapshot = Snapshot.query.get(snapshot_id)
if not snapshot:
return error("Unable to find snapshot.")
if snapshot.status != SnapshotStatus.active:
return error("Snapshot is in an invalid state: %s" %
snapshot.status)
for project in projects:
plans = get_build_plans(project)
for plan in plans:
plan_options = plan.get_item_options()
allow_snapshot = '1' == plan_options.get(
'snapshot.allow', '0') or plan.snapshot_plan
if allow_snapshot and not SnapshotImage.get(
plan, snapshot_id):
# We want to create a build using a specific snapshot but no image
# was found for this plan so fail.
return error("Snapshot cannot be applied to %s's %s" %
(project.slug, plan.label))
# 3. find revision
try:
revision = identify_revision(repository, args.sha)
except MissingRevision:
# if the default fails, we absolutely can't continue and the
# client should send a valid revision
return error("Unable to find commit %s in %s." %
(args.sha, repository.url),
problems=['sha', 'repository'])
# get default values for arguments
if revision:
if not author:
author = revision.author
if not label:
label = revision.subject
# only default the message if its absolutely not set
if message is None:
message = revision.message
sha = revision.sha
else:
sha = args.sha
if not args.target:
target = sha[:12]
else:
target = args.target[:128]
if not label:
if message:
label = message.splitlines()[0]
if not label:
label = 'A homeless build'
label = label[:128]
# 4. Check for patch
if args.patch_file:
fp = StringIO()
for line in args.patch_file:
fp.write(line)
patch_file = fp
else:
patch_file = None
if patch_file:
patch = Patch(
repository=repository,
parent_revision_sha=sha,
diff=patch_file.getvalue(),
)
db.session.add(patch)
else:
patch = None
project_options = ProjectOptionsHelper.get_options(
projects, ['build.file-whitelist'])
# mark as commit or diff build
if not patch:
is_commit_build = True
else:
is_commit_build = False
apply_project_files_trigger = args.apply_project_files_trigger
if apply_project_files_trigger is None:
apply_project_files_trigger = args.apply_file_whitelist
if apply_project_files_trigger is None:
if is_commit_build:
apply_project_files_trigger = False
else:
apply_project_files_trigger = True
if apply_project_files_trigger:
if patch:
diff_parser = DiffParser(patch.diff)
files_changed = diff_parser.get_changed_files()
elif revision:
try:
files_changed = _get_revision_changed_files(
repository, revision)
except MissingRevision:
return error("Unable to find commit %s in %s." %
(args.sha, repository.url),
problems=['sha', 'repository'])
else:
# the only way that revision can be null is if this repo does not have a vcs backend
logging.warning(
'Revision and patch are both None for sha %s. This is because the repo %s does not have a VCS backend.',
sha, repository.url)
files_changed = None
else:
# we won't be applying file whitelist, so there is no need to get the list of changed files.
files_changed = None
collection_id = uuid.uuid4()
builds = []
for project in projects:
plan_list = get_build_plans(project)
if not plan_list:
logging.warning('No plans defined for project %s',
project.slug)
continue
# 5. apply project whitelist as appropriate
if args.project_whitelist is not None and project.slug not in args.project_whitelist:
logging.info('Project %s is not in the supplied whitelist',
project.slug)
continue
forced_sha = sha
# TODO(dcramer): find_green_parent_sha needs to take branch
# into account
# if patch_file:
# forced_sha = find_green_parent_sha(
# project=project,
# sha=sha,
# )
# 6. apply file whitelist as appropriate
diff = None
if patch is not None:
diff = patch.diff
try:
if (apply_project_files_trigger and files_changed is not None
and not files_changed_should_trigger_project(
files_changed, project,
project_options[project.id], sha, diff)):
logging.info(
'Changed files do not trigger build for project %s',
project.slug)
continue
except InvalidDiffError:
# ok, the build will fail and the user will be notified.
pass
except ProjectConfigError:
author_name = '(Unknown)'
if author:
author_name = author.name
logging.error(
'Project config for project %s is not in a valid format. Author is %s.',
project.slug,
author_name,
exc_info=True)
# 7. create/ensure build
if args.ensure_only:
potentials = list(
Build.query.filter(
Build.project_id == project.id,
Build.source.has(revision_sha=sha, patch=patch),
).order_by(Build.date_created.desc() # newest first
).limit(1))
if len(potentials) == 0:
builds.append(
create_build(project=project,
collection_id=collection_id,
sha=forced_sha,
target=target,
label=label,
message=message,
author=author,
patch=patch,
source_data=patch_data,
tag=tag,
snapshot_id=snapshot_id,
no_snapshot=no_snapshot))
else:
builds.append(potentials[0])
else:
builds.append(
create_build(project=project,
collection_id=collection_id,
sha=forced_sha,
target=target,
label=label,
message=message,
author=author,
patch=patch,
source_data=patch_data,
tag=tag,
snapshot_id=snapshot_id,
no_snapshot=no_snapshot))
return self.respond(builds)
def post(self):
"""
Notify Changes of a newly created diff.
Depending on system configuration, this may create 0 or more new builds,
and the resulting response will be a list of those build objects.
"""
args = self.parser.parse_args()
repository = args.repository
if not args.repository:
return error("Repository not found")
projects = list(
Project.query.options(subqueryload_all('plans'), ).filter(
Project.status == ProjectStatus.active,
Project.repository_id == repository.id,
))
# no projects bound to repository
if not projects:
return self.respond([])
options = dict(
db.session.query(
ProjectOption.project_id, ProjectOption.value).filter(
ProjectOption.project_id.in_([p.id for p in projects]),
ProjectOption.name.in_([
'phabricator.diff-trigger',
])))
projects = [p for p in projects if options.get(p.id, '1') == '1']
if not projects:
return self.respond([])
statsreporter.stats().incr('diffs_posted_from_phabricator')
label = args.label[:128]
author = args.author
message = args.message
sha = args.sha
target = 'D{}'.format(args['phabricator.revisionID'])
try:
identify_revision(repository, sha)
except MissingRevision:
# This may just be a broken request (which is why we respond with a 400) but
# it also might indicate Phabricator and Changes being out of sync somehow,
# so we err on the side of caution and log it as an error.
logging.error(
"Diff %s was posted for an unknown revision (%s, %s)", target,
sha, repository.url)
return error("Unable to find commit %s in %s." %
(sha, repository.url),
problems=['sha', 'repository'])
source_data = {
'phabricator.buildTargetPHID': args['phabricator.buildTargetPHID'],
'phabricator.diffID': args['phabricator.diffID'],
'phabricator.revisionID': args['phabricator.revisionID'],
'phabricator.revisionURL': args['phabricator.revisionURL'],
}
patch = Patch(
repository=repository,
parent_revision_sha=sha,
diff=''.join(args.patch_file),
)
db.session.add(patch)
source = Source(
patch=patch,
repository=repository,
revision_sha=sha,
data=source_data,
)
db.session.add(source)
phabricatordiff = try_create(
PhabricatorDiff, {
'diff_id': args['phabricator.diffID'],
'revision_id': args['phabricator.revisionID'],
'url': args['phabricator.revisionURL'],
'source': source,
})
if phabricatordiff is None:
logging.error("Diff %s, Revision %s already exists",
args['phabricator.diffID'],
args['phabricator.revisionID'])
return error("Diff already exists within Changes")
project_options = ProjectOptionsHelper.get_options(
projects, ['build.file-whitelist'])
diff_parser = DiffParser(patch.diff)
files_changed = diff_parser.get_changed_files()
collection_id = uuid.uuid4()
builds = []
for project in projects:
plan_list = get_build_plans(project)
if not plan_list:
logging.warning('No plans defined for project %s',
project.slug)
continue
try:
if not files_changed_should_trigger_project(
files_changed,
project,
project_options[project.id],
sha,
diff=patch.diff):
logging.info(
'No changed files matched project trigger for project %s',
project.slug)
continue
except InvalidDiffError:
# ok, the build will fail and the user will be notified
pass
except ProjectConfigError:
logging.error(
'Project config for project %s is not in a valid format. Author is %s.',
project.slug,
author.name,
exc_info=True)
builds.append(
create_build(
project=project,
collection_id=collection_id,
sha=sha,
target=target,
label=label,
message=message,
author=author,
patch=patch,
tag="phabricator",
))
# This is the counterpoint to the above 'diffs_posted_from_phabricator';
# at this point we've successfully processed the diff, so comparing this
# stat to the above should give us the phabricator diff failure rate.
statsreporter.stats().incr(
'diffs_successfully_processed_from_phabricator')
return self.respond(builds)
def post(self):
"""
Note: If ``patch`` is specified ``sha`` is assumed to be the original
base revision to apply the patch. It is **not** guaranteed to be the rev
used to apply the patch. See ``find_green_parent_sha`` for the logic of
identifying the correct revision.
"""
args = self.parser.parse_args()
if not (args.project or args.repository
or args['repository[phabricator.callsign]']):
return '{"error": "Need project or repository"}', 400
if args.patch_data:
try:
patch_data = json.loads(args.patch_data)
except Exception:
return '{"error": "Invalid patch data (must be JSON dict)"}', 400
if not isinstance(patch_data, dict):
return '{"error": "Invalid patch data (must be JSON dict)"}', 400
else:
patch_data = None
if args.project:
projects = [args.project]
repository = Repository.query.get(args.project.repository_id)
elif args.repository:
repository = args.repository
projects = list(
Project.query.options(
subqueryload_all(
Project.project_plans, ProjectPlan.plan), ).filter(
Project.status == ProjectStatus.active,
Project.repository_id == repository.id,
))
elif args['repository[phabricator.callsign]']:
repository = args['repository[phabricator.callsign]']
projects = list(
Project.query.options(
subqueryload_all(
Project.project_plans, ProjectPlan.plan), ).filter(
Project.status == ProjectStatus.active,
Project.repository_id == repository.id,
))
if not projects:
return '{"error": "Unable to find project(s)."}', 400
if args.patch_file:
# eliminate projects without diff builds
options = dict(
db.session.query(
ProjectOption.project_id, ProjectOption.value).filter(
ProjectOption.project_id.in_([p.id for p in projects]),
ProjectOption.name.in_([
'build.allow-patches',
])))
projects = [p for p in projects if options.get(p.id, '1') == '1']
if not projects:
return self.respond([])
label = args.label
author = args.author
message = args.message
try:
revision = identify_revision(repository, args.sha)
except MissingRevision:
# if the default fails, we absolutely can't continue and the
# client should send a valid revision
return '{"error": "Unable to find a commit to build."}', 400
if revision:
if not author:
author = revision.author
if not label:
label = revision.subject
# only default the message if its absolutely not set
if message is None:
message = revision.message
sha = revision.sha
else:
sha = args.sha
if not args.target:
target = sha[:12]
else:
target = args.target[:128]
if not label:
if message:
label = message.splitlines()[0]
if not label:
label = 'A homeless build'
label = label[:128]
if args.patch_file:
fp = StringIO()
for line in args.patch_file:
fp.write(line)
patch_file = fp
else:
patch_file = None
if patch_file:
patch = Patch(
repository=repository,
parent_revision_sha=sha,
diff=patch_file.getvalue(),
)
db.session.add(patch)
else:
patch = None
builds = []
for project in projects:
plan_list = list(project.plans)
if not plan_list:
logging.warning('No plans defined for project %s',
project.slug)
continue
if plan_list and patch_file:
options = dict(
db.session.query(
ItemOption.item_id, ItemOption.value).filter(
ItemOption.item_id.in_([p.id for p in plan_list]),
ItemOption.name.in_([
'build.allow-patches',
])))
plan_list = [
p for p in plan_list if options.get(p.id, '1') == '1'
]
# no plans remained
if not plan_list:
continue
if patch_file:
forced_sha = find_green_parent_sha(
project=project,
sha=sha,
)
else:
forced_sha = sha
builds.append(
create_build(
project=project,
sha=forced_sha,
target=target,
label=label,
message=message,
author=author,
patch=patch,
source_data=patch_data,
))
return self.respond(builds)
def post(self):
args = self.parser.parse_args()
if not (args.project or args.repository):
return '{"error": "Need project or repository"}', 400
if args.patch_file and not args.patch_label:
return '{"error": "Missing patch label"}', 400
if args.project:
projects = [args.project]
repository = Repository.query.get(args.project.repository_id)
else:
repository = args.repository
projects = list(Project.query.options(
subqueryload_all(Project.project_plans, ProjectPlan.plan),
).filter(
Project.status == ProjectStatus.active,
Project.repository_id == repository.id,
))
if not projects:
return '{"error": "Unable to find project(s)."}', 400
if args.patch_file:
# eliminate projects without diff builds
options = dict(
db.session.query(
ProjectOption.project_id, ProjectOption.value
).filter(
ProjectOption.project_id.in_([p.id for p in projects]),
ProjectOption.name.in_([
'build.allow-patches',
])
)
)
projects = [
p for p in projects
if options.get(p.id, '1') == '1'
]
if not projects:
return self.respond([])
label = args.label
author = args.author
message = args.message
revision = identify_revision(repository, args.sha)
if revision:
if not author:
author = revision.author
if not label:
label = revision.subject
# only default the message if its absolutely not set
if message is None:
message = revision.message
sha = revision.sha
else:
sha = args.sha
if not args.target:
if args.patch_label:
target = args.patch_label[:128]
else:
target = sha[:12]
else:
target = args.target[:128]
if not label:
if message:
label = message.splitlines()[0]
if not label:
label = 'A homeless build'
label = label[:128]
if args.patch_file:
fp = StringIO()
for line in args.patch_file:
fp.write(line)
patch_file = fp
else:
patch_file = None
builds = []
for project in projects:
plan_list = list(project.plans)
if not plan_list:
logging.warning('No plans defined for project %s', project.slug)
continue
if plan_list and patch_file:
options = dict(
db.session.query(
ItemOption.item_id, ItemOption.value
).filter(
ItemOption.item_id.in_([p.id for p in plan_list]),
ItemOption.name.in_([
'build.allow-patches',
])
)
)
plan_list = [
p for p in plan_list
if options.get(p.id, '1') == '1'
]
# no plans remained
if not plan_list:
continue
if patch_file:
patch = Patch(
repository=repository,
project=project,
parent_revision_sha=args.sha,
label=args.patch_label,
diff=patch_file.getvalue(),
)
db.session.add(patch)
else:
patch = None
builds.append(create_build(
project=project,
sha=sha,
target=target,
label=label,
message=message,
author=author,
patch=patch,
))
return self.respond(builds)
def post_impl(self):
"""
Notify Changes of a newly created diff.
Depending on system configuration, this may create 0 or more new builds,
and the resulting response will be a list of those build objects.
"""
# we manually check for arg presence here so we can send a more specific
# error message to the user (rather than a plain 400)
args = self.parser.parse_args()
if not args.repository:
# No need to postback a comment for this
statsreporter.stats().incr("diffs_repository_not_found")
return error("Repository not found")
repository = args.repository
projects = list(
Project.query.options(subqueryload_all('plans'), ).filter(
Project.status == ProjectStatus.active,
Project.repository_id == repository.id,
))
# no projects bound to repository
if not projects:
return self.respond([])
options = dict(
db.session.query(
ProjectOption.project_id, ProjectOption.value).filter(
ProjectOption.project_id.in_([p.id for p in projects]),
ProjectOption.name.in_([
'phabricator.diff-trigger',
])))
# Filter out projects that aren't configured to run builds off of diffs
# - Diff trigger disabled
# - No build plans
projects = [
p for p in projects
if options.get(p.id, '1') == '1' and get_build_plans(p)
]
if not projects:
return self.respond([])
statsreporter.stats().incr('diffs_posted_from_phabricator')
label = args.label[:128]
author = args.author
message = args.message
sha = args.sha
target = 'D%s' % args['phabricator.revisionID']
try:
identify_revision(repository, sha)
except MissingRevision:
# This may just be a broken request (which is why we respond with a 400) but
# it also might indicate Phabricator and Changes being out of sync somehow,
# so we err on the side of caution and log it as an error.
logging.error(
"Diff %s was posted for an unknown revision (%s, %s)", target,
sha, repository.url)
# We should postback since this can happen if a user diffs dependent revisions
statsreporter.stats().incr("diffs_missing_base_revision")
return self.postback_error(
"Unable to find base revision {revision} in {repo} on Changes. Some possible reasons:\n"
" - You may be working on multiple stacked diffs in your local repository.\n"
" {revision} only exists in your local copy. Changes thus cannot apply your patch\n"
" - If you are sure that's not the case, it's possible you applied your patch to an extremely\n"
" recent revision which Changes hasn't picked up yet. Retry in a minute\n"
.format(
revision=sha,
repo=repository.url,
),
target,
problems=['sha', 'repository'])
source_data = {
'phabricator.buildTargetPHID': args['phabricator.buildTargetPHID'],
'phabricator.diffID': args['phabricator.diffID'],
'phabricator.revisionID': args['phabricator.revisionID'],
'phabricator.revisionURL': args['phabricator.revisionURL'],
}
patch = Patch(
repository=repository,
parent_revision_sha=sha,
diff=''.join(line.decode('utf-8') for line in args.patch_file),
)
db.session.add(patch)
source = Source(
patch=patch,
repository=repository,
revision_sha=sha,
data=source_data,
)
db.session.add(source)
phabricatordiff = try_create(
PhabricatorDiff, {
'diff_id': args['phabricator.diffID'],
'revision_id': args['phabricator.revisionID'],
'url': args['phabricator.revisionURL'],
'source': source,
})
if phabricatordiff is None:
logging.warning("Diff %s, Revision %s already exists",
args['phabricator.diffID'],
args['phabricator.revisionID'])
# No need to inform user about this explicitly
statsreporter.stats().incr("diffs_already_exists")
return error("Diff already exists within Changes")
project_options = ProjectOptionsHelper.get_options(
projects, ['build.file-whitelist'])
diff_parser = DiffParser(patch.diff)
files_changed = diff_parser.get_changed_files()
collection_id = uuid.uuid4()
builds = []
for project in projects:
plan_list = get_build_plans(project)
# We already filtered out empty build plans
assert plan_list, ('No plans defined for project {}'.format(
project.slug))
try:
if not files_changed_should_trigger_project(
files_changed,
project,
project_options[project.id],
sha,
diff=patch.diff):
logging.info(
'No changed files matched project trigger for project %s',
project.slug)
continue
except InvalidDiffError:
# ok, the build will fail and the user will be notified
pass
except ProjectConfigError:
logging.error(
'Project config for project %s is not in a valid format. Author is %s.',
project.slug,
author.name,
exc_info=True)
builds.append(
create_build(
project=project,
collection_id=collection_id,
sha=sha,
target=target,
label=label,
message=message,
author=author,
patch=patch,
tag="phabricator",
))
# This is the counterpoint to the above 'diffs_posted_from_phabricator';
# at this point we've successfully processed the diff, so comparing this
# stat to the above should give us the phabricator diff failure rate.
statsreporter.stats().incr(
'diffs_successfully_processed_from_phabricator')
return self.respond(builds)
def post(self):
"""
Note: If ``patch`` is specified ``sha`` is assumed to be the original
base revision to apply the patch. It is **not** guaranteed to be the rev
used to apply the patch. See ``find_green_parent_sha`` for the logic of
identifying the correct revision.
"""
args = self.parser.parse_args()
if not (args.project or args.repository
or args['repository[phabricator.callsign]']):
return error("Project or repository must be specified",
problems=[
"project", "repository",
"repository[phabricator.callsign]"
])
if args.patch_data:
try:
patch_data = json.loads(args.patch_data)
except Exception:
return error("Invalid patch data (must be JSON dict)",
problems=["patch[data]"])
if not isinstance(patch_data, dict):
return error("Invalid patch data (must be JSON dict)",
problems=["patch[data]"])
else:
patch_data = None
if args.project:
projects = [args.project]
repository = Repository.query.get(args.project.repository_id)
elif args.repository:
repository = args.repository
projects = list(
Project.query.options(subqueryload_all('plans'), ).filter(
Project.status == ProjectStatus.active,
Project.repository_id == repository.id,
))
elif args['repository[phabricator.callsign]']:
repository = args['repository[phabricator.callsign]']
projects = list(
Project.query.options(subqueryload_all('plans'), ).filter(
Project.status == ProjectStatus.active,
Project.repository_id == repository.id,
))
if not projects:
return error("Unable to find project(s).")
label = args.label
author = args.author
message = args.message
tag = args.tag
if not tag and args.patch_file:
tag = 'patch'
try:
revision = identify_revision(repository, args.sha)
except MissingRevision:
# if the default fails, we absolutely can't continue and the
# client should send a valid revision
return error("Unable to find commit %s in %s." %
(args.sha, repository.url),
problems=['sha', 'repository'])
if revision:
if not author:
author = revision.author
if not label:
label = revision.subject
# only default the message if its absolutely not set
if message is None:
message = revision.message
sha = revision.sha
else:
sha = args.sha
if not args.target:
target = sha[:12]
else:
target = args.target[:128]
if not label:
if message:
label = message.splitlines()[0]
if not label:
label = 'A homeless build'
label = label[:128]
if args.patch_file:
fp = StringIO()
for line in args.patch_file:
fp.write(line)
patch_file = fp
else:
patch_file = None
if patch_file:
patch = Patch(
repository=repository,
parent_revision_sha=sha,
diff=patch_file.getvalue(),
)
db.session.add(patch)
else:
patch = None
project_options = ProjectOptionsHelper.get_options(
projects, ['build.file-whitelist'])
if patch:
diff_parser = DiffParser(patch.diff)
files_changed = diff_parser.get_changed_files()
else:
files_changed = None
collection_id = uuid.uuid4()
builds = []
for project in projects:
plan_list = get_build_plans(project)
if not plan_list:
logging.warning('No plans defined for project %s',
project.slug)
continue
forced_sha = sha
# TODO(dcramer): find_green_parent_sha needs to take branch
# into account
# if patch_file:
# forced_sha = find_green_parent_sha(
# project=project,
# sha=sha,
# )
if files_changed and not in_project_files_whitelist(
project_options[project.id], files_changed):
logging.info(
'No changed files matched build.file-whitelist for project %s',
project.slug)
continue
builds.append(
create_build(
project=project,
collection_id=collection_id,
sha=forced_sha,
target=target,
label=label,
message=message,
author=author,
patch=patch,
source_data=patch_data,
tag=tag,
))
return self.respond(builds)