def test_get_config_invalid_diff(self):
project = self.create_project()
with mock.patch.object(project, 'get_config') as mock_get_config:
mock_get_config.side_effect = InvalidDiffError
policy, messages = get_selective_testing_policy(project, 'a' * 40, diff='diff')
mock_get_config.assert_called_once_with('a' * 40, 'diff')
assert len(messages) > 0
assert policy is SelectiveTestingPolicy.disabled
def post(self, build_id):
args = self.parser.parse_args()
build = Build.query.options(
joinedload('project', innerjoin=True),
joinedload('author'),
joinedload('source').joinedload('revision'),
).get(build_id)
if build is None:
return '', 404
selective_testing_policy = SelectiveTestingPolicy.disabled
build_message = None
if args.selective_testing and project_lib.contains_active_autogenerated_plan(
build.project):
if not build.source.patch:
selective_testing_policy, reasons = get_selective_testing_policy(
build.project, build.source.revision_sha, None)
if reasons:
if selective_testing_policy is SelectiveTestingPolicy.disabled:
reasons = [
"Selective testing was requested but not done because:"
] + [' ' + m for m in reasons]
build_message = '\n'.join(reasons)
else:
# NOTE: for diff builds, it makes sense to just do selective testing,
# since it will never become a parent build and will never be used to
# calculate revision results.
selective_testing_policy = SelectiveTestingPolicy.enabled
collection_id = uuid.uuid4()
new_build = create_build(
project=build.project,
collection_id=collection_id,
label=build.label,
target=build.target,
message=build.message,
author=build.author,
source=build.source,
cause=Cause.retry,
selective_testing_policy=selective_testing_policy,
)
if build_message:
message = BuildMessage(
build=new_build,
text=build_message,
)
db.session.add(message)
db.session.commit()
return '', 302, {
'Location': '/api/0/builds/{0}/'.format(new_build.id.hex)
}
def test_config(self):
project = self.create_project()
with mock.patch.object(project, 'get_config') as mock_get_config:
with mock.patch.object(ProjectOptionsHelper, 'get_whitelisted_paths') as mock_get_whitelisted_paths:
mock_get_config.return_value = {
'build.file-blacklist': [],
'bazel.selective-testing-enabled': False,
}
mock_get_whitelisted_paths.return_value = []
policy, messages = get_selective_testing_policy(project, 'a' * 40, diff='diff')
mock_get_config.assert_called_once_with('a' * 40, 'diff')
mock_get_whitelisted_paths.assert_called_once_with(project)
assert len(messages) > 0
assert policy is SelectiveTestingPolicy.disabled
def post(self, build_id):
args = self.parser.parse_args()
build = Build.query.options(
joinedload('project', innerjoin=True),
joinedload('author'),
joinedload('source').joinedload('revision'),
).get(build_id)
if build is None:
return '', 404
selective_testing_policy = SelectiveTestingPolicy.disabled
build_message = None
if args.selective_testing and project_lib.contains_active_autogenerated_plan(build.project):
if not build.source.patch:
selective_testing_policy, reasons = get_selective_testing_policy(build.project, build.source.revision_sha, None)
if reasons:
if selective_testing_policy is SelectiveTestingPolicy.disabled:
reasons = ["Selective testing was requested but not done because:"] + [' ' + m for m in reasons]
build_message = '\n'.join(reasons)
else:
# NOTE: for diff builds, it makes sense to just do selective testing,
# since it will never become a parent build and will never be used to
# calculate revision results.
selective_testing_policy = SelectiveTestingPolicy.enabled
collection_id = uuid.uuid4()
new_build = create_build(
project=build.project,
collection_id=collection_id,
label=build.label,
target=build.target,
message=build.message,
author=build.author,
source=build.source,
cause=Cause.retry,
selective_testing_policy=selective_testing_policy,
)
if build_message:
message = BuildMessage(
build=new_build,
text=build_message,
)
db.session.add(message)
db.session.commit()
return '', 302, {'Location': '/api/0/builds/{0}/'.format(new_build.id.hex)}
def post(self):
"""
Create a new commit or diff build. The API roughly goes like this:
1. Identify the project(s) to build for. This can be done by specifying
``project``, ``repository``, or ``repository[callsign]``. If a repository is
specified somehow, then all projects for that repository are considered
for building.
2. Using the ``sha``, find the appropriate revision object. This may
involve updating the repo.
3. If ``patch`` is given, then apply the patch and mark this as a diff build.
Otherwise, this is a commit build.
4. If ``snapshot_id`` is given, verify that the snapshot can be used by all
projects.
5. If provided, apply project_whitelist, filtering out projects not in
this whitelist.
6. Based on the flag ``apply_project_files_trigger`` (see comment on the argument
itself for default values), decide whether or not to filter out projects
by file blacklist and whitelist.
7. Attach metadata and create/ensure existence of a build for each project,
depending on the flag ``ensure_only``.
NOTE: In ensure-only mode, the collection_ids of the returned builds are
not necessarily identical, as we give new builds new collection IDs
and preserve the existing builds' collection IDs.
NOTE: If ``patch`` is specified ``sha`` is assumed to be the original
base revision to apply the patch.
Not relevant until we fix TODO: ``sha`` is **not** guaranteed to be the rev
used to apply the patch. See ``find_green_parent_sha`` for the logic of
identifying the correct revision.
"""
args = self.parser.parse_args()
if args.patch_file and args.ensure_only:
return error("Ensure-only mode does not work with a diff build yet.",
problems=["patch", "ensure_only"])
if not (args.project or args.repository or args['repository[phabricator.callsign]']):
return error("Project or repository must be specified",
problems=["project", "repository", "repository[phabricator.callsign]"])
# read arguments
if args.patch_data:
try:
patch_data = json.loads(args.patch_data)
except Exception:
return error("Invalid patch data (must be JSON dict)",
problems=["patch[data]"])
if not isinstance(patch_data, dict):
return error("Invalid patch data (must be JSON dict)",
problems=["patch[data]"])
else:
patch_data = None
# 1. identify project(s)
projects, repository = try_get_projects_and_repository(args)
if not projects:
return error("Unable to find project(s).")
# read arguments
label = args.label
author = args.author
message = args.message
tag = args.tag
snapshot_id = args.snapshot_id
no_snapshot = args.no_snapshot
cause = Cause[args.cause]
if no_snapshot and snapshot_id:
return error("Cannot specify snapshot with no_snapshot option")
if not tag and args.patch_file:
tag = 'patch'
# 2. validate snapshot
if snapshot_id:
snapshot = Snapshot.query.get(snapshot_id)
if not snapshot:
return error("Unable to find snapshot.")
if snapshot.status != SnapshotStatus.active:
return error("Snapshot is in an invalid state: %s" % snapshot.status)
for project in projects:
plans = get_build_plans(project)
for plan in plans:
plan_options = plan.get_item_options()
allow_snapshot = '1' == plan_options.get('snapshot.allow', '1') or plan.snapshot_plan
if allow_snapshot and not SnapshotImage.get(plan, snapshot_id):
# We want to create a build using a specific snapshot but no image
# was found for this plan so fail.
return error("Snapshot cannot be applied to %s's %s" % (project.slug, plan.label))
# 3. find revision
try:
revision = identify_revision(repository, args.sha)
except MissingRevision:
# if the default fails, we absolutely can't continue and the
# client should send a valid revision
return error("Unable to find commit %s in %s." % (args.sha, repository.url),
problems=['sha', 'repository'])
# get default values for arguments
if revision:
if not author:
author = revision.author
if not label:
label = revision.subject
# only default the message if its absolutely not set
if message is None:
message = revision.message
sha = revision.sha
else:
sha = args.sha
if not args.target:
target = sha[:12]
else:
target = args.target[:128]
if not label:
if message:
label = message.splitlines()[0]
if not label:
label = 'A homeless build'
label = label[:128]
# 4. Check for patch
if args.patch_file:
fp = StringIO()
for line in args.patch_file:
fp.write(line)
patch_file = fp
else:
patch_file = None
if patch_file:
patch = Patch(
repository=repository,
parent_revision_sha=sha,
diff=patch_file.getvalue(),
)
db.session.add(patch)
else:
patch = None
project_options = ProjectOptionsHelper.get_options(projects, ['build.file-whitelist'])
# mark as commit or diff build
if not patch:
is_commit_build = True
else:
is_commit_build = False
apply_project_files_trigger = args.apply_project_files_trigger
if apply_project_files_trigger is None:
apply_project_files_trigger = args.apply_file_whitelist
if apply_project_files_trigger is None:
if is_commit_build:
apply_project_files_trigger = False
else:
apply_project_files_trigger = True
if apply_project_files_trigger:
if patch:
diff_parser = DiffParser(patch.diff)
files_changed = diff_parser.get_changed_files()
elif revision:
try:
files_changed = _get_revision_changed_files(repository, revision)
except MissingRevision:
return error("Unable to find commit %s in %s." % (args.sha, repository.url),
problems=['sha', 'repository'])
else:
# the only way that revision can be null is if this repo does not have a vcs backend
logging.warning('Revision and patch are both None for sha %s. This is because the repo %s does not have a VCS backend.', sha, repository.url)
files_changed = None
else:
# we won't be applying file whitelist, so there is no need to get the list of changed files.
files_changed = None
collection_id = uuid.uuid4()
builds = []
for project in projects:
plan_list = get_build_plans(project)
if not plan_list:
logging.warning('No plans defined for project %s', project.slug)
continue
# 5. apply project whitelist as appropriate
if args.project_whitelist is not None and project.slug not in args.project_whitelist:
logging.info('Project %s is not in the supplied whitelist', project.slug)
continue
forced_sha = sha
# TODO(dcramer): find_green_parent_sha needs to take branch
# into account
# if patch_file:
# forced_sha = find_green_parent_sha(
# project=project,
# sha=sha,
# )
# 6. apply file whitelist as appropriate
diff = None
if patch is not None:
diff = patch.diff
if (
apply_project_files_trigger and
files_changed is not None and
not files_changed_should_trigger_project(
files_changed, project, project_options[project.id], sha, diff)
):
logging.info('Changed files do not trigger build for project %s', project.slug)
continue
# 7. create/ensure build
build_message = None
selective_testing_policy = SelectiveTestingPolicy.disabled
if args.selective_testing and project_lib.contains_active_autogenerated_plan(project):
if is_commit_build:
selective_testing_policy, reasons = get_selective_testing_policy(project, sha, diff)
if reasons:
if selective_testing_policy is SelectiveTestingPolicy.disabled:
reasons = ["Selective testing was requested but not done because:"] + [' ' + m for m in reasons]
build_message = '\n'.join(reasons)
else:
# NOTE: for diff builds, it makes sense to just do selective testing,
# since it will never become a parent build and will never be used to
# calculate revision results.
selective_testing_policy = SelectiveTestingPolicy.enabled
if args.ensure_only:
potentials = list(Build.query.filter(
Build.project_id == project.id,
Build.source.has(revision_sha=sha, patch=patch),
).order_by(
Build.date_created.desc() # newest first
).limit(1))
if len(potentials) == 0:
builds.append(create_build(
project=project,
collection_id=collection_id,
sha=forced_sha,
target=target,
label=label,
message=message,
author=author,
patch=patch,
source_data=patch_data,
tag=tag,
cause=cause,
snapshot_id=snapshot_id,
no_snapshot=no_snapshot,
selective_testing_policy=selective_testing_policy,
))
else:
builds.append(potentials[0])
else:
builds.append(create_build(
project=project,
collection_id=collection_id,
sha=forced_sha,
target=target,
label=label,
message=message,
author=author,
patch=patch,
source_data=patch_data,
tag=tag,
cause=cause,
snapshot_id=snapshot_id,
no_snapshot=no_snapshot,
selective_testing_policy=selective_testing_policy,
))
if build_message:
message = BuildMessage(
build=builds[-1],
text=build_message,
)
db.session.add(message)
db.session.commit()
return self.respond(builds)
def post(self):
"""
Create a new commit or diff build. The API roughly goes like this:
1. Identify the project(s) to build for. This can be done by specifying
``project``, ``repository``, or ``repository[callsign]``. If a repository is
specified somehow, then all projects for that repository are considered
for building.
2. Using the ``sha``, find the appropriate revision object. This may
involve updating the repo.
3. If ``patch`` is given, then apply the patch and mark this as a diff build.
Otherwise, this is a commit build.
4. If ``snapshot_id`` is given, verify that the snapshot can be used by all
projects.
5. If provided, apply project_whitelist, filtering out projects not in
this whitelist.
6. Based on the flag ``apply_project_files_trigger`` (see comment on the argument
itself for default values), decide whether or not to filter out projects
by file blacklist and whitelist.
7. Attach metadata and create/ensure existence of a build for each project,
depending on the flag ``ensure_only``.
NOTE: In ensure-only mode, the collection_ids of the returned builds are
not necessarily identical, as we give new builds new collection IDs
and preserve the existing builds' collection IDs.
NOTE: If ``patch`` is specified ``sha`` is assumed to be the original
base revision to apply the patch.
Not relevant until we fix TODO: ``sha`` is **not** guaranteed to be the rev
used to apply the patch. See ``find_green_parent_sha`` for the logic of
identifying the correct revision.
"""
args = self.parser.parse_args()
if args.patch_file and args.ensure_only:
return error(
"Ensure-only mode does not work with a diff build yet.",
problems=["patch", "ensure_only"])
if not (args.project or args.repository
or args['repository[phabricator.callsign]']):
return error("Project or repository must be specified",
problems=[
"project", "repository",
"repository[phabricator.callsign]"
])
# read arguments
if args.patch_data:
try:
patch_data = json.loads(args.patch_data)
except Exception:
return error("Invalid patch data (must be JSON dict)",
problems=["patch[data]"])
if not isinstance(patch_data, dict):
return error("Invalid patch data (must be JSON dict)",
problems=["patch[data]"])
else:
patch_data = None
# 1. identify project(s)
projects, repository = try_get_projects_and_repository(args)
if not projects:
return error("Unable to find project(s).")
# read arguments
label = args.label
author = args.author
message = args.message
tag = args.tag
snapshot_id = args.snapshot_id
no_snapshot = args.no_snapshot
cause = Cause[args.cause]
if no_snapshot and snapshot_id:
return error("Cannot specify snapshot with no_snapshot option")
if not tag and args.patch_file:
tag = 'patch'
# 2. validate snapshot
if snapshot_id:
snapshot = Snapshot.query.get(snapshot_id)
if not snapshot:
return error("Unable to find snapshot.")
if snapshot.status != SnapshotStatus.active:
return error("Snapshot is in an invalid state: %s" %
snapshot.status)
for project in projects:
plans = get_build_plans(project)
for plan in plans:
plan_options = plan.get_item_options()
allow_snapshot = '1' == plan_options.get(
'snapshot.allow', '1') or plan.snapshot_plan
if allow_snapshot and not SnapshotImage.get(
plan, snapshot_id):
# We want to create a build using a specific snapshot but no image
# was found for this plan so fail.
return error("Snapshot cannot be applied to %s's %s" %
(project.slug, plan.label))
# 3. find revision
try:
revision = identify_revision(repository, args.sha)
except MissingRevision:
# if the default fails, we absolutely can't continue and the
# client should send a valid revision
return error("Unable to find commit %s in %s." %
(args.sha, repository.url),
problems=['sha', 'repository'])
# get default values for arguments
if revision:
if not author:
author = revision.author
if not label:
label = revision.subject
# only default the message if its absolutely not set
if message is None:
message = revision.message
sha = revision.sha
else:
sha = args.sha
if not args.target:
target = sha[:12]
else:
target = args.target[:128]
if not label:
if message:
label = message.splitlines()[0]
if not label:
label = 'A homeless build'
label = label[:128]
# 4. Check for patch
if args.patch_file:
fp = StringIO()
for line in args.patch_file:
fp.write(line)
patch_file = fp
else:
patch_file = None
if patch_file:
patch = Patch(
repository=repository,
parent_revision_sha=sha,
diff=patch_file.getvalue(),
)
db.session.add(patch)
else:
patch = None
project_options = ProjectOptionsHelper.get_options(
projects, ['build.file-whitelist'])
# mark as commit or diff build
if not patch:
is_commit_build = True
else:
is_commit_build = False
apply_project_files_trigger = args.apply_project_files_trigger
if apply_project_files_trigger is None:
apply_project_files_trigger = args.apply_file_whitelist
if apply_project_files_trigger is None:
if is_commit_build:
apply_project_files_trigger = False
else:
apply_project_files_trigger = True
if apply_project_files_trigger:
if patch:
diff_parser = DiffParser(patch.diff)
files_changed = diff_parser.get_changed_files()
elif revision:
try:
files_changed = _get_revision_changed_files(
repository, revision)
except MissingRevision:
return error("Unable to find commit %s in %s." %
(args.sha, repository.url),
problems=['sha', 'repository'])
else:
# the only way that revision can be null is if this repo does not have a vcs backend
logging.warning(
'Revision and patch are both None for sha %s. This is because the repo %s does not have a VCS backend.',
sha, repository.url)
files_changed = None
else:
# we won't be applying file whitelist, so there is no need to get the list of changed files.
files_changed = None
collection_id = uuid.uuid4()
builds = []
for project in projects:
plan_list = get_build_plans(project)
if not plan_list:
logging.warning('No plans defined for project %s',
project.slug)
continue
# 5. apply project whitelist as appropriate
if args.project_whitelist is not None and project.slug not in args.project_whitelist:
logging.info('Project %s is not in the supplied whitelist',
project.slug)
continue
forced_sha = sha
# TODO(dcramer): find_green_parent_sha needs to take branch
# into account
# if patch_file:
# forced_sha = find_green_parent_sha(
# project=project,
# sha=sha,
# )
# 6. apply file whitelist as appropriate
diff = None
if patch is not None:
diff = patch.diff
if (apply_project_files_trigger and files_changed is not None
and not files_changed_should_trigger_project(
files_changed, project, project_options[project.id],
sha, diff)):
logging.info(
'Changed files do not trigger build for project %s',
project.slug)
continue
# 7. create/ensure build
build_message = None
selective_testing_policy = SelectiveTestingPolicy.disabled
if args.selective_testing and project_lib.contains_active_autogenerated_plan(
project):
if is_commit_build:
selective_testing_policy, reasons = get_selective_testing_policy(
project, sha, diff)
if reasons:
if selective_testing_policy is SelectiveTestingPolicy.disabled:
reasons = [
"Selective testing was requested but not done because:"
] + [' ' + m for m in reasons]
build_message = '\n'.join(reasons)
else:
# NOTE: for diff builds, it makes sense to just do selective testing,
# since it will never become a parent build and will never be used to
# calculate revision results.
selective_testing_policy = SelectiveTestingPolicy.enabled
if args.ensure_only:
potentials = list(
Build.query.filter(
Build.project_id == project.id,
Build.source.has(revision_sha=sha, patch=patch),
).order_by(Build.date_created.desc() # newest first
).limit(1))
if len(potentials) == 0:
builds.append(
create_build(
project=project,
collection_id=collection_id,
sha=forced_sha,
target=target,
label=label,
message=message,
author=author,
patch=patch,
source_data=patch_data,
tag=tag,
cause=cause,
snapshot_id=snapshot_id,
no_snapshot=no_snapshot,
selective_testing_policy=selective_testing_policy,
))
else:
builds.append(potentials[0])
else:
builds.append(
create_build(
project=project,
collection_id=collection_id,
sha=forced_sha,
target=target,
label=label,
message=message,
author=author,
patch=patch,
source_data=patch_data,
tag=tag,
cause=cause,
snapshot_id=snapshot_id,
no_snapshot=no_snapshot,
selective_testing_policy=selective_testing_policy,
))
if build_message:
message = BuildMessage(
build=builds[-1],
text=build_message,
)
db.session.add(message)
db.session.commit()
return self.respond(builds)