def test_tenant_queries_repo_with_tenant(default_repo):
auth.set_current_tenant(auth.Tenant(access={default_repo.id: Permission.read}))
assert list(Repository.query.all()) == [default_repo]
def test_tenant_allows_public_repos(default_repo):
auth.set_current_tenant(auth.Tenant())
repo = factories.RepositoryFactory(name="public", public=True)
assert list(Repository.query.all()) == [repo]
def test_tenant_limits_to_access(default_repo):
auth.set_current_tenant(
auth.Tenant(access={default_repo.id: Permission.read}))
build = factories.BuildFactory(repository=default_repo)
assert list(Build.query.all()) == [build]
def sync_repo(repo_id, max_log_passes=10, force=False, time_limit=300):
auth.set_current_tenant(auth.Tenant(access={repo_id: Permission.admin}))
repo = Repository.query.get(repo_id)
if not repo:
current_app.logger.error("Repository %s not found", repo_id)
return
if (not force and repo.last_update_attempt and repo.last_update_attempt >
(timezone.now() - current_app.config["REPO_SYNC_INTERVAL"])):
current_app.logger.warning(
"Repository %s was synced recently, refusing to sync", repo.id)
return
try:
vcs = repo.get_vcs()
except UnknownRepositoryBackend:
current_app.logger.warning("Repository %s has no VCS backend set",
repo.id)
return
if repo.status != RepositoryStatus.active:
current_app.logger.info("Repository %s is not active", repo.id)
return
Repository.query.filter(Repository.id == repo.id).update(
{"last_update_attempt": timezone.now()})
db.session.commit()
try:
if vcs.exists():
vcs.update(allow_cleanup=True)
else:
vcs.clone()
except InvalidPublicKey:
# TODO(dcramer): this is a quick short-circuit for repo syncing, which will
# at least prevent workers from endlessly querying repos which were revoked.
# Ideally this would be implemented in a larger number of places (maybe via
# a context manager?)
repo.status = RepositoryStatus.inactive
ItemOption.query.filter(
ItemOption.item_id == repo.id,
ItemOption.name == "auth.private-key").delete()
db.session.add(repo)
db.session.commit()
return
# TODO(dcramer): this doesn't collect commits in non-default branches
might_have_more = True
parent = None
while might_have_more and max_log_passes:
might_have_more = False
for commit in vcs.log(parent=parent):
revision, created = commit.save(repo)
db.session.commit()
if not created:
break
current_app.logger.info("Created revision {}".format(repo.id))
might_have_more = True
parent = commit.sha
max_log_passes -= 1
Repository.query.filter(Repository.id == repo.id).update(
{"last_update": datetime.now(timezone.utc)})
db.session.commit()
# is there more data to sync?
return might_have_more
def test_tenant_allows_public_repos(default_repo):
auth.set_current_tenant(
auth.Tenant(access={default_repo.id: Permission.read}))
repo = factories.RepositoryFactory(name="public", public=True)
build = factories.BuildFactory(repository=repo)
assert list(Build.query.all()) == [build]
def test_tenant_allows_public_repos_with_acess(default_repo):
auth.set_current_tenant(auth.Tenant())
repo = factories.RepositoryFactory(name="public", public=True)
build = factories.BuildFactory(repository=repo)
assert list(Build.query.all()) == [build]
def aggregate_build_stats(build_id: UUID):
"""
Updates various denormalized / aggregate attributes on Build per its
jobs. These attributes include start and completion dates, as well as
the status and result.
"""
# now we pull in the entirety of the build's data to aggregate state upward
lock_key = 'build:{build_id}'.format(build_id=build_id, )
with redis.lock(lock_key):
build = Build.query.unrestricted_unsafe().with_for_update(
nowait=True).get(build_id)
if not build:
raise ValueError(
'Unable to find build with id = {}'.format(build_id))
auth.set_current_tenant(
auth.Tenant(repository_ids=[build.repository_id]))
record_coverage_stats(build.id)
job_list = Job.query.filter(Job.build_id == build.id)
was_finished = build.status == Status.finished
is_finished = all(p.status == Status.finished for p in job_list)
# ensure build's dates are reflective of jobs
build.date_started = safe_agg(min,
(j.date_started
for j in job_list if j.date_started))
if is_finished:
build.date_finished = safe_agg(
max, (j.date_finished for j in job_list if j.date_finished))
else:
build.date_finished = None
# if theres any failure, the build failed
if any(j.result is Result.failed for j in job_list
if not j.allow_failure):
build.result = Result.failed
# else, if we're finished, we can aggregate from results
elif is_finished:
if not job_list:
build.result = Result.errored
elif not any(j for j in job_list if not j.allow_failure):
build.result = Result.passed
else:
build.result = aggregate_result(
(j.result for j in job_list if not j.allow_failure))
# we should never get here as long we've got jobs and correct data
else:
build.result = Result.unknown
if is_finished:
build.status = Status.finished
else:
# ensure we dont set the status to finished unless it actually is
new_status = aggregate_status((j.status for j in job_list))
if build.status != new_status:
build.status = new_status
db.session.add(build)
db.session.commit()
# we dont bother aggregating stats unless we're finished
if build.status == Status.finished and not was_finished:
for stat in AGGREGATED_BUILD_STATS:
aggregate_stat_for_build(build, stat)
db.session.commit()
send_build_notifications.delay(build_id=build.id)
def test_full(default_job):
auth.set_current_tenant(
auth.Tenant(repository_ids=[default_job.repository_id]))
results = [
ZeusTestResult(job=default_job,
name='test_bar',
package='tests.changes.handlers.test_xunit',
result=Result.failed,
message='collection failed',
duration=156,
artifacts=[{
'name':
'artifact_name',
'type':
'text',
'base64':
b64encode('sample content'.encode('utf-8'))
}]),
ZeusTestResult(
job=default_job,
name='test_foo',
package='tests.changes.handlers.test_coverage',
result=Result.passed,
message='foobar failed',
duration=12,
),
]
manager = ZeusTestResultManager(default_job)
manager.save(results)
testcase_list = sorted(ZeusTestCase.query.all(), key=lambda x: x.name)
assert len(testcase_list) == 2
for test in testcase_list:
assert test.job_id == default_job.id
assert testcase_list[
0].name == 'tests.changes.handlers.test_coverage.test_foo'
assert testcase_list[0].result == Result.passed
assert testcase_list[0].message == 'foobar failed'
assert testcase_list[0].duration == 12
assert testcase_list[
1].name == 'tests.changes.handlers.test_xunit.test_bar'
assert testcase_list[1].result == Result.failed
assert testcase_list[1].message == 'collection failed'
assert testcase_list[1].duration == 156
artifacts = list(Artifact.query.unrestricted_unsafe().filter(
Artifact.testcase_id == testcase_list[1].id))
assert len(artifacts) == 1
assert artifacts[0].file.get_file().read() == b'sample content'
teststat = ItemStat.query.filter(
ItemStat.name == 'tests.count',
ItemStat.item_id == default_job.id,
)[0]
assert teststat.value == 2
teststat = ItemStat.query.filter(
ItemStat.name == 'tests.failures',
ItemStat.item_id == default_job.id,
)[0]
assert teststat.value == 1
teststat = ItemStat.query.filter(
ItemStat.name == 'tests.duration',
ItemStat.item_id == default_job.id,
)[0]
assert teststat.value == 168
def process_travis_webhook(hook_id: str, payload: dict, timestamp_ms: int):
# TODO(dcramer): we want to utilize timestamp_ms to act as a version and
# ensure we dont process older updates after newer updates are already present
with nplusone.ignore("eager_load"):
hook = (Hook.query.unrestricted_unsafe().options(
joinedload("repository")).get(hook_id))
auth.set_current_tenant(
auth.Tenant(access={hook.repository_id: Permission.admin}))
data = {"ref": payload["commit"], "url": payload["build_url"]}
domain = urlparse(data["url"]).netloc
try:
if payload["pull_request"]:
data["label"] = "PR #{} - {}".format(
payload["pull_request_number"], payload["pull_request_title"])
upsert_change_request(
repository=hook.repository,
provider="github",
external_id=str(payload["pull_request_number"]),
data={
"parent_revision_sha": payload["base_commit"],
"head_revision_sha": payload["head_commit"],
"message": payload["pull_request_title"],
},
)
response = upsert_build(hook=hook,
external_id=str(payload["id"]),
data=data)
build = Build.query.get(response.json()["id"])
for job_payload in payload["matrix"]:
upsert_job(
build=build,
hook=hook,
external_id=str(job_payload["id"]),
data={
"status": ("finished" if job_payload["status"] is not None
else "in_progress"),
"result":
get_result(job_payload["state"]),
"allow_failure":
bool(job_payload["allow_failure"]),
"label":
get_job_label(job_payload),
"url":
"https://{domain}/{owner}/{name}/jobs/{job_id}".format(
domain=domain,
owner=payload["repository"]["owner_name"],
name=payload["repository"]["name"],
job_id=job_payload["id"],
),
"started_at":
(dateutil.parser.parse(job_payload["started_at"])
if job_payload["started_at"] else None),
"finished_at":
(dateutil.parser.parse(job_payload["finished_at"])
if job_payload["finished_at"] else None),
},
)
except ApiError:
current_app.logger.error("travis.webhook-unexpected-error",
exc_info=True)
raise
def test_full(default_job):
auth.set_current_tenant(
auth.Tenant(access={default_job.repository_id: Permission.read}))
results = [
ZeusTestResult(
job=default_job,
name="test_bar",
package="tests.changes.handlers.test_xunit",
result=Result.failed,
message="collection failed",
duration=156,
artifacts=[{
"name": "artifact_name",
"type": "text",
"base64": b64encode("sample content".encode("utf-8")),
}],
),
ZeusTestResult(
job=default_job,
name="test_foo",
package="tests.changes.handlers.test_coverage",
result=Result.passed,
message="foobar failed",
duration=12,
),
]
manager = ZeusTestResultManager(default_job)
manager.save(results)
testcase_list = sorted(ZeusTestCase.query.all(), key=lambda x: x.name)
assert len(testcase_list) == 2
for test in testcase_list:
assert test.job_id == default_job.id
assert testcase_list[
0].name == "tests.changes.handlers.test_coverage.test_foo"
assert testcase_list[0].result == Result.passed
assert testcase_list[0].message == "foobar failed"
assert testcase_list[0].duration == 12
assert testcase_list[
1].name == "tests.changes.handlers.test_xunit.test_bar"
assert testcase_list[1].result == Result.failed
assert testcase_list[1].message == "collection failed"
assert testcase_list[1].duration == 156
artifacts = list(Artifact.query.unrestricted_unsafe().filter(
Artifact.testcase_id == testcase_list[1].id))
assert len(artifacts) == 1
assert artifacts[0].file.get_file().read() == b"sample content"
teststat = ItemStat.query.filter(ItemStat.name == "tests.count",
ItemStat.item_id == default_job.id)[0]
assert teststat.value == 2
teststat = ItemStat.query.filter(ItemStat.name == "tests.failures",
ItemStat.item_id == default_job.id)[0]
assert teststat.value == 1
teststat = ItemStat.query.filter(ItemStat.name == "tests.duration",
ItemStat.item_id == default_job.id)[0]
assert teststat.value == 168
def test_failure_origin(default_repo):
auth.set_current_tenant(
auth.Tenant(access={default_repo.id: Permission.read}))
new_revision = factories.RevisionFactory(repository=default_repo)
new_source = factories.SourceFactory(revision=new_revision)
new_build = factories.BuildFactory(source=new_source, failed=True)
new_job = factories.JobFactory(build=new_build, failed=True)
new_testcase = factories.TestCaseFactory(job=new_job, failed=True)
new_testcase2 = factories.TestCaseFactory(job=new_job, passed=True)
old_revision = factories.RevisionFactory(
repository=default_repo,
date_created=new_revision.date_created - timedelta(hours=1),
)
old_source = factories.SourceFactory(revision=old_revision,
date_created=new_source.date_created -
timedelta(hours=1))
old_build = factories.BuildFactory(
source=old_source,
failed=True,
date_created=new_build.date_created - timedelta(hours=1),
)
old_job = factories.JobFactory(
build=old_build,
failed=True,
date_created=new_job.date_created - timedelta(hours=1),
)
factories.TestCaseFactory(job=old_job, failed=True, name=new_testcase.name)
oldold_revision = factories.RevisionFactory(
repository=default_repo,
date_created=old_revision.date_created - timedelta(hours=1),
)
oldold_source = factories.SourceFactory(
revision=oldold_revision,
date_created=old_source.date_created - timedelta(hours=1),
)
factories.BuildFactory(
source=oldold_source,
passed=True,
date_created=old_build.date_created - timedelta(hours=1),
)
schema = AggregateTestCaseSummarySchema(many=True,
strict=True,
context={"build": new_build})
result = schema.dump([
AggregateTestCase(
hash=new_testcase.hash,
name=new_testcase.name,
runs=[[
str(new_testcase.id),
str(new_testcase.job_id),
new_testcase.duration,
int(new_testcase.result),
]],
),
AggregateTestCase(
hash=new_testcase2.hash,
name=new_testcase2.name,
runs=[[
str(new_testcase2.id),
str(new_testcase2.job_id),
new_testcase2.duration,
int(new_testcase2.result),
]],
),
]).data
assert len(result) == 2
assert result[0]["hash"] == new_testcase.hash
assert result[0]["name"] == new_testcase.name
assert result[0]["origin_build"]["id"] == str(old_build.id)
assert result[1]["hash"] == new_testcase2.hash
assert result[1]["name"] == new_testcase2.name
assert result[1]["origin_build"] is None
def default_repo_write_tenant(default_repo):
auth.set_current_tenant(
auth.Tenant(access={default_repo.id: Permission.write}))
def default_repo_tenant(default_repo):
auth.set_current_tenant(
auth.Tenant(access={default_repo.id: Permission.read}))
