def default_job(default_build):
return factories.JobFactory(
build=default_build,
date_started=timezone.now() - timedelta(minutes=6),
date_finished=timezone.now(),
passed=True,
)
def test_repo_revision_list(client, db_session, default_login, default_user,
git_repo_config):
repo = factories.RepositoryFactory.create(
backend=RepositoryBackend.git,
provider=RepositoryProvider.github,
url=git_repo_config.url,
)
db_session.add(RepositoryAccess(user=default_user, repository=repo))
db_session.flush()
revision = factories.RevisionFactory.create(sha=git_repo_config.commits[0],
repository=repo)
source = factories.SourceFactory.create(revision=revision)
factories.BuildFactory.create(source=source,
date_created=timezone.now() -
timedelta(minutes=1))
build = factories.BuildFactory.create(source=source,
date_created=timezone.now())
resp = client.get("/api/repos/{}/revisions".format(repo.get_full_name()))
assert resp.status_code == 200
data = resp.json()
assert len(data) == 2
assert data[0]["sha"] == git_repo_config.commits[0]
assert data[0]["latest_build"]["id"] == str(build.id)
assert data[1]["sha"] == git_repo_config.commits[1]
assert data[1]["latest_build"] is None
def cleanup_builds():
# find any artifacts which seemingly are stuck (not enqueued)
queryset = Artifact.query.unrestricted_unsafe().filter(
Artifact.status != Status.finished,
Artifact.date_updated < timezone.now() - timedelta(minutes=15),
)
for result in queryset:
Artifact.query.unrestricted_unsafe().filter(
Artifact.status != Status.finished,
Artifact.id == result.id).update({"date_updated": timezone.now()})
db.session.flush()
process_artifact(artifact_id=result.id)
# first we timeout any jobs which have been sitting for far too long
Job.query.unrestricted_unsafe().filter(
Job.status != Status.finished,
Job.date_updated < timezone.now() - timedelta(hours=1),
).update({
"status": Status.finished,
"result": Result.errored,
"date_updated": timezone.now(),
"date_finished": timezone.now(),
})
db.session.commit()
queryset = Build.query.unrestricted_unsafe().filter(
Build.status != Status.finished,
~Job.query.filter(Job.build_id == Build.id,
Job.status != Status.finished).exists(),
)
for build in queryset:
aggregate_build_stats(build_id=build.id)
def default_build(default_revision):
return factories.BuildFactory(
revision=default_revision,
authors=default_revision.authors,
date_started=timezone.now() - timedelta(minutes=6),
date_finished=timezone.now(),
passed=True,
)
def process_artifact(artifact_id, manager=None, force=False, **kwargs):
artifact = Artifact.query.unrestricted_unsafe().get(artifact_id)
if artifact is None:
current_app.logger.error("Artifact %s not found", artifact_id)
return
if artifact.status == Status.finished and not force:
current_app.logger.info(
"Skipping artifact processing (%s) - already marked as finished",
artifact_id,
)
return
artifact.status = Status.in_progress
artifact.date_started = timezone.now()
db.session.add(artifact)
db.session.flush()
auth.set_current_tenant(
auth.RepositoryTenant(repository_id=artifact.repository_id))
job = Job.query.get(artifact.job_id)
if job.result == Result.aborted:
current_app.logger.info(
"Skipping artifact processing (%s) - Job aborted", artifact_id)
artifact.status = Status.finished
db.session.add(artifact)
db.session.commit()
return
if artifact.file:
if manager is None:
manager = default_manager
try:
with db.session.begin_nested():
manager.process(artifact)
except Exception:
current_app.logger.exception(
"Unrecoverable exception processing artifact %s: %s",
artifact.job_id,
artifact,
)
else:
current_app.logger.info(
"Skipping artifact processing (%s) due to missing file",
artifact_id)
artifact.status = Status.finished
artifact.date_finished = timezone.now()
db.session.add(artifact)
db.session.commit()
# we always aggregate results to avoid locking here
aggregate_build_stats_for_job.delay(job_id=job.id)
def track_user(response):
from zeus import auth
from zeus.utils import timezone
user = auth.get_current_user(fetch=False)
if user and user.date_active < timezone.now() - timedelta(minutes=5):
user.date_active = timezone.now()
db.session.add(user)
db.session.commit()
return response
def get(self):
"""
Return various details about the installation.
"""
one_day_ago = timezone.now() - timedelta(days=1)
thirty_days_ago = timezone.now() - timedelta(days=30)
return {
"process": {
"id": str(metrics.guid),
"uptime": metrics.uptime,
"connections": metrics.connections.value,
"hits": {
"5m": metrics.hits.count(300),
"15m": metrics.hits.count(900),
"1h": metrics.hits.count(3600),
},
},
"config": {
"debug": current_app.debug,
"environment": current_app.config["SENTRY_ENVIRONMENT"],
"release": current_app.config["SENTRY_RELEASE"],
"pubsubEndpoint": current_app.config["PUBSUB_ENDPOINT"],
},
"stats": {
"builds": {
"24h": (Build.query.unrestricted_unsafe().filter(
Build.date_created > one_day_ago).count()),
"30d": (Build.query.unrestricted_unsafe().filter(
Build.date_created > thirty_days_ago).count()),
},
"repos": {
"24h":
(db.session.query(Build.repository_id).distinct().filter(
Build.date_created > one_day_ago).distinct().count()),
"30d":
(db.session.query(Build.repository_id).distinct().filter(
Build.date_created > thirty_days_ago).distinct().count(
)),
},
"users": {
"24h":
User.query.filter(User.date_active > one_day_ago).count(),
"30d":
User.query.filter(
User.date_active > thirty_days_ago).count(),
},
},
}
def sync_all_repos():
queryset = Repository.query.unrestricted_unsafe().filter(
Repository.status == RepositoryStatus.active,
or_(
Repository.last_update_attempt <
(timezone.now() - timedelta(minutes=5)),
Repository.last_update_attempt is None))
for repo in queryset:
sync_repo.delay(repo_id=repo.id, )
Repository.query.filter(Repository.id == repo.id, ).update({
'last_update_attempt':
timezone.now(),
})
db.session.commit()
def get_user_from_request() -> Optional[User]:
expire = session.get('expire')
if not expire:
return None
try:
expire = datetime.utcfromtimestamp(expire).replace(tzinfo=timezone.utc)
except Exception:
current_app.logger.exception('invalid session expirey')
del session['expire']
return None
if expire <= timezone.now():
current_app.logger.info('session expired')
del session['expire']
return None
try:
uid = session['uid']
except KeyError:
current_app.logger.error('missing uid session key', exc_info=True)
del session['expire']
return None
return User.query.get(uid)
def test_repo_revision_list(
client,
db_session,
default_login,
default_revision,
default_repo,
default_repo_access,
default_user,
mock_vcs_server,
):
mock_vcs_server.replace(
mock_vcs_server.GET,
"http://localhost:8070/stmt/log",
json={"log": [{
"sha": default_revision.sha
}]},
)
factories.BuildFactory.create(revision=default_revision,
date_created=timezone.now() -
timedelta(minutes=1))
factories.BuildFactory.create(revision=default_revision, passed=True)
resp = client.get("/api/repos/{}/revisions".format(
default_repo.get_full_name()))
assert resp.status_code == 200
data = resp.json()
assert len(data) == 1
assert data[0]["sha"] == default_revision.sha
assert data[0]["latest_build"]["status"] == "finished"
def cleanup_build_stats(task_limit=100):
# find any builds which should be marked as finished but aren't
queryset = (Build.query.unrestricted_unsafe().filter(
Build.status != Status.finished,
Build.date_started < timezone.now() - timedelta(minutes=15),
~Job.query.filter(Job.build_id == Build.id,
Job.status != Status.finished).exists(),
).limit(task_limit))
for build in queryset:
current_app.logger.warning("cleanup: aggregate_build_stats %s",
build.id)
try:
aggregate_build_stats(build_id=build.id)
except Exception:
current_app.logger.exception("cleanup: aggregate_build_stats %s",
build.id)
results = (Build.query.unrestricted_unsafe().filter(
Build.status != Status.finished,
Build.result != Result.errored).update({"status": Status.finished}))
if results:
current_app.logger.warning(
"cleanup: cleanup_build_stats [unfinished; errored] affected rows %s",
results,
)
db.session.commit()
def get_user_from_request() -> Optional[User]:
expire = session.get("expire")
if not expire:
return None
try:
expire = datetime.utcfromtimestamp(expire).replace(tzinfo=timezone.utc)
except Exception:
current_app.logger.exception("invalid session expirey")
del session["expire"]
return None
if expire <= timezone.now():
current_app.logger.info("session expired")
del session["expire"]
return None
try:
uid = session["uid"]
except KeyError:
current_app.logger.error("missing uid session key", exc_info=True)
del session["expire"]
return None
return User.query.get(uid)
def put(self, job: Job):
"""
Update a job.
"""
result = self.schema_from_request(job_schema, partial=True)
if result.errors:
return self.respond(result.errors, 403)
was_unfinished = job.status != Status.finished
for key, value in result.data.items():
if getattr(job, key) != value:
setattr(job, key, value)
if db.session.is_modified(job):
db.session.add(job)
if job.status == Status.finished and was_unfinished and not job.date_finished:
job.date_finished = timezone.now()
if job.status == Status.finished and has_unprocessed_artifacts(job.id):
job.status = Status.collecting_results
db.session.commit()
aggregate_build_stats_for_job.delay(job_id=job.id)
return self.respond_with_schema(job_schema, job)
def cleanup_pending_artifacts(task_limit=100):
queryset = (PendingArtifact.query.unrestricted_unsafe().filter(
PendingArtifact.date_created < timezone.now() -
timedelta(days=1)).limit(1000))
for result in queryset:
with db.session.begin_nested():
current_app.logger.warning(
"cleanup: process_pending_artifact %s [expired]", result.id)
if result.file:
result.file.delete()
db.session.delete(result)
db.session.commit()
# find any pending artifacts which seemingly are stuck (not enqueued)
queryset = (db.session.query(PendingArtifact.id).filter(
Build.external_id == PendingArtifact.external_build_id,
Build.provider == PendingArtifact.provider,
Build.repository_id == PendingArtifact.repository_id,
Build.status == Status.finished,
).limit(task_limit))
for result in queryset:
current_app.logger.warning(
"cleanup: process_pending_artifact %s [build_finished]", result.id)
try:
process_pending_artifact(pending_artifact_id=result.id)
except UnknownJob:
# do we just axe it?
pass
def test_merge_build_group_different_providers(client, default_login,
default_revision):
now = timezone.now()
later = now + timedelta(minutes=1)
build1 = factories.BuildFactory.create(
revision=default_revision,
provider="provider1",
date_started=now,
date_finished=now,
)
build2 = factories.BuildFactory.create(
revision=default_revision,
provider="provider2",
date_started=later,
date_finished=later,
)
merged_build = merge_build_group([build1, build2])
assert merged_build.ref == build1.ref
assert merged_build.revision_sha is build1.revision_sha
assert merged_build.label == build1.label
assert merged_build.original == [build1, build2]
assert merged_build.status == Status(
max(build1.status.value, build2.status.value))
assert merged_build.result == Result(
max(build1.result.value, build2.result.value))
assert merged_build.date_started == now
assert merged_build.date_finished == later
class RepositoryFactory(ModelFactory):
id = GUIDFactory()
owner_name = factory.Faker("word")
name = factory.Faker("word")
url = factory.LazyAttribute(
lambda o: "[email protected]:%s/%s.git" % (o.owner_name, o.name)
)
backend = models.RepositoryBackend.git
status = models.RepositoryStatus.active
provider = models.RepositoryProvider.github
external_id = factory.LazyAttribute(lambda o: "{}/{}".format(o.owner_name, o.name))
public = False
date_created = factory.LazyAttribute(lambda o: timezone.now())
class Meta:
model = models.Repository
class Params:
unknown = factory.Trait(backend=models.RepositoryBackend.unknown)
github = factory.Trait(
backend=models.RepositoryBackend.git,
provider=models.RepositoryProvider.github,
external_id=factory.LazyAttribute(
lambda o: "{}/{}".format(o.owner_name, o.name)
),
data=factory.LazyAttribute(
lambda o: {"full_name": "{}/{}".format(o.owner_name, o.name)}
),
)
class JobFactory(ModelFactory):
id = GUIDFactory()
build = factory.SubFactory('zeus.factories.BuildFactory')
build_id = factory.SelfAttribute('build.id')
project = factory.SelfAttribute('build.project')
project_id = factory.SelfAttribute('project.id')
organization = factory.SelfAttribute('project.organization')
organization_id = factory.SelfAttribute('organization.id')
label = factory.faker.Faker('sentence')
result = factory.Iterator([Result.failed, Result.passed])
status = factory.Iterator(
[Status.queued, Status.in_progress, Status.finished])
date_created = factory.LazyAttribute(
lambda o: timezone.now() - timedelta(minutes=30))
date_started = factory.LazyAttribute(lambda o: (faker.date_time_between(
o.date_created, o.date_created + timedelta(minutes=1)
) if o.status in [Status.finished, Status.in_progress] else None))
date_finished = factory.LazyAttribute(lambda o: faker.date_time_between(
o.date_started, o.date_started + timedelta(minutes=10)) if o.status ==
Status.finished else None)
class Meta:
model = models.Job
class Params:
queued = factory.Trait(result=Result.unknown, status=Status.queued)
in_progress = factory.Trait(result=Result.unknown,
status=Status.in_progress)
failed = factory.Trait(result=Result.failed, status=Status.finished)
passed = factory.Trait(result=Result.passed, status=Status.finished)
aborted = factory.Trait(result=Result.aborted, status=Status.finished)
class ChangeRequestFactory(ModelFactory):
id = GUIDFactory()
message = factory.faker.Faker("sentence")
author = factory.SubFactory(
"zeus.factories.AuthorFactory",
repository=factory.SelfAttribute("..repository"))
author_id = factory.SelfAttribute("author.id")
parent_revision = factory.SubFactory("zeus.factories.RevisionFactory")
parent_revision_sha = factory.SelfAttribute("parent_revision.sha")
repository = factory.SelfAttribute("parent_revision.repository")
repository_id = factory.SelfAttribute("parent_revision.repository_id")
date_created = factory.LazyAttribute(
lambda o: timezone.now() - timedelta(minutes=30))
class Meta:
model = models.ChangeRequest
class Params:
github = factory.Trait(
provider="github",
external_id=factory.LazyAttribute(
lambda o: str(randint(10000, 999999))),
head_revision_sha=factory.SelfAttribute("head_revision.sha"),
head_revision=factory.SubFactory("zeus.factories.RevisionFactory"),
url=factory.LazyAttribute(
lambda o: "https://github.com/{}/{}/issues/{}".format(
o.repository.owner_name, o.repository.name, o.external_id
)),
)
def login_user(user_id: str, session=session, current_datetime=None):
session["uid"] = str(user_id)
session["expire"] = int(
((current_datetime or timezone.now()) +
current_app.config["PERMANENT_SESSION_LIFETIME"]).strftime("%s"))
session.permanent = True
with sentry_sdk.configure_scope() as scope:
scope.user = {"id": str(user_id)}
class UserFactory(ModelFactory):
id = GUIDFactory()
email = factory.Faker("email")
date_created = factory.LazyAttribute(lambda o: timezone.now())
date_active = factory.LazyAttribute(lambda o: o.date_created)
class Meta:
model = models.User
def test_cleanup_pending_artifacts_old_file(mocker, db_session):
pending_artifact = factories.PendingArtifactFactory.create(
date_created=timezone.now() - timedelta(days=2))
cleanup_pending_artifacts()
assert not PendingArtifact.query.unrestricted_unsafe().get(
pending_artifact.id)
def put(self, job: Job):
"""
Update a job.
"""
result = self.schema_from_request(job_schema, partial=True)
if result.errors:
return self.respond(result.errors, 403)
prev_status = job.status
for key, value in result.data.items():
if getattr(job, key) != value:
setattr(job, key, value)
if db.session.is_modified(job):
if job.status == Status.queued:
job.date_started = None
job.result = Result.unknown
elif job.status == Status.in_progress and prev_status != Status.in_progress:
# TODO(dcramer): this is effectively 'restart' on a job, and we need to
# decide how Zeus should deal with it. We either could orphan/hide/remove the
# current job, or alternatively we would want to truncate all of its children
# which is fairly complex.
if not result.data.get("date_started"):
job.date_started = timezone.now()
if "result" not in result.data:
job.result = Result.unknown
if (
job.status == Status.finished
and prev_status != job.status
and not result.data.get("date_finished")
):
job.date_finished = timezone.now()
if not job.date_started:
job.date_started = job.date_created
elif job.status != Status.finished:
job.date_finished = None
if job.status == Status.finished and has_unprocessed_artifacts(job.id):
job.status = Status.collecting_results
db.session.add(job)
db.session.commit()
aggregate_build_stats_for_job.delay(job_id=job.id)
return self.respond_with_schema(job_schema, job)
def test_cleanup_artifacts_old_file(mocker, db_session):
artifact = factories.ArtifactFactory.create(status=Status.finished,
date_created=timezone.now() -
timedelta(days=45))
cleanup_artifacts()
assert artifact.status == Status.expired
assert not artifact.file
def default_login(client, default_user):
with client.session_transaction() as session:
# XXX(dcramer): could use auth.login_user here, but that makes most tests dependent
# on that one function
session['uid'] = default_user.id
session['expire'] = int(
(timezone.now() + timedelta(days=1)).strftime('%s'))
yield default_user
class JobFactory(ModelFactory):
id = GUIDFactory()
label = factory.faker.Faker("sentence")
build = factory.SubFactory(
"zeus.factories.BuildFactory",
status=factory.SelfAttribute("..status"),
result=factory.SelfAttribute("..result"),
date_created=factory.SelfAttribute("..date_created"),
date_started=factory.SelfAttribute("..date_started"),
date_finished=factory.SelfAttribute("..date_finished"),
)
build_id = factory.SelfAttribute("build.id")
repository = factory.SelfAttribute("build.repository")
repository_id = factory.SelfAttribute("repository.id")
label = factory.faker.Faker("sentence")
result = factory.Iterator([Result.failed, Result.passed])
status = factory.Iterator(
[Status.queued, Status.in_progress, Status.finished])
allow_failure = False
date_created = factory.LazyAttribute(
lambda o: timezone.now() - timedelta(minutes=30))
date_started = factory.LazyAttribute(lambda o: (faker.date_time_between(
o.date_created, o.date_created + timedelta(minutes=1)
) if o.status in [Status.finished, Status.in_progress] else None))
date_finished = factory.LazyAttribute(lambda o: faker.date_time_between(
o.date_started, o.date_started + timedelta(minutes=10)) if o.status ==
Status.finished else None)
date_updated = factory.LazyAttribute(
lambda o: o.date_finished or o.date_started or o.date_created)
class Meta:
model = models.Job
class Params:
queued = factory.Trait(
result=Result.unknown,
status=Status.queued,
date_started=None,
date_finished=None,
)
in_progress = factory.Trait(result=Result.unknown,
status=Status.in_progress,
date_finished=None)
finished = factory.Trait(status=Status.finished)
failed = factory.Trait(result=Result.failed, status=Status.finished)
passed = factory.Trait(result=Result.passed, status=Status.finished)
aborted = factory.Trait(result=Result.aborted, status=Status.finished)
travis = factory.Trait(
provider="travis-ci",
external_id=factory.LazyAttribute(
lambda o: str(randint(10000, 999999))),
url=factory.LazyAttribute(
lambda o: "https://travis-ci.org/{}/{}/jobs/{}".format(
o.repository.owner_name, o.repository.name, o.external_id
)),
)
def refresh(self, expires_at=None):
if expires_at is None:
expires_at = timezone.now() + DEFAULT_EXPIRATION
with db.session.begin_nested():
self.token = type(self).generate_token()
self.refresh_token = type(self).generate_token()
self.expires_at = expires_at
db.session.add(self)
db.session.commit()
def cleanup_jobs(task_limit=100):
# timeout any jobs which have been sitting for far too long
results = (
Job.query.unrestricted_unsafe().filter(
Job.status != Status.finished,
or_(
Job.date_updated < timezone.now() - timedelta(hours=1),
Job.date_updated == None, # NOQA
),
).update({
"status": Status.finished,
"result": Result.errored,
"date_updated": timezone.now(),
"date_finished": timezone.now(),
}))
if results:
current_app.logger.warning("cleanup: cleanup_jobs affected rows %s",
results)
db.session.commit()
def login_user(user_id: str, session=session, current_datetime=None):
session["uid"] = str(user_id)
session["expire"] = int(
(
(current_datetime or timezone.now())
+ current_app.config["PERMANENT_SESSION_LIFETIME"]
).strftime(
"%s"
)
)
session.permanent = True
def send_build_notifications(build_id: UUID, time_limit=30):
build = Build.query.unrestricted_unsafe().get(build_id)
if not build:
raise ValueError("Unable to find build with id = {}".format(build_id))
if not build.date_started:
current_app.logger.warn(
"send_build_notifications: build %s missing date_started",
build_id)
return
if not build.date_finished:
current_app.logger.warn(
"send_build_notifications: build %s missing date_finished",
build_id)
return
auth.set_current_tenant(
auth.RepositoryTenant(repository_id=build.repository_id))
# double check that the build is still finished and only send when
# its failing
if build.result != Result.failed or build.status != Status.finished:
current_app.logger.warn(
"send_build_notifications: build %s not marked as failed",
build_id)
return
if build.date_finished < timezone.now() - timedelta(days=1):
current_app.logger.warn(
"send_build_notifications: build %s fimished a long time ago",
build_id)
return
if build.date_started < timezone.now() - timedelta(days=7):
current_app.logger.warn(
"send_build_notifications: build %s started a long time ago",
build_id)
return
email.send_email_notification(build=build)
def sync_all_repos():
queryset = Repository.query.unrestricted_unsafe().filter(
Repository.status == RepositoryStatus.active,
or_(
Repository.last_update_attempt <
(timezone.now() - current_app.config["REPO_SYNC_INTERVAL"]),
Repository.last_update_attempt is None,
),
)
for repo in queryset:
sync_repo.delay(repo_id=repo.id)
