def default_job(default_build): return factories.JobFactory( build=default_build, date_started=timezone.now() - timedelta(minutes=6), date_finished=timezone.now(), passed=True, )
def test_repo_revision_list(client, db_session, default_login, default_user, git_repo_config): repo = factories.RepositoryFactory.create( backend=RepositoryBackend.git, provider=RepositoryProvider.github, url=git_repo_config.url, ) db_session.add(RepositoryAccess(user=default_user, repository=repo)) db_session.flush() revision = factories.RevisionFactory.create(sha=git_repo_config.commits[0], repository=repo) source = factories.SourceFactory.create(revision=revision) factories.BuildFactory.create(source=source, date_created=timezone.now() - timedelta(minutes=1)) build = factories.BuildFactory.create(source=source, date_created=timezone.now()) resp = client.get("/api/repos/{}/revisions".format(repo.get_full_name())) assert resp.status_code == 200 data = resp.json() assert len(data) == 2 assert data[0]["sha"] == git_repo_config.commits[0] assert data[0]["latest_build"]["id"] == str(build.id) assert data[1]["sha"] == git_repo_config.commits[1] assert data[1]["latest_build"] is None
def cleanup_builds(): # find any artifacts which seemingly are stuck (not enqueued) queryset = Artifact.query.unrestricted_unsafe().filter( Artifact.status != Status.finished, Artifact.date_updated < timezone.now() - timedelta(minutes=15), ) for result in queryset: Artifact.query.unrestricted_unsafe().filter( Artifact.status != Status.finished, Artifact.id == result.id).update({"date_updated": timezone.now()}) db.session.flush() process_artifact(artifact_id=result.id) # first we timeout any jobs which have been sitting for far too long Job.query.unrestricted_unsafe().filter( Job.status != Status.finished, Job.date_updated < timezone.now() - timedelta(hours=1), ).update({ "status": Status.finished, "result": Result.errored, "date_updated": timezone.now(), "date_finished": timezone.now(), }) db.session.commit() queryset = Build.query.unrestricted_unsafe().filter( Build.status != Status.finished, ~Job.query.filter(Job.build_id == Build.id, Job.status != Status.finished).exists(), ) for build in queryset: aggregate_build_stats(build_id=build.id)
def default_build(default_revision): return factories.BuildFactory( revision=default_revision, authors=default_revision.authors, date_started=timezone.now() - timedelta(minutes=6), date_finished=timezone.now(), passed=True, )
def process_artifact(artifact_id, manager=None, force=False, **kwargs): artifact = Artifact.query.unrestricted_unsafe().get(artifact_id) if artifact is None: current_app.logger.error("Artifact %s not found", artifact_id) return if artifact.status == Status.finished and not force: current_app.logger.info( "Skipping artifact processing (%s) - already marked as finished", artifact_id, ) return artifact.status = Status.in_progress artifact.date_started = timezone.now() db.session.add(artifact) db.session.flush() auth.set_current_tenant( auth.RepositoryTenant(repository_id=artifact.repository_id)) job = Job.query.get(artifact.job_id) if job.result == Result.aborted: current_app.logger.info( "Skipping artifact processing (%s) - Job aborted", artifact_id) artifact.status = Status.finished db.session.add(artifact) db.session.commit() return if artifact.file: if manager is None: manager = default_manager try: with db.session.begin_nested(): manager.process(artifact) except Exception: current_app.logger.exception( "Unrecoverable exception processing artifact %s: %s", artifact.job_id, artifact, ) else: current_app.logger.info( "Skipping artifact processing (%s) due to missing file", artifact_id) artifact.status = Status.finished artifact.date_finished = timezone.now() db.session.add(artifact) db.session.commit() # we always aggregate results to avoid locking here aggregate_build_stats_for_job.delay(job_id=job.id)
def track_user(response): from zeus import auth from zeus.utils import timezone user = auth.get_current_user(fetch=False) if user and user.date_active < timezone.now() - timedelta(minutes=5): user.date_active = timezone.now() db.session.add(user) db.session.commit() return response
def get(self): """ Return various details about the installation. """ one_day_ago = timezone.now() - timedelta(days=1) thirty_days_ago = timezone.now() - timedelta(days=30) return { "process": { "id": str(metrics.guid), "uptime": metrics.uptime, "connections": metrics.connections.value, "hits": { "5m": metrics.hits.count(300), "15m": metrics.hits.count(900), "1h": metrics.hits.count(3600), }, }, "config": { "debug": current_app.debug, "environment": current_app.config["SENTRY_ENVIRONMENT"], "release": current_app.config["SENTRY_RELEASE"], "pubsubEndpoint": current_app.config["PUBSUB_ENDPOINT"], }, "stats": { "builds": { "24h": (Build.query.unrestricted_unsafe().filter( Build.date_created > one_day_ago).count()), "30d": (Build.query.unrestricted_unsafe().filter( Build.date_created > thirty_days_ago).count()), }, "repos": { "24h": (db.session.query(Build.repository_id).distinct().filter( Build.date_created > one_day_ago).distinct().count()), "30d": (db.session.query(Build.repository_id).distinct().filter( Build.date_created > thirty_days_ago).distinct().count( )), }, "users": { "24h": User.query.filter(User.date_active > one_day_ago).count(), "30d": User.query.filter( User.date_active > thirty_days_ago).count(), }, }, }
def sync_all_repos(): queryset = Repository.query.unrestricted_unsafe().filter( Repository.status == RepositoryStatus.active, or_( Repository.last_update_attempt < (timezone.now() - timedelta(minutes=5)), Repository.last_update_attempt is None)) for repo in queryset: sync_repo.delay(repo_id=repo.id, ) Repository.query.filter(Repository.id == repo.id, ).update({ 'last_update_attempt': timezone.now(), }) db.session.commit()
def get_user_from_request() -> Optional[User]: expire = session.get('expire') if not expire: return None try: expire = datetime.utcfromtimestamp(expire).replace(tzinfo=timezone.utc) except Exception: current_app.logger.exception('invalid session expirey') del session['expire'] return None if expire <= timezone.now(): current_app.logger.info('session expired') del session['expire'] return None try: uid = session['uid'] except KeyError: current_app.logger.error('missing uid session key', exc_info=True) del session['expire'] return None return User.query.get(uid)
def test_repo_revision_list( client, db_session, default_login, default_revision, default_repo, default_repo_access, default_user, mock_vcs_server, ): mock_vcs_server.replace( mock_vcs_server.GET, "http://localhost:8070/stmt/log", json={"log": [{ "sha": default_revision.sha }]}, ) factories.BuildFactory.create(revision=default_revision, date_created=timezone.now() - timedelta(minutes=1)) factories.BuildFactory.create(revision=default_revision, passed=True) resp = client.get("/api/repos/{}/revisions".format( default_repo.get_full_name())) assert resp.status_code == 200 data = resp.json() assert len(data) == 1 assert data[0]["sha"] == default_revision.sha assert data[0]["latest_build"]["status"] == "finished"
def cleanup_build_stats(task_limit=100): # find any builds which should be marked as finished but aren't queryset = (Build.query.unrestricted_unsafe().filter( Build.status != Status.finished, Build.date_started < timezone.now() - timedelta(minutes=15), ~Job.query.filter(Job.build_id == Build.id, Job.status != Status.finished).exists(), ).limit(task_limit)) for build in queryset: current_app.logger.warning("cleanup: aggregate_build_stats %s", build.id) try: aggregate_build_stats(build_id=build.id) except Exception: current_app.logger.exception("cleanup: aggregate_build_stats %s", build.id) results = (Build.query.unrestricted_unsafe().filter( Build.status != Status.finished, Build.result != Result.errored).update({"status": Status.finished})) if results: current_app.logger.warning( "cleanup: cleanup_build_stats [unfinished; errored] affected rows %s", results, ) db.session.commit()
def get_user_from_request() -> Optional[User]: expire = session.get("expire") if not expire: return None try: expire = datetime.utcfromtimestamp(expire).replace(tzinfo=timezone.utc) except Exception: current_app.logger.exception("invalid session expirey") del session["expire"] return None if expire <= timezone.now(): current_app.logger.info("session expired") del session["expire"] return None try: uid = session["uid"] except KeyError: current_app.logger.error("missing uid session key", exc_info=True) del session["expire"] return None return User.query.get(uid)
def put(self, job: Job): """ Update a job. """ result = self.schema_from_request(job_schema, partial=True) if result.errors: return self.respond(result.errors, 403) was_unfinished = job.status != Status.finished for key, value in result.data.items(): if getattr(job, key) != value: setattr(job, key, value) if db.session.is_modified(job): db.session.add(job) if job.status == Status.finished and was_unfinished and not job.date_finished: job.date_finished = timezone.now() if job.status == Status.finished and has_unprocessed_artifacts(job.id): job.status = Status.collecting_results db.session.commit() aggregate_build_stats_for_job.delay(job_id=job.id) return self.respond_with_schema(job_schema, job)
def cleanup_pending_artifacts(task_limit=100): queryset = (PendingArtifact.query.unrestricted_unsafe().filter( PendingArtifact.date_created < timezone.now() - timedelta(days=1)).limit(1000)) for result in queryset: with db.session.begin_nested(): current_app.logger.warning( "cleanup: process_pending_artifact %s [expired]", result.id) if result.file: result.file.delete() db.session.delete(result) db.session.commit() # find any pending artifacts which seemingly are stuck (not enqueued) queryset = (db.session.query(PendingArtifact.id).filter( Build.external_id == PendingArtifact.external_build_id, Build.provider == PendingArtifact.provider, Build.repository_id == PendingArtifact.repository_id, Build.status == Status.finished, ).limit(task_limit)) for result in queryset: current_app.logger.warning( "cleanup: process_pending_artifact %s [build_finished]", result.id) try: process_pending_artifact(pending_artifact_id=result.id) except UnknownJob: # do we just axe it? pass
def test_merge_build_group_different_providers(client, default_login, default_revision): now = timezone.now() later = now + timedelta(minutes=1) build1 = factories.BuildFactory.create( revision=default_revision, provider="provider1", date_started=now, date_finished=now, ) build2 = factories.BuildFactory.create( revision=default_revision, provider="provider2", date_started=later, date_finished=later, ) merged_build = merge_build_group([build1, build2]) assert merged_build.ref == build1.ref assert merged_build.revision_sha is build1.revision_sha assert merged_build.label == build1.label assert merged_build.original == [build1, build2] assert merged_build.status == Status( max(build1.status.value, build2.status.value)) assert merged_build.result == Result( max(build1.result.value, build2.result.value)) assert merged_build.date_started == now assert merged_build.date_finished == later
class RepositoryFactory(ModelFactory): id = GUIDFactory() owner_name = factory.Faker("word") name = factory.Faker("word") url = factory.LazyAttribute( lambda o: "[email protected]:%s/%s.git" % (o.owner_name, o.name) ) backend = models.RepositoryBackend.git status = models.RepositoryStatus.active provider = models.RepositoryProvider.github external_id = factory.LazyAttribute(lambda o: "{}/{}".format(o.owner_name, o.name)) public = False date_created = factory.LazyAttribute(lambda o: timezone.now()) class Meta: model = models.Repository class Params: unknown = factory.Trait(backend=models.RepositoryBackend.unknown) github = factory.Trait( backend=models.RepositoryBackend.git, provider=models.RepositoryProvider.github, external_id=factory.LazyAttribute( lambda o: "{}/{}".format(o.owner_name, o.name) ), data=factory.LazyAttribute( lambda o: {"full_name": "{}/{}".format(o.owner_name, o.name)} ), )
class JobFactory(ModelFactory): id = GUIDFactory() build = factory.SubFactory('zeus.factories.BuildFactory') build_id = factory.SelfAttribute('build.id') project = factory.SelfAttribute('build.project') project_id = factory.SelfAttribute('project.id') organization = factory.SelfAttribute('project.organization') organization_id = factory.SelfAttribute('organization.id') label = factory.faker.Faker('sentence') result = factory.Iterator([Result.failed, Result.passed]) status = factory.Iterator( [Status.queued, Status.in_progress, Status.finished]) date_created = factory.LazyAttribute( lambda o: timezone.now() - timedelta(minutes=30)) date_started = factory.LazyAttribute(lambda o: (faker.date_time_between( o.date_created, o.date_created + timedelta(minutes=1) ) if o.status in [Status.finished, Status.in_progress] else None)) date_finished = factory.LazyAttribute(lambda o: faker.date_time_between( o.date_started, o.date_started + timedelta(minutes=10)) if o.status == Status.finished else None) class Meta: model = models.Job class Params: queued = factory.Trait(result=Result.unknown, status=Status.queued) in_progress = factory.Trait(result=Result.unknown, status=Status.in_progress) failed = factory.Trait(result=Result.failed, status=Status.finished) passed = factory.Trait(result=Result.passed, status=Status.finished) aborted = factory.Trait(result=Result.aborted, status=Status.finished)
class ChangeRequestFactory(ModelFactory): id = GUIDFactory() message = factory.faker.Faker("sentence") author = factory.SubFactory( "zeus.factories.AuthorFactory", repository=factory.SelfAttribute("..repository")) author_id = factory.SelfAttribute("author.id") parent_revision = factory.SubFactory("zeus.factories.RevisionFactory") parent_revision_sha = factory.SelfAttribute("parent_revision.sha") repository = factory.SelfAttribute("parent_revision.repository") repository_id = factory.SelfAttribute("parent_revision.repository_id") date_created = factory.LazyAttribute( lambda o: timezone.now() - timedelta(minutes=30)) class Meta: model = models.ChangeRequest class Params: github = factory.Trait( provider="github", external_id=factory.LazyAttribute( lambda o: str(randint(10000, 999999))), head_revision_sha=factory.SelfAttribute("head_revision.sha"), head_revision=factory.SubFactory("zeus.factories.RevisionFactory"), url=factory.LazyAttribute( lambda o: "https://github.com/{}/{}/issues/{}".format( o.repository.owner_name, o.repository.name, o.external_id )), )
def login_user(user_id: str, session=session, current_datetime=None): session["uid"] = str(user_id) session["expire"] = int( ((current_datetime or timezone.now()) + current_app.config["PERMANENT_SESSION_LIFETIME"]).strftime("%s")) session.permanent = True with sentry_sdk.configure_scope() as scope: scope.user = {"id": str(user_id)}
class UserFactory(ModelFactory): id = GUIDFactory() email = factory.Faker("email") date_created = factory.LazyAttribute(lambda o: timezone.now()) date_active = factory.LazyAttribute(lambda o: o.date_created) class Meta: model = models.User
def test_cleanup_pending_artifacts_old_file(mocker, db_session): pending_artifact = factories.PendingArtifactFactory.create( date_created=timezone.now() - timedelta(days=2)) cleanup_pending_artifacts() assert not PendingArtifact.query.unrestricted_unsafe().get( pending_artifact.id)
def put(self, job: Job): """ Update a job. """ result = self.schema_from_request(job_schema, partial=True) if result.errors: return self.respond(result.errors, 403) prev_status = job.status for key, value in result.data.items(): if getattr(job, key) != value: setattr(job, key, value) if db.session.is_modified(job): if job.status == Status.queued: job.date_started = None job.result = Result.unknown elif job.status == Status.in_progress and prev_status != Status.in_progress: # TODO(dcramer): this is effectively 'restart' on a job, and we need to # decide how Zeus should deal with it. We either could orphan/hide/remove the # current job, or alternatively we would want to truncate all of its children # which is fairly complex. if not result.data.get("date_started"): job.date_started = timezone.now() if "result" not in result.data: job.result = Result.unknown if ( job.status == Status.finished and prev_status != job.status and not result.data.get("date_finished") ): job.date_finished = timezone.now() if not job.date_started: job.date_started = job.date_created elif job.status != Status.finished: job.date_finished = None if job.status == Status.finished and has_unprocessed_artifacts(job.id): job.status = Status.collecting_results db.session.add(job) db.session.commit() aggregate_build_stats_for_job.delay(job_id=job.id) return self.respond_with_schema(job_schema, job)
def test_cleanup_artifacts_old_file(mocker, db_session): artifact = factories.ArtifactFactory.create(status=Status.finished, date_created=timezone.now() - timedelta(days=45)) cleanup_artifacts() assert artifact.status == Status.expired assert not artifact.file
def default_login(client, default_user): with client.session_transaction() as session: # XXX(dcramer): could use auth.login_user here, but that makes most tests dependent # on that one function session['uid'] = default_user.id session['expire'] = int( (timezone.now() + timedelta(days=1)).strftime('%s')) yield default_user
class JobFactory(ModelFactory): id = GUIDFactory() label = factory.faker.Faker("sentence") build = factory.SubFactory( "zeus.factories.BuildFactory", status=factory.SelfAttribute("..status"), result=factory.SelfAttribute("..result"), date_created=factory.SelfAttribute("..date_created"), date_started=factory.SelfAttribute("..date_started"), date_finished=factory.SelfAttribute("..date_finished"), ) build_id = factory.SelfAttribute("build.id") repository = factory.SelfAttribute("build.repository") repository_id = factory.SelfAttribute("repository.id") label = factory.faker.Faker("sentence") result = factory.Iterator([Result.failed, Result.passed]) status = factory.Iterator( [Status.queued, Status.in_progress, Status.finished]) allow_failure = False date_created = factory.LazyAttribute( lambda o: timezone.now() - timedelta(minutes=30)) date_started = factory.LazyAttribute(lambda o: (faker.date_time_between( o.date_created, o.date_created + timedelta(minutes=1) ) if o.status in [Status.finished, Status.in_progress] else None)) date_finished = factory.LazyAttribute(lambda o: faker.date_time_between( o.date_started, o.date_started + timedelta(minutes=10)) if o.status == Status.finished else None) date_updated = factory.LazyAttribute( lambda o: o.date_finished or o.date_started or o.date_created) class Meta: model = models.Job class Params: queued = factory.Trait( result=Result.unknown, status=Status.queued, date_started=None, date_finished=None, ) in_progress = factory.Trait(result=Result.unknown, status=Status.in_progress, date_finished=None) finished = factory.Trait(status=Status.finished) failed = factory.Trait(result=Result.failed, status=Status.finished) passed = factory.Trait(result=Result.passed, status=Status.finished) aborted = factory.Trait(result=Result.aborted, status=Status.finished) travis = factory.Trait( provider="travis-ci", external_id=factory.LazyAttribute( lambda o: str(randint(10000, 999999))), url=factory.LazyAttribute( lambda o: "https://travis-ci.org/{}/{}/jobs/{}".format( o.repository.owner_name, o.repository.name, o.external_id )), )
def refresh(self, expires_at=None): if expires_at is None: expires_at = timezone.now() + DEFAULT_EXPIRATION with db.session.begin_nested(): self.token = type(self).generate_token() self.refresh_token = type(self).generate_token() self.expires_at = expires_at db.session.add(self) db.session.commit()
def cleanup_jobs(task_limit=100): # timeout any jobs which have been sitting for far too long results = ( Job.query.unrestricted_unsafe().filter( Job.status != Status.finished, or_( Job.date_updated < timezone.now() - timedelta(hours=1), Job.date_updated == None, # NOQA ), ).update({ "status": Status.finished, "result": Result.errored, "date_updated": timezone.now(), "date_finished": timezone.now(), })) if results: current_app.logger.warning("cleanup: cleanup_jobs affected rows %s", results) db.session.commit()
def login_user(user_id: str, session=session, current_datetime=None): session["uid"] = str(user_id) session["expire"] = int( ( (current_datetime or timezone.now()) + current_app.config["PERMANENT_SESSION_LIFETIME"] ).strftime( "%s" ) ) session.permanent = True
def send_build_notifications(build_id: UUID, time_limit=30): build = Build.query.unrestricted_unsafe().get(build_id) if not build: raise ValueError("Unable to find build with id = {}".format(build_id)) if not build.date_started: current_app.logger.warn( "send_build_notifications: build %s missing date_started", build_id) return if not build.date_finished: current_app.logger.warn( "send_build_notifications: build %s missing date_finished", build_id) return auth.set_current_tenant( auth.RepositoryTenant(repository_id=build.repository_id)) # double check that the build is still finished and only send when # its failing if build.result != Result.failed or build.status != Status.finished: current_app.logger.warn( "send_build_notifications: build %s not marked as failed", build_id) return if build.date_finished < timezone.now() - timedelta(days=1): current_app.logger.warn( "send_build_notifications: build %s fimished a long time ago", build_id) return if build.date_started < timezone.now() - timedelta(days=7): current_app.logger.warn( "send_build_notifications: build %s started a long time ago", build_id) return email.send_email_notification(build=build)
def sync_all_repos(): queryset = Repository.query.unrestricted_unsafe().filter( Repository.status == RepositoryStatus.active, or_( Repository.last_update_attempt < (timezone.now() - current_app.config["REPO_SYNC_INTERVAL"]), Repository.last_update_attempt is None, ), ) for repo in queryset: sync_repo.delay(repo_id=repo.id)