def test_ingest_pulse_jobs(pulse_jobs, test_repository, push_stored, failure_classifications, mock_log_parser): """ Ingest a job through the JSON Schema validated JobLoader used by Pulse """ jl = JobLoader() revision = push_stored[0]["revision"] for job in pulse_jobs: job["origin"]["revision"] = revision jl.process_job(job) jobs = Job.objects.all() assert len(jobs) == 5 assert [job.taskcluster_metadata for job in jobs] assert set(TaskclusterMetadata.objects.values_list( 'task_id', flat=True)) == set(['IYyscnNMTLuxzna7PNqUJQ', 'XJCbbRQ6Sp-UL1lL-tw5ng', 'ZsSzJQu3Q7q2MfehIBAzKQ', 'bIzVZt9jQQKgvQYD3a2HQw']) job_logs = JobLog.objects.filter(job_id=1) assert job_logs.count() == 2 logs_expected = [{"name": "builds-4h", "url": "http://ftp.mozilla.org/pub/mozilla.org/spidermonkey/tinderbox-builds/mozilla-inbound-linux64/mozilla-inbound_linux64_spidermonkey-warnaserr-bm57-build1-build352.txt.gz", "parse_status": 0}, {"name": "errorsummary_json", "url": "http://mozilla-releng-blobs.s3.amazonaws.com/blobs/Mozilla-Inbound-Non-PGO/sha512/05c7f57df6583c6351c6b49e439e2678e0f43c2e5b66695ea7d096a7519e1805f441448b5ffd4cc3b80b8b2c74b244288fda644f55ed0e226ef4e25ba02ca466", "parse_status": 0}] assert [{"name": item.name, "url": item.url, "parse_status": item.status} for item in job_logs.all()] == logs_expected assert JobDetail.objects.count() == 2
def test_ingest_pending_pulse_job(pulse_jobs, push_stored, failure_classifications, mock_log_parser): """ Test that ingesting a pending job (1) works and (2) ingests the taskcluster metadata """ jl = JobLoader() pulse_job = pulse_jobs[0] revision = push_stored[0]["revision"] pulse_job["origin"]["revision"] = revision pulse_job["state"] = "pending" jl.process_job(pulse_job, 'https://firefox-ci-tc.services.mozilla.com') jobs = Job.objects.all() assert len(jobs) == 1 job = jobs[0] assert job.taskcluster_metadata assert job.taskcluster_metadata.task_id == 'IYyscnNMTLuxzna7PNqUJQ' # should not have processed any log or details for pending jobs assert JobLog.objects.count() == 2 # we're no longer storing artifacts in this table assert JobDetail.objects.count() == 0
def test_skip_unscheduled(first_job, failure_classifications, mock_log_parser): jl = JobLoader() first_job["state"] = "unscheduled" jl.process_job(first_job, 'https://firefox-ci-tc.services.mozilla.com') assert not Job.objects.count()
def test_skip_unscheduled(first_job, failure_classifications, mock_log_parser): jl = JobLoader() first_job["state"] = "unscheduled" jl.process_job(first_job) assert not Job.objects.count()
def test_ingest_pulse_jobs_with_missing_push(pulse_jobs): """ Ingest jobs with missing pushes, so they should throw an exception """ jl = JobLoader() job = pulse_jobs[0] job["origin"]["revision"] = "1234567890123456789012345678901234567890" with pytest.raises(MissingPushException): for pulse_job in pulse_jobs: jl.process_job(pulse_job) # if one job isn't ready, except on the whole batch. They'll retry as a # task after the timeout. assert Job.objects.count() == 0
def test_ingest_pulse_jobs_bad_project(pulse_jobs, test_repository, push_stored, failure_classifications, mock_log_parser): """ Test ingesting a pulse job with bad repo will skip, ingest others """ jl = JobLoader() revision = push_stored[0]["revision"] job = pulse_jobs[0] job["origin"]["revision"] = revision job["origin"]["project"] = "ferd" for pulse_job in pulse_jobs: jl.process_job(pulse_job) # length of pulse jobs is 5, so one will be skipped due to bad project assert Job.objects.count() == 4
def test_ingest_pulse_job_with_long_job_type_name( pulse_jobs, test_repository, push_stored, failure_classifications, mock_log_parser ): """ Ingest a job through the JSON Schema validated JobLoader used by Pulse """ job = pulse_jobs[0] jl = JobLoader() revision = push_stored[0]["revision"] job["display"][ "jobName" ] = "this is a very long string that exceeds the 100 character size that was the previous limit by just a little bit" job["origin"]["revision"] = revision jl.process_job(job, 'https://firefox-ci-tc.services.mozilla.com') jobs = Job.objects.all() assert len(jobs) == 1
def test_new_job_transformation(new_pulse_jobs, new_transformed_jobs, failure_classifications): jl = JobLoader() for message in new_pulse_jobs: # "task_id" which is not really the task_id job_guid = message["taskId"] (decoded_task_id, _) = job_guid.split("/") # As of slugid v2, slugid.encode() returns a string not bytestring under Python 3. taskId = slugid.encode(uuid.UUID(decoded_task_id)) transformed_job = jl.process_job(message, 'https://firefox-ci-tc.services.mozilla.com') # Not all messages from Taskcluster will be processed if transformed_job: assert new_transformed_jobs[taskId] == transformed_job
def test_ingest_pulse_jobs_with_missing_push(pulse_jobs): """ Ingest jobs with missing pushes, so they should throw an exception """ jl = JobLoader() job = pulse_jobs[0] job["origin"]["revision"] = "1234567890123456789012345678901234567890" responses.add( responses.GET, "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/IYyscnNMTLuxzna7PNqUJQ", json={}, content_type='application/json', status=200, ) with pytest.raises(ObjectDoesNotExist): for pulse_job in pulse_jobs: jl.process_job(pulse_job, 'https://firefox-ci-tc.services.mozilla.com') # if one job isn't ready, except on the whole batch. They'll retry as a # task after the timeout. assert Job.objects.count() == 0
def test_ingest_pending_pulse_job(pulse_jobs, push_stored, failure_classifications, mock_log_parser): """ Test that ingesting a pending job (1) works and (2) ingests the taskcluster metadata """ jl = JobLoader() pulse_job = pulse_jobs[0] revision = push_stored[0]["revision"] pulse_job["origin"]["revision"] = revision pulse_job["state"] = "pending" jl.process_job(pulse_job) jobs = Job.objects.all() assert len(jobs) == 1 job = jobs[0] assert job.taskcluster_metadata assert job.taskcluster_metadata.task_id == 'IYyscnNMTLuxzna7PNqUJQ' # should not have processed any log or details for pending jobs assert JobLog.objects.count() == 2 assert JobDetail.objects.count() == 2