def test_calculate_durations(jm, test_repository, mock_log_parser): """ Test the calculation of average job durations and their use during subsequent job ingestion. """ rs = result_set() jm.store_result_set_data([rs]) now = int(time.time()) first_job_duration = 120 first_job = job_data(revision=rs['revision'], start_timestamp=now, end_timestamp=now + first_job_duration) jm.store_job_data([first_job]) # Generate average duration based on the first job. call_command('calculate_durations') # Ingest the same job type again to check that the pre-generated # average duration is used during ingestion. second_job_duration = 142 second_job = job_data(revision=rs['revision'], start_timestamp=now, end_timestamp=now + second_job_duration, job_guid='a-different-unique-guid') jm.store_job_data([second_job]) ingested_second_job = jm.get_job(2)[0] assert ingested_second_job['running_eta'] == first_job_duration # Check that the average duration is updated now that there are two jobs. call_command('calculate_durations') durations = JobDuration.objects.all() assert len(durations) == 1 expected_duration = int(round((first_job_duration + second_job_duration) / 2)) assert durations[0].average_duration == expected_duration
def test_bad_date_value_ingestion(jm, initial_data, mock_log_parser): """ Test ingesting an blob with bad date value """ rs = result_set() blob = job_data(start_timestamp="foo", revision_hash=rs["revision_hash"]) jm.store_result_set_data([rs]) jm.store_job_data([blob])
def test_bad_date_value_ingestion(jm, test_repository, mock_log_parser): """ Test ingesting an blob with bad date value """ rs = result_set() blob = job_data(start_timestamp="foo", revision=rs['revision']) jm.store_result_set_data([rs]) jm.store_job_data([blob])
def test_bad_date_value_ingestion(jm, initial_data, mock_log_parser): """ Test ingesting an blob with bad date value """ rs = result_set() blob = job_data(start_timestamp="foo", revision_hash=rs['revision_hash']) jm.store_job_data([blob]) jm.store_result_set_data([rs]) jm.process_objects(1) # Confirm that we don't get a ValueError when casting a non-number last_error = get_objectstore_last_error( jm) == u"invalid literal for long() with base 10: 'foo'" assert last_error == False
def test_process_objects(jm, initial_data, mock_log_parser): """Claims and processes a chunk of unprocessed JSON jobs data blobs.""" # Load some rows into the objectstore rs = result_set() blobs = [ job_data(submit_timestamp="1330454755", job_guid="guid1", revision_hash=rs['revision_hash']), job_data(submit_timestamp="1330454756", job_guid="guid2", revision_hash=rs['revision_hash']), job_data(submit_timestamp="1330454757", job_guid="guid3", revision_hash=rs['revision_hash']), ] jm.store_result_set_data([rs]) jm.store_job_data(blobs) # just process two rows jm.process_objects(2, raise_errors=True) test_run_rows = jm.get_dhub( jm.CT_JOBS).execute(proc="jobs_test.selects.jobs") date_set = set([r['submit_timestamp'] for r in test_run_rows]) expected_dates = set([1330454755, 1330454756, 1330454757]) complete_count = jm.get_dhub(jm.CT_OBJECTSTORE).execute( proc="objectstore_test.counts.complete")[0]["complete_count"] loading_count = jm.get_dhub(jm.CT_OBJECTSTORE).execute( proc="objectstore_test.counts.loading")[0]["loading_count"] jm.disconnect() assert complete_count == 2 assert loading_count == 0 assert date_set.issubset(expected_dates) assert len(date_set) == 2
def test_bad_date_value_ingestion(jm, initial_data, mock_log_parser): """ Test ingesting an blob with bad date value """ rs = result_set() blob = job_data(start_timestamp="foo", revision_hash=rs['revision_hash']) jm.store_job_data([blob]) jm.store_result_set_data([rs]) jm.process_objects(1) # Confirm that we don't get a ValueError when casting a non-number last_error = get_objectstore_last_error( jm) == u"invalid literal for long() with base 10: 'foo'" jm.disconnect() assert last_error == False
def test_process_objects(jm, initial_data, mock_log_parser): """Claims and processes a chunk of unprocessed JSON jobs data blobs.""" # Load some rows into the objectstore rs = result_set() blobs = [ job_data(submit_timestamp="1330454755", job_guid="guid1", revision_hash=rs['revision_hash']), job_data(submit_timestamp="1330454756", job_guid="guid2", revision_hash=rs['revision_hash']), job_data(submit_timestamp="1330454757", job_guid="guid3", revision_hash=rs['revision_hash']), ] jm.store_result_set_data([rs]) jm.store_job_data(blobs) # just process two rows jm.process_objects(2, raise_errors=True) test_run_rows = jm.get_dhub(jm.CT_JOBS).execute( proc="jobs_test.selects.jobs") date_set = set([r['submit_timestamp'] for r in test_run_rows]) expected_dates = set([1330454755, 1330454756, 1330454757]) complete_count = jm.get_dhub(jm.CT_OBJECTSTORE).execute( proc="objectstore_test.counts.complete")[0]["complete_count"] loading_count = jm.get_dhub(jm.CT_OBJECTSTORE).execute( proc="objectstore_test.counts.loading")[0]["loading_count"] jm.disconnect() assert complete_count == 2 assert loading_count == 0 assert date_set.issubset(expected_dates) assert len(date_set) == 2