def test_calculate_durations(jm, test_repository, mock_log_parser):
"""
Test the calculation of average job durations and their use during
subsequent job ingestion.
"""
rs = result_set()
jm.store_result_set_data([rs])
now = int(time.time())
first_job_duration = 120
first_job = job_data(revision=rs['revision'],
start_timestamp=now,
end_timestamp=now + first_job_duration)
jm.store_job_data([first_job])
# Generate average duration based on the first job.
call_command('calculate_durations')
# Ingest the same job type again to check that the pre-generated
# average duration is used during ingestion.
second_job_duration = 142
second_job = job_data(revision=rs['revision'],
start_timestamp=now,
end_timestamp=now + second_job_duration,
job_guid='a-different-unique-guid')
jm.store_job_data([second_job])
ingested_second_job = jm.get_job(2)[0]
assert ingested_second_job['running_eta'] == first_job_duration
# Check that the average duration is updated now that there are two jobs.
call_command('calculate_durations')
durations = JobDuration.objects.all()
assert len(durations) == 1
expected_duration = int(round((first_job_duration + second_job_duration) / 2))
assert durations[0].average_duration == expected_duration
def test_calculate_durations(jm, test_repository, mock_log_parser):
"""
Test the calculation of average job durations and their use during
subsequent job ingestion.
"""
rs = result_set()
jm.store_result_set_data([rs])
now = int(time.time())
first_job_duration = 120
first_job = job_data(revision=rs['revision'],
start_timestamp=now,
end_timestamp=now + first_job_duration)
jm.store_job_data([first_job])
# Generate average duration based on the first job.
call_command('calculate_durations')
# Ingest the same job type again to check that the pre-generated
# average duration is used during ingestion.
second_job_duration = 142
second_job = job_data(revision=rs['revision'],
start_timestamp=now,
end_timestamp=now + second_job_duration,
job_guid='a-different-unique-guid')
jm.store_job_data([second_job])
ingested_second_job = jm.get_job(2)[0]
assert ingested_second_job['running_eta'] == first_job_duration
# Check that the average duration is updated now that there are two jobs.
call_command('calculate_durations')
durations = JobDuration.objects.all()
assert len(durations) == 1
expected_duration = int(round((first_job_duration + second_job_duration) / 2))
assert durations[0].average_duration == expected_duration
def test_bad_date_value_ingestion(jm, initial_data, mock_log_parser):
"""
Test ingesting an blob with bad date value
"""
rs = result_set()
blob = job_data(start_timestamp="foo", revision_hash=rs["revision_hash"])
jm.store_result_set_data([rs])
jm.store_job_data([blob])
def test_bad_date_value_ingestion(jm, test_repository, mock_log_parser):
"""
Test ingesting an blob with bad date value
"""
rs = result_set()
blob = job_data(start_timestamp="foo", revision=rs['revision'])
jm.store_result_set_data([rs])
jm.store_job_data([blob])
def test_bad_date_value_ingestion(jm, test_repository, mock_log_parser):
"""
Test ingesting an blob with bad date value
"""
rs = result_set()
blob = job_data(start_timestamp="foo",
revision=rs['revision'])
jm.store_result_set_data([rs])
jm.store_job_data([blob])
def test_bad_date_value_ingestion(jm, initial_data, mock_log_parser):
"""
Test ingesting an blob with bad date value
"""
rs = result_set()
blob = job_data(start_timestamp="foo",
revision_hash=rs['revision_hash'])
jm.store_job_data([blob])
jm.store_result_set_data([rs])
jm.process_objects(1)
# Confirm that we don't get a ValueError when casting a non-number
last_error = get_objectstore_last_error(
jm) == u"invalid literal for long() with base 10: 'foo'"
assert last_error == False
def test_process_objects(jm, initial_data, mock_log_parser):
"""Claims and processes a chunk of unprocessed JSON jobs data blobs."""
# Load some rows into the objectstore
rs = result_set()
blobs = [
job_data(submit_timestamp="1330454755",
job_guid="guid1",
revision_hash=rs['revision_hash']),
job_data(submit_timestamp="1330454756",
job_guid="guid2",
revision_hash=rs['revision_hash']),
job_data(submit_timestamp="1330454757",
job_guid="guid3",
revision_hash=rs['revision_hash']),
]
jm.store_result_set_data([rs])
jm.store_job_data(blobs)
# just process two rows
jm.process_objects(2, raise_errors=True)
test_run_rows = jm.get_dhub(
jm.CT_JOBS).execute(proc="jobs_test.selects.jobs")
date_set = set([r['submit_timestamp'] for r in test_run_rows])
expected_dates = set([1330454755, 1330454756, 1330454757])
complete_count = jm.get_dhub(jm.CT_OBJECTSTORE).execute(
proc="objectstore_test.counts.complete")[0]["complete_count"]
loading_count = jm.get_dhub(jm.CT_OBJECTSTORE).execute(
proc="objectstore_test.counts.loading")[0]["loading_count"]
jm.disconnect()
assert complete_count == 2
assert loading_count == 0
assert date_set.issubset(expected_dates)
assert len(date_set) == 2
def test_bad_date_value_ingestion(jm, initial_data, mock_log_parser):
"""
Test ingesting an blob with bad date value
"""
rs = result_set()
blob = job_data(start_timestamp="foo", revision_hash=rs['revision_hash'])
jm.store_job_data([blob])
jm.store_result_set_data([rs])
jm.process_objects(1)
# Confirm that we don't get a ValueError when casting a non-number
last_error = get_objectstore_last_error(
jm) == u"invalid literal for long() with base 10: 'foo'"
jm.disconnect()
assert last_error == False
def test_process_objects(jm, initial_data, mock_log_parser):
"""Claims and processes a chunk of unprocessed JSON jobs data blobs."""
# Load some rows into the objectstore
rs = result_set()
blobs = [
job_data(submit_timestamp="1330454755",
job_guid="guid1", revision_hash=rs['revision_hash']),
job_data(submit_timestamp="1330454756",
job_guid="guid2", revision_hash=rs['revision_hash']),
job_data(submit_timestamp="1330454757",
job_guid="guid3", revision_hash=rs['revision_hash']),
]
jm.store_result_set_data([rs])
jm.store_job_data(blobs)
# just process two rows
jm.process_objects(2, raise_errors=True)
test_run_rows = jm.get_dhub(jm.CT_JOBS).execute(
proc="jobs_test.selects.jobs")
date_set = set([r['submit_timestamp'] for r in test_run_rows])
expected_dates = set([1330454755, 1330454756, 1330454757])
complete_count = jm.get_dhub(jm.CT_OBJECTSTORE).execute(
proc="objectstore_test.counts.complete")[0]["complete_count"]
loading_count = jm.get_dhub(jm.CT_OBJECTSTORE).execute(
proc="objectstore_test.counts.loading")[0]["loading_count"]
jm.disconnect()
assert complete_count == 2
assert loading_count == 0
assert date_set.issubset(expected_dates)
assert len(date_set) == 2