Beispiel #1
0
def test_calculate_durations(jm, test_repository, mock_log_parser):
    """
    Test the calculation of average job durations and their use during
    subsequent job ingestion.
    """
    rs = result_set()
    jm.store_result_set_data([rs])
    now = int(time.time())

    first_job_duration = 120
    first_job = job_data(revision=rs['revision'],
                         start_timestamp=now,
                         end_timestamp=now + first_job_duration)
    jm.store_job_data([first_job])

    # Generate average duration based on the first job.
    call_command('calculate_durations')

    # Ingest the same job type again to check that the pre-generated
    # average duration is used during ingestion.
    second_job_duration = 142
    second_job = job_data(revision=rs['revision'],
                          start_timestamp=now,
                          end_timestamp=now + second_job_duration,
                          job_guid='a-different-unique-guid')
    jm.store_job_data([second_job])
    ingested_second_job = jm.get_job(2)[0]
    assert ingested_second_job['running_eta'] == first_job_duration

    # Check that the average duration is updated now that there are two jobs.
    call_command('calculate_durations')
    durations = JobDuration.objects.all()
    assert len(durations) == 1
    expected_duration = int(round((first_job_duration + second_job_duration) / 2))
    assert durations[0].average_duration == expected_duration
def test_calculate_durations(jm, test_repository, mock_log_parser):
    """
    Test the calculation of average job durations and their use during
    subsequent job ingestion.
    """
    rs = result_set()
    jm.store_result_set_data([rs])
    now = int(time.time())

    first_job_duration = 120
    first_job = job_data(revision=rs['revision'],
                         start_timestamp=now,
                         end_timestamp=now + first_job_duration)
    jm.store_job_data([first_job])

    # Generate average duration based on the first job.
    call_command('calculate_durations')

    # Ingest the same job type again to check that the pre-generated
    # average duration is used during ingestion.
    second_job_duration = 142
    second_job = job_data(revision=rs['revision'],
                          start_timestamp=now,
                          end_timestamp=now + second_job_duration,
                          job_guid='a-different-unique-guid')
    jm.store_job_data([second_job])
    ingested_second_job = jm.get_job(2)[0]
    assert ingested_second_job['running_eta'] == first_job_duration

    # Check that the average duration is updated now that there are two jobs.
    call_command('calculate_durations')
    durations = JobDuration.objects.all()
    assert len(durations) == 1
    expected_duration = int(round((first_job_duration + second_job_duration) / 2))
    assert durations[0].average_duration == expected_duration
def test_bad_date_value_ingestion(jm, initial_data, mock_log_parser):
    """
    Test ingesting an blob with bad date value

    """
    rs = result_set()
    blob = job_data(start_timestamp="foo", revision_hash=rs["revision_hash"])

    jm.store_result_set_data([rs])
    jm.store_job_data([blob])
Beispiel #4
0
def test_bad_date_value_ingestion(jm, test_repository, mock_log_parser):
    """
    Test ingesting an blob with bad date value

    """
    rs = result_set()
    blob = job_data(start_timestamp="foo", revision=rs['revision'])

    jm.store_result_set_data([rs])
    jm.store_job_data([blob])
def test_bad_date_value_ingestion(jm, test_repository, mock_log_parser):
    """
    Test ingesting an blob with bad date value

    """
    rs = result_set()
    blob = job_data(start_timestamp="foo",
                    revision=rs['revision'])

    jm.store_result_set_data([rs])
    jm.store_job_data([blob])
def test_bad_date_value_ingestion(jm, initial_data, mock_log_parser):
    """
    Test ingesting an blob with bad date value

    """
    rs = result_set()
    blob = job_data(start_timestamp="foo",
                    revision_hash=rs['revision_hash'])

    jm.store_job_data([blob])

    jm.store_result_set_data([rs])

    jm.process_objects(1)

    # Confirm that we don't get a ValueError when casting a non-number
    last_error = get_objectstore_last_error(
        jm) == u"invalid literal for long() with base 10: 'foo'"

    assert last_error == False
Beispiel #7
0
def test_process_objects(jm, initial_data, mock_log_parser):
    """Claims and processes a chunk of unprocessed JSON jobs data blobs."""
    # Load some rows into the objectstore

    rs = result_set()

    blobs = [
        job_data(submit_timestamp="1330454755",
                 job_guid="guid1",
                 revision_hash=rs['revision_hash']),
        job_data(submit_timestamp="1330454756",
                 job_guid="guid2",
                 revision_hash=rs['revision_hash']),
        job_data(submit_timestamp="1330454757",
                 job_guid="guid3",
                 revision_hash=rs['revision_hash']),
    ]

    jm.store_result_set_data([rs])

    jm.store_job_data(blobs)

    # just process two rows
    jm.process_objects(2, raise_errors=True)

    test_run_rows = jm.get_dhub(
        jm.CT_JOBS).execute(proc="jobs_test.selects.jobs")
    date_set = set([r['submit_timestamp'] for r in test_run_rows])
    expected_dates = set([1330454755, 1330454756, 1330454757])

    complete_count = jm.get_dhub(jm.CT_OBJECTSTORE).execute(
        proc="objectstore_test.counts.complete")[0]["complete_count"]
    loading_count = jm.get_dhub(jm.CT_OBJECTSTORE).execute(
        proc="objectstore_test.counts.loading")[0]["loading_count"]

    jm.disconnect()

    assert complete_count == 2
    assert loading_count == 0
    assert date_set.issubset(expected_dates)
    assert len(date_set) == 2
Beispiel #8
0
def test_bad_date_value_ingestion(jm, initial_data, mock_log_parser):
    """
    Test ingesting an blob with bad date value

    """
    rs = result_set()
    blob = job_data(start_timestamp="foo", revision_hash=rs['revision_hash'])

    jm.store_job_data([blob])

    jm.store_result_set_data([rs])

    jm.process_objects(1)

    # Confirm that we don't get a ValueError when casting a non-number
    last_error = get_objectstore_last_error(
        jm) == u"invalid literal for long() with base 10: 'foo'"

    jm.disconnect()

    assert last_error == False
def test_process_objects(jm, initial_data, mock_log_parser):
    """Claims and processes a chunk of unprocessed JSON jobs data blobs."""
    # Load some rows into the objectstore

    rs = result_set()

    blobs = [
        job_data(submit_timestamp="1330454755",
                 job_guid="guid1", revision_hash=rs['revision_hash']),
        job_data(submit_timestamp="1330454756",
                 job_guid="guid2", revision_hash=rs['revision_hash']),
        job_data(submit_timestamp="1330454757",
                 job_guid="guid3", revision_hash=rs['revision_hash']),
    ]

    jm.store_result_set_data([rs])

    jm.store_job_data(blobs)

    # just process two rows
    jm.process_objects(2, raise_errors=True)

    test_run_rows = jm.get_dhub(jm.CT_JOBS).execute(
        proc="jobs_test.selects.jobs")
    date_set = set([r['submit_timestamp'] for r in test_run_rows])
    expected_dates = set([1330454755, 1330454756, 1330454757])

    complete_count = jm.get_dhub(jm.CT_OBJECTSTORE).execute(
        proc="objectstore_test.counts.complete")[0]["complete_count"]
    loading_count = jm.get_dhub(jm.CT_OBJECTSTORE).execute(
        proc="objectstore_test.counts.loading")[0]["loading_count"]

    jm.disconnect()

    assert complete_count == 2
    assert loading_count == 0
    assert date_set.issubset(expected_dates)
    assert len(date_set) == 2