Exemple #1
0
def test_ingest_random(storage, dynamodb_records_table, random_s3_file_maker):
    url, metadata = random_s3_file_maker()
    ingester = Ingester(storage)
    ingester.ingest(url)
    records = [dict(r) for r in dynamodb_records_table.scan()]
    assert len(records) >= 1
    for r in records:
        assert r['metadata'] == metadata
def test_ingest_random(storage, dynamodb_records_table, random_s3_file_maker):
    url, metadata = random_s3_file_maker()
    ingester = Ingester(storage)
    ingester.ingest(url)
    records = [dict(r) for r in dynamodb_records_table.scan()]
    assert len(records) >= 1
    for r in records:
        assert r['metadata'] == metadata
def test_ingest_no_end(storage, dynamodb_records_table, s3_file_from_metadata,
                       random_metadata):
    del(random_metadata['end'])
    url = 's3://foo/' + random_metadata['id']
    s3_file_from_metadata(url, random_metadata)
    ingester = Ingester(storage)
    ingester.ingest(url)
    records = [dict(r) for r in dynamodb_records_table.scan()]
    assert len(records) >= 1
    for r in records:
        assert r['metadata'] == random_metadata
Exemple #4
0
def test_ingest_no_end(storage, dynamodb_records_table, s3_file_from_metadata,
                       random_metadata):
    del (random_metadata['end'])
    url = 's3://foo/' + random_metadata['id']
    s3_file_from_metadata(url, random_metadata)
    ingester = Ingester(storage)
    ingester.ingest(url)
    records = [dict(r) for r in dynamodb_records_table.scan()]
    assert len(records) >= 1

    # we expect a null end key to come back when the user leaves it out.
    random_metadata['end'] = None
    for r in records:
        assert r['metadata'] == random_metadata
Exemple #5
0
def ingester(storage, sqs_queue, sns_topic_arn):
    reporter = SNSReporter(sns_topic_arn)
    return Ingester(storage, queue=sqs_queue, reporter=reporter)
Exemple #6
0
def test_listen_no_queue(storage):
    ingester = Ingester(storage)
    with pytest.raises(InsufficientConfiguration):
        ingester.listen(timeout=1)
def test_listen_no_queue(storage):
    ingester = Ingester(storage)
    with pytest.raises(InsufficientConfiguration):
        ingester.listen(timeout=1)