def test_ingest_random(storage, dynamodb_records_table, random_s3_file_maker):
url, metadata = random_s3_file_maker()
ingester = Ingester(storage)
ingester.ingest(url)
records = [dict(r) for r in dynamodb_records_table.scan()]
assert len(records) >= 1
for r in records:
assert r['metadata'] == metadata
def test_ingest_random(storage, dynamodb_records_table, random_s3_file_maker):
url, metadata = random_s3_file_maker()
ingester = Ingester(storage)
ingester.ingest(url)
records = [dict(r) for r in dynamodb_records_table.scan()]
assert len(records) >= 1
for r in records:
assert r['metadata'] == metadata
def test_ingest_no_end(storage, dynamodb_records_table, s3_file_from_metadata,
random_metadata):
del(random_metadata['end'])
url = 's3://foo/' + random_metadata['id']
s3_file_from_metadata(url, random_metadata)
ingester = Ingester(storage)
ingester.ingest(url)
records = [dict(r) for r in dynamodb_records_table.scan()]
assert len(records) >= 1
for r in records:
assert r['metadata'] == random_metadata
def test_ingest_no_end(storage, dynamodb_records_table, s3_file_from_metadata,
random_metadata):
del (random_metadata['end'])
url = 's3://foo/' + random_metadata['id']
s3_file_from_metadata(url, random_metadata)
ingester = Ingester(storage)
ingester.ingest(url)
records = [dict(r) for r in dynamodb_records_table.scan()]
assert len(records) >= 1
# we expect a null end key to come back when the user leaves it out.
random_metadata['end'] = None
for r in records:
assert r['metadata'] == random_metadata
def ingester(storage, sqs_queue, sns_topic_arn):
reporter = SNSReporter(sns_topic_arn)
return Ingester(storage, queue=sqs_queue, reporter=reporter)
def test_listen_no_queue(storage):
ingester = Ingester(storage)
with pytest.raises(InsufficientConfiguration):
ingester.listen(timeout=1)
def test_listen_no_queue(storage):
ingester = Ingester(storage)
with pytest.raises(InsufficientConfiguration):
ingester.listen(timeout=1)