def test_sqsfeedstorage_and_sqsexporter():
with mock_sqs():
sqs = boto3.resource('sqs', region_name='eu-central-1')
queue = sqs.create_queue(QueueName='bar')
storage = sqsfeedexport.SQSFeedStorage('sqs://bar')
assert storage.queue_name == 'bar'
deck = storage.open()
exporter = sqsfeedexport.SQSExporter(deck)
# do the `scrapy.extensions.feedexport.FeedExporter` song and dance
exporter.start_exporting()
for item in examples:
exporter.export_item(item)
exporter.finish_exporting()
assert len(deck) == 6
# call the private method directly to avoid the deferToThread call
storage._store_in_thread(deck)
# now check what we've got
messages = queue.receive_messages(MaxNumberOfMessages=10,
MessageAttributeNames=['All'])
for index in xrange(6):
assert messages[index].body == 'ScrapyItem'
assert messages[index].message_attributes == \
sqsfeedexport.translate_item_to_message(examples[index])['MessageAttributes']
assert len(messages) == 6
def test_empty_scrapy_response(monkeypatch):
def fake_uuid():
return '12345-67890'
monkeypatch.setattr('sqsfeedexport.uuid4', fake_uuid)
assert sqsfeedexport.translate_item_to_message({}) == {
'Id': '12345-67890',
'MessageBody': 'ScrapyItem'
}
def test_batch_send_entry_translation(monkeypatch):
def fake_uuid():
return '12345-67890'
monkeypatch.setattr('sqsfeedexport.uuid4', fake_uuid)
message = sqsfeedexport.translate_item_to_message(examples[0])
assert message == {
'Id': '12345-67890',
'MessageBody': 'ScrapyItem',
'MessageAttributes': {
'company': {
'StringValue': 'Nings ',
'DataType': 'String'
},
'deadline': {
'StringValue': '2017-06-13T00:00:00',
'DataType': 'String'
},
'posted': {
'StringValue': '2017-06-06T10:15:00',
'DataType': 'String'
},
'spider': {
'StringValue': 'alfred',
'DataType': 'String'
},
'title': {
'StringValue':
'Okkur vantar sendla, ert \u00fe\u00fa r\u00e9tta manneskjan?',
'DataType': 'String'
},
'url': {
'StringValue': 'https://alfred.is/starf/11076',
'DataType': 'String'
}
}
}
def test_no_nonetypes(monkeypatch):
def fake_uuid():
return '12345-67890'
monkeypatch.setattr('sqsfeedexport.uuid4', fake_uuid)
assert sqsfeedexport.translate_item_to_message(example_with_nones) == {
'Id': '12345-67890',
'MessageBody': 'ScrapyItem',
'MessageAttributes': {
'posted': {
'StringValue': '2017-05-12',
'DataType': 'String'
},
'title': {
'StringValue':
'Lausar st\u00f6\u00f0ur vi\u00f0 Finnbogasta\u00f0ask\u00f3la',
'DataType': 'String'
},
'url': {
'StringValue': 'http://www.mbl.is/atvinna/3612/',
'DataType': 'String'
}
}
}