def inspire_record():
"""Return results from the pipeline."""
from scrapy.http import TextResponse
spider = aps_spider.APSSpider()
items = spider.parse(fake_response_from_file('aps/aps_single_response.json', response_type=TextResponse))
pipeline = InspireAPIPushPipeline()
return pipeline.process_item(items.next(), spider)
def inspire_record():
"""Return results from the pipeline."""
from scrapy.http import TextResponse
spider = aps_spider.APSSpider()
items = spider.parse(
fake_response_from_file('aps/aps_single_response.json',
response_type=TextResponse))
pipeline = InspireAPIPushPipeline()
return pipeline.process_item(items.next(), spider)
def test_prepare_payload(
tmpdir, json_spider_record, spider, expected_response,
):
"""Test that the generated payload is ok."""
_, json_record = json_spider_record
os.environ['SCRAPY_JOB'] = 'scrapy_job'
os.environ['SCRAPY_FEED_URI'] = 'scrapy_feed_uri'
os.environ['SCRAPY_LOG_FILE'] = 'scrapy_log_file'
pipeline = InspireAPIPushPipeline()
pipeline.open_spider(spider)
pipeline.process_item(json_record, spider)
result = pipeline._prepare_payload(spider)
# acquisition_source has a timestamp
result['results_data'][0]['acquisition_source'].pop('datetime')
expected_response['results_data'][0]['acquisition_source'].pop('date')
for record in result['results_data']:
validate(record, 'hep')
for res, exp in zip(
result['results_data'],
expected_response['results_data'],
):
for key in res:
assert key in exp
assert res[key] == exp[key]
assert result == expected_response
def test_prepare_payload(
tmpdir,
json_spider_record,
spider,
expected_response,
):
"""Test that the generated payload is ok."""
_, json_record = json_spider_record
os.environ['SCRAPY_JOB'] = 'scrapy_job'
os.environ['SCRAPY_FEED_URI'] = 'scrapy_feed_uri'
fixed_time = expected_response[0]['results_data'][0]['acquisition_source'][
'datetime']
freezer = freeze_time(fixed_time)
freezer.start()
pipeline = InspireAPIPushPipeline()
pipeline.open_spider(spider)
pipeline.process_item(json_record, spider)
results = pipeline._prepare_payload(spider)
for result, response in zip(results, expected_response):
validate(result['results_data'][0]['record'], 'hep')
record = result['results_data'][0]['record']
for key in record:
assert key in response['results_data'][0]
assert record[key] == response['results_data'][0][key]
assert sorted(result) == sorted(response)
freezer.stop()