def test_run(self):
crawler_process_mock = mock.Mock()
notifier_mock = mock.Mock()
extractor_mock = mock.Mock()
repository_mock = mock.Mock()
collection1 = RecordsCollection().add_record(
TestRecord("test_record_1")).add_record(
TestRecord("test_record_2"))
collection2 = RecordsCollection().add_record(
TestRecord("test_record_2")).add_record(
TestRecord("test_record_3"))
extractor_mock.get_records.return_value = collection1
repository_mock.load_records.return_value = collection2
self.assertIsNone(
WebSpiderExtractionsProcessor(
crawler_process=crawler_process_mock,
records_collections_notifier=notifier_mock).
register_extraction(title="test",
extractor=extractor_mock,
repository=repository_mock).run())
crawler_process_mock.start.assert_called_once()
notifier_mock.send_notification.assert_called_with([
("test", collection1 - collection2)
])
def test_equality(self):
self.assertEqual(
RecordsCollection().add_record(VinylRecord("a", 1, "l")),
RecordsCollection().add_record(VinylRecord("a", 1, "l")))
self.assertNotEqual(
RecordsCollection().add_record(VinylRecord("a", 1, "l")),
RecordsCollection().add_record(VinylRecord("a", 2, "l")))
def load_records(self) -> RecordsCollection:
"""
:raises InvalidJsonError: if the file contains an invalid JSON
:raises RecordFactoryException: if the record creation fails
"""
records_collection = RecordsCollection()
if not os.path.isfile(self.file_path):
return records_collection
try:
with open(self.file_path) as f:
for record in json.load(f):
records_collection.add_record(
self.record_factory.create_from_dict(record))
return records_collection
except json.JSONDecodeError:
raise InvalidJsonError
def save_records(self, records: RecordsCollection):
with open(self.file_path, "w") as f:
f.write(
json.dumps([
self.record_dict_mapper.get_mapped_record(r)
for r in records.get_records()
]))
def test_subtraction(self):
self.assertEqual(
RecordsCollection().add_record(VinylRecord(
"a", 1, "l")).add_record(VinylRecord("b", 2, "l")) -
RecordsCollection().add_record(VinylRecord("a", 1, "l")),
RecordsCollection().add_record(VinylRecord("b", 2, "l")))
self.assertEqual(
RecordsCollection().add_record(VinylRecord(
"a", 1, "l")).add_record(VinylRecord("b", 2, "l")) -
RecordsCollection().add_record(VinylRecord("c", 1, "l")),
RecordsCollection().add_record(VinylRecord(
"a", 1, "l")).add_record(VinylRecord("b", 2, "l")))
def test_save_success(self):
file_path = self.STORAGE_PATH + "/save_success.json"
repository = JsonFileRecordsRepository(
file_path=file_path,
record_factory=self.record_factory,
record_dict_mapper=self.record_dict_mapper)
repository.save_records(RecordsCollection().add_record(
TestRecord("test 1")).add_record(TestRecord("test 2")))
self.assertTrue(os.path.isfile(file_path))
with open(file_path) as f:
self.assertEqual(json.load(f), [
{
"name": "test 1"
},
{
"name": "test 2"
},
])
def test_contains(self):
collection = RecordsCollection().add_record(VinylRecord("a", 1, "l"))
self.assertIn(VinylRecord("a", 1, "l"), collection)
self.assertNotIn(VinylRecord("a", 2, "l"), collection)
def __init__(self, crawler_process: CrawlerProcess):
self.crawler_process = crawler_process
self.records = RecordsCollection()
self._register_spider()