def test_ship_finder_valid_response_no_eta(self):
resp = response_factory(
os.path.join(FIXTURE_PATH, 'response_mmsi_477744900.no-eta.json'),
klass=FakeTextResponse,
meta={'vessels': self._vessel_dict},
)
for key in ['nextDestination_destination', 'nextDestination_eta']:
del self._item[key]
spider = ShipFinderSpider()
with patch('kp_scrapers.spiders.ais.shipfinder.ShipFinderSpider.logger') as logger_mock:
with patch(VESSELS_MODULE, ForcedCollection(self._list_of_two_vessels)):
for item in spider.parse(resp):
for key in self._item:
self.assertEqual(
item[key],
self._item[key],
'Key {}: {} != {}'.format(key, item[key], self._item[key]),
)
logger_mock.warning.assert_called_with(
"ShipFinder's response for vessel {} contains no ETA.".format(
self._vessel_list[0]['imo']
)
)
def test_ship_finder_valid_response(self):
resp = response_factory(
os.path.join(FIXTURE_PATH, 'response_mmsi_477744900.json'),
klass=FakeTextResponse,
meta={'vessels': self._vessel_dict},
)
spider = ShipFinderSpider()
with patch(VESSELS_MODULE, ForcedCollection(self._list_of_one_vessel)):
spider.parse(resp)
def test_ship_finder_valid_response_with_vessel_twice(self):
resp = response_factory(
os.path.join(FIXTURE_PATH, 'response_twice_mmsi_477744900.json'),
klass=FakeTextResponse,
meta={'vessels': self._vessel_dict},
)
spider = ShipFinderSpider()
with patch(VESSELS_MODULE, ForcedCollection(self._list_of_two_vessels)):
for item in spider.parse(resp):
for key in strip_meta_fields(item):
self.assertEqual(item[key], self._item[key])
def test_ship_finder_valid_response_with_missing_vessel(self):
resp = response_factory(
os.path.join(FIXTURE_PATH, 'response_mmsi_477744900.json'),
klass=FakeTextResponse,
meta={'vessels': self._vessel_dict},
)
with patch(
'kp_scrapers.spiders.ais.shipfinder.ShipFinderSpider.logger', new_callable=PropertyMock
):
spider = ShipFinderSpider()
with patch(VESSELS_MODULE, ForcedCollection(self._list_of_two_vessels)):
spider.parse(resp)
def test_ship_finder_valid_response_bad_eta(self):
# Ship Finder may give bad eta "02-30 00:28" for 30th of February.
# Check that "ValueError: day is out of range for month" does not raise
# when trying to parse date
resp = response_factory(
os.path.join(FIXTURE_PATH, 'response_mmsi_477744900.bad-eta.json'),
klass=FakeTextResponse,
meta={'vessels': self._vessel_dict},
)
spider = ShipFinderSpider()
for item in spider.parse(resp):
pass
def test_ship_finder_invalid_json_response(self):
self._vessel_dict['533189000'] = self._vessel_dict['477744900']
resp = response_factory(
os.path.join(FIXTURE_PATH, 'response_mmsi_533189000-invalid.json'),
klass=FakeTextResponse,
# Content of meta does not really matters here.
meta={'vessels': self._vessel_dict},
)
spider = ShipFinderSpider()
count = 0
for item in spider.parse(resp):
count += 1
self.assertEqual(count, 0)
def test_vessel_added_error(self):
response = response_factory(os.path.join(FIXTURE_PATH,
'imo-not-found.xml'),
meta={'vessel': self.vessel})
with patch(
'kp_scrapers.spiders.ais.marinetraffic.MarineTrafficSpider.logger'
):
spider = MarineTrafficSpider(fleet='True')
spider.check_vessel_was_added(response)
spider.logger.error.assert_called_once_with(
"Could not add vessel {name} (IMO: {imo}, MMSI: {mmsi}): "
"{message}".format(message="VESSEL IMO NOT FOUND IN DATABASE",
**self.vessel))
def test_no_crash_when_both_fleet_and_vessel_list_are_empty(self):
with patch(VESSELS_MODULE, new=ForcedCollection()):
spider = MarineTrafficSpider(
fleet='True',
fleet_name='MT',
setkey='abcdef',
getkey='123456',
allow_removal='true',
)
response = response_factory(
os.path.join(FIXTURE_PATH, 'fleet-empty.xml'))
requests = []
for request in spider.update_fleet(response):
requests.append(request)
self.assertEqual(len(requests), 0)
def test_vessel_successfully_added(self):
response = response_factory(os.path.join(FIXTURE_PATH,
'vessel-added.xml'),
meta={'vessel': self.vessel})
with patch(
'kp_scrapers.spiders.ais.marinetraffic.MarineTrafficSpider.logger'
) as logger:
spider = MarineTrafficSpider(fleet='True')
spider.check_vessel_was_added(response)
self.assertEqual(logger.info.call_count, 1)
self.assertEqual(
logger.info.call_args_list[-1][0][0],
"Vessel {name} (IMO: {imo}, MMSI: {mmsi}) added to MT"
" fleet.".format(message="FLEET ITEM UPDATED", **self.vessel),
)
def test_ship_finder_invalid_http_response(self):
resp = response_factory(
os.path.join(FIXTURE_PATH, 'response_mmsi_477744900.json'),
klass=FakeTextResponse,
meta={'vessels': self._vessel_dict},
)
resp.status = 404
with patch('kp_scrapers.spiders.ais.shipfinder.ShipFinderSpider.logger') as logger_mock:
spider = ShipFinderSpider()
with patch(VESSELS_MODULE, ForcedCollection(self._list_of_two_vessels)):
for item in spider.parse(resp):
pass
logger_mock.error.assert_called_with(
'Bad HTTP response from shipfinder API: {}'.format(resp.body)
)
def test_vessel_removal_when_fleet_empty_vessel_list_contains_one_vessel_other_provider(
self):
self.vessel['providers'] = ['Other Provider']
with patch(VESSELS_MODULE, new=ForcedCollection([self.vessel])):
spider = MarineTrafficSpider(
fleet='True',
fleet_name='MT',
setkey='abcdef',
getkey='123456',
allow_removal='true',
)
response = response_factory(
os.path.join(FIXTURE_PATH, 'fleet-empty.xml'))
requests = []
for request in spider.update_fleet(response):
requests.append(request)
self.assertEqual(len(requests), 0)
def test_vessel_removal_when_fleet_contains_one_vessel_and_vessel_list_empty(
self):
with patch(VESSELS_MODULE, new=ForcedCollection()):
spider = MarineTrafficSpider(fleet='True',
fleet_name='MT',
setkey='abcdef',
getkey='123456',
removal='true')
response = response_factory(os.path.join(FIXTURE_PATH,
'fleet-one-vessel.xml'),
klass=FakeXmlResponse)
requests = []
for request in spider.update_fleet(response):
requests.append(request)
self.assertEqual(len(requests), 1)
request = requests[0]
self.assertIn('action', request.meta)
self.assertEqual(request.meta['action'], 'delete')
self.assertIn('imo:{}'.format(self.vessel['imo']), request.url)
self.assertNotIn('active:1', request.url)
self.assertIn('delete:1', request.url)