def test_vessels_collection_from_local_cache(self): static_data._BASE_LOCAL_CACHE = fixtures_path() fleet = static_data.vessels() self.assertTrue(isinstance(fleet, static_data.Collection)) for vessel in fleet: self.assertTrue(vessel.get('imo')) static_data._BASE_LOCAL_CACHE = ORIGINAL_CACHE_PATH
def test_parse_terminated_port_call_file(self): filename = fixtures_path('port_authorities', 'ferrol', 'FINALIZADOS 19.02.15.txt') with open(filename, 'r') as f: table = FerrolTable(f.read(), filename, Mock()) res = table.parse() for item in res: print('*** Result: ', item)
def test_parse_initiated_port_call_file2(self): filename = fixtures_path('port_authorities', 'ferrol', 'INICIADOS 24.02.2015.txt') with open(filename, 'r') as f: table = FerrolTable(f.read(), filename, Mock()) res = table.parse() for item in res: print('*** Result: ', item)
def test_parse_port_call_enquiry_file(self): filename = fixtures_path('port_authorities', 'ferrol', 'SOLICITADOS 19.02,15.txt') with open(filename, 'r') as f: table = FerrolTable(f.read(), filename, Mock()) res = table.parse() for item in res: print('*** Result: ', item)
def test_first_pass_port_names(self): name = "VPR-08-04-2020.PDF" filename = fixtures_path('port_authorities', 'benlines', name) content = Path(filename).read_bytes() logger.info(content) spider = BenlinesSpider() port_names = spider.extract_port_names(content) self.assertEqual(40, len(port_names)) self.assertEqual(port_names[12], "HAZIRA(ADANI)") self.assertEqual(port_names[28], "MUMBAI (JNPT)") self.assertEqual(port_names[38], "VADINAR")
def test_slot_grain(): fixture_path = fixtures_path('slot', 'grain.html') spider = SlotGrainSpider() with open(fixture_path) as f: response = HtmlResponse(BASE_URL, status=200, body=f.read(), encoding='utf-8') results = [ { 'date': '2018-07-21T00:00:00', 'installation_id': 3513, 'seller': '' }, { 'date': '2018-07-22T00:00:00', 'installation_id': 3513, 'seller': '' }, { 'date': '2018-07-24T00:00:00', 'installation_id': 3513, 'seller': '' }, { 'date': '2018-07-27T00:00:00', 'installation_id': 3513, 'seller': '' }, { 'date': '2018-07-28T00:00:00', 'installation_id': 3513, 'seller': '' }, { 'date': '2018-07-29T00:00:00', 'installation_id': 3513, 'seller': '' }, ] computed = list(spider.parse(response)) for k, e in enumerate(results): for rk, rv in e.items(): nt.assert_equal(rv, computed[k][rk])
def test_zeebrugge_spider(self): # given with open(fixtures_path('slot', 'zeebrugge.html')) as source: response = HtmlResponse(url=self.start_url, status=200, body=source.read(), encoding='utf-8') # when results = [ _trim_item(item, 'date', 'installation_id', 'seller') for item in self.spider.parse(response) ] # then self.assertCountEqual(results, [{ 'date': '2019-10-07', 'installation_id': 3430, 'seller': 'FLUXYSLNG' }])
def test_uk_port_operator_sentout(): fixture_path = fixtures_path('operator', 'uk_sentout', 'response.csv') mock_data_path = Mock(return_value="test.json") with patch('kp_scrapers.spiders.bases.persist_data_manager.data_path', new=mock_data_path): spider = UKOperatorHistoricalSendOutSpider() with open(fixture_path) as f: body = f.read() grouped_results = {} for e in RESULTS_TABLE: grouped_results.setdefault(e['date'], {}).setdefault(e['city'], e['output_o']) spider.parse_csv(body) computed = list(spider.parse_csv(body)) nt.assert_equal(len(computed), len(RESULTS_TABLE)) for e in computed: nt.assert_equal(float(e['output_o']), grouped_results[e['date']][e['city']])
from unittest import TestCase from unittest.mock import patch from scrapy import Selector from kp_scrapers.models.base import strip_meta_fields from kp_scrapers.spiders.ais.marinetraffic import MarineTrafficSpider from tests._helpers.mocks import ( FakeResponse, FakeXmlResponse, fixtures_path, ForcedCollection, response_factory, ) FIXTURE_PATH = fixtures_path('ais', 'marinetraffic') VESSELS_MODULE = 'kp_scrapers.lib.static_data.vessels' class MarineTrafficTestCase(TestCase): def setUp(self): self.vessel = { 'imo': '0000000', 'name': 'Some Vessel', 'mmsi': '111111111', 'status': 'Active', 'status_detail': 'In Service/Commission', } self.api_key = '123456' def test_parse_response(self):
# -*- coding: utf-8; -*- from __future__ import absolute_import, unicode_literals from datetime import datetime from unittest import TestCase from unittest.mock import patch, PropertyMock from kp_scrapers.models.base import strip_meta_fields from kp_scrapers.spiders.ais.vesselfinder import KEY_MAP, VesselFinderApi from tests._helpers.date import DateTimeWithChosenNow from tests._helpers.mocks import fixtures_path, inject_fixture, response_factory FIXTURE_PATH = fixtures_path('ais', 'vesselfinder') class VesselFinderResponseTestCase(TestCase): def test_key_map_haeding_remove_bad_value(self): _, transformer = KEY_MAP['HEADING'] # we're good, give us as is self.assertEqual(transformer(200), 200) # we're good, give as an int self.assertEqual(transformer('200'), 200) # above 360°, remove that self.assertIsNone(transformer(4000)) class VesselFinderApiTestCase(TestCase): def setUp(self): self.response = { 'provider_id': 'VF API', 'aisType': 'T-AIS',
# -*- coding: utf-8; -*- from __future__ import absolute_import, unicode_literals import os import unittest from unittest.mock import patch from kp_scrapers.models.base import strip_meta_fields from kp_scrapers.spiders.ais.vesseltracker import VesselTrackerSpider from tests._helpers.mocks import fixtures_path FIXTURE_PATH = fixtures_path('ais', 'vesseltracker') class FakeSoapResponse(object): def __init__(self, filename): """Mocks soap method call response. Parameters ---------- filename (str): a file from which load the XML data to use as the response from a SOAP method call. """ with open(os.path.join(FIXTURE_PATH, filename), 'r') as f: self._xml_response = f.read() def as_xml(self): return self._xml_response
import datetime as dt import os from unittest import TestCase from scrapy import Selector import six from kp_scrapers.spiders.ais.exactearth import api, constants, parser, spider from tests._helpers.mocks import FakeXmlResponse, fixtures_path, MockStaticData MODULE_PATH = os.path.dirname(os.path.dirname(__file__)) FIXTURES_PATH = fixtures_path('ais', 'exactearth') FIXTURE_RESPONSE_PATH = os.path.join(FIXTURES_PATH, 'latest-vessel-info.xml') FIXTURE_VESSEL = { "ais_type": "T-AIS", "message_type": "1", "next_destination_eta": '{y}-06-17T02:00:00'.format(y=dt.datetime.utcnow().year), "next_destination_ais_type": "T-AIS", "next_destination_destination": "SINGAPORE", "position": { "ais_type": "T-AIS", "course": 207.4, "heading": 206.0, "lat": 29.678498333333334, "lon": 126.184325, "nav_state": 0,
def test_1(self): name = "VPR-08-04-2020.PDF" filename = fixtures_path('port_authorities', 'benlines', name) content = Path(filename).read_bytes() spider = BenlinesSpider() port_names = spider.extract_port_names(content) rows = spider.extract_port_data(content) data = {} for row in rows: data[row["vessel"]["name"]] = row print(data) # port_name=row[0] # table_id=row[1] # info=row[2] # if data.get(port_name,None) is None: # data[port_name] = {} # if data[port_name].get(table_id,None) is None: #self.assertListEqual(sorted(port_names),sorted(list(data.keys()))) self.assertEqual(566, len(data.keys())) d = data["CERVIA"] self.assertEqual("SANDHEAD ANCHORAGE", d["berth"]) d = data["WEI LUN JU LONG"] self.assertEqual("8", d["berth"]) self.assertEqual("discharge", d["cargoes"][0]["movement"]) self.assertEqual("COKING COAL", d["cargoes"][0]["product"]) self.assertEqual(26803, float(d["cargoes"][0]["volume"])) d = data["JAG PRABHA"] self.assertEqual("Q4", d["berth"]) self.assertEqual(2, len(d["cargoes"])) self.assertEqual("discharge", d["cargoes"][1]["movement"]) self.assertEqual("HIGH SPEED DIESEL (HSD )", d["cargoes"][1]["product"]) self.assertEqual(8000, float(d["cargoes"][1]["volume"])) d = data["FRANCESCO CORRADO"] self.assertEqual("discharge", d["cargoes"][0]["movement"]) self.assertEqual("STEAM COAL", d["cargoes"][0]["product"]) self.assertEqual(71619.540, float(d["cargoes"][0]["volume"])) d = data["JAOHAR RANIM"] self.assertEqual("CJ-2", d["berth"]) self.assertEqual(2, len(d["cargoes"])) self.assertEqual("load", d["cargoes"][1]["movement"]) self.assertEqual("SUGAR", d["cargoes"][1]["product"]) self.assertEqual(0, float(d["cargoes"][1]["volume"])) d = data["LPG/C BERLIAN EKUATOR"] self.assertEqual(2, len(d["cargoes"])) self.assertEqual("discharge", d["cargoes"][1]["movement"]) self.assertEqual("BUTANE", d["cargoes"][1]["product"]) self.assertEqual(9982, float(d["cargoes"][1]["volume"])) d = data["STOLT PONDO"] self.assertEqual(1, len(d["cargoes"])) self.assertEqual("discharge", d["cargoes"][0]["movement"]) self.assertEqual("PALM FATTY ACID", d["cargoes"][0]["product"]) self.assertEqual(29499, float(d["cargoes"][0]["volume"]))
# -*- coding: utf-8; -*- from __future__ import absolute_import, unicode_literals from datetime import datetime import os from unittest import skip, TestCase from unittest.mock import patch, PropertyMock import pytz from kp_scrapers.models.base import strip_meta_fields from kp_scrapers.spiders.ais.shipfinder import ShipFinderSpider from tests._helpers.mocks import FakeTextResponse, fixtures_path, ForcedCollection, response_factory FIXTURE_PATH = fixtures_path('ais', 'shipfinder') VESSELS_MODULE = 'kp_scrapers.lib.static_data.vessels' @skip('shipfinder source is deprecated') class ShipFinderTestCase(TestCase): def setUp(self): self._vessel_list = [ {'name': 'KIKYO', 'imo': '9415703', 'mmsi': '477744900', 'callsign': 'VRGR7'} ] self._vessel_dict = {v['mmsi']: v for v in self._vessel_list} # The content of the item expected to be yielded by the parse function. self._item = { 'master_name': 'KIKYO', 'master_imo': '9415703', 'master_mmsi': '477744900',
def _get_table(self, name): filename = fixtures_path('port_authorities', 'ferrol', name) with open(filename, 'r') as f: table = FerrolTable(f.read(), filename, Mock()) # Spider return table
def setUp(self): fake_file = fixtures_path('weather', 'jtwc_doc.kml') self.mock_cyclones_index = FakeXmlResponse(fake_file) DateTimeWithChosenNow.choosen_now = dt.datetime(2017, 6, 12, 0, 0)