def test_vessels_collection_from_local_cache(self):
static_data._BASE_LOCAL_CACHE = fixtures_path()
fleet = static_data.vessels()
self.assertTrue(isinstance(fleet, static_data.Collection))
for vessel in fleet:
self.assertTrue(vessel.get('imo'))
static_data._BASE_LOCAL_CACHE = ORIGINAL_CACHE_PATH
def test_parse_terminated_port_call_file(self):
filename = fixtures_path('port_authorities', 'ferrol',
'FINALIZADOS 19.02.15.txt')
with open(filename, 'r') as f:
table = FerrolTable(f.read(), filename, Mock())
res = table.parse()
for item in res:
print('*** Result: ', item)
def test_parse_initiated_port_call_file2(self):
filename = fixtures_path('port_authorities', 'ferrol',
'INICIADOS 24.02.2015.txt')
with open(filename, 'r') as f:
table = FerrolTable(f.read(), filename, Mock())
res = table.parse()
for item in res:
print('*** Result: ', item)
def test_parse_port_call_enquiry_file(self):
filename = fixtures_path('port_authorities', 'ferrol',
'SOLICITADOS 19.02,15.txt')
with open(filename, 'r') as f:
table = FerrolTable(f.read(), filename, Mock())
res = table.parse()
for item in res:
print('*** Result: ', item)
def test_first_pass_port_names(self):
name = "VPR-08-04-2020.PDF"
filename = fixtures_path('port_authorities', 'benlines', name)
content = Path(filename).read_bytes()
logger.info(content)
spider = BenlinesSpider()
port_names = spider.extract_port_names(content)
self.assertEqual(40, len(port_names))
self.assertEqual(port_names[12], "HAZIRA(ADANI)")
self.assertEqual(port_names[28], "MUMBAI (JNPT)")
self.assertEqual(port_names[38], "VADINAR")
def test_slot_grain():
fixture_path = fixtures_path('slot', 'grain.html')
spider = SlotGrainSpider()
with open(fixture_path) as f:
response = HtmlResponse(BASE_URL,
status=200,
body=f.read(),
encoding='utf-8')
results = [
{
'date': '2018-07-21T00:00:00',
'installation_id': 3513,
'seller': ''
},
{
'date': '2018-07-22T00:00:00',
'installation_id': 3513,
'seller': ''
},
{
'date': '2018-07-24T00:00:00',
'installation_id': 3513,
'seller': ''
},
{
'date': '2018-07-27T00:00:00',
'installation_id': 3513,
'seller': ''
},
{
'date': '2018-07-28T00:00:00',
'installation_id': 3513,
'seller': ''
},
{
'date': '2018-07-29T00:00:00',
'installation_id': 3513,
'seller': ''
},
]
computed = list(spider.parse(response))
for k, e in enumerate(results):
for rk, rv in e.items():
nt.assert_equal(rv, computed[k][rk])
def test_zeebrugge_spider(self):
# given
with open(fixtures_path('slot', 'zeebrugge.html')) as source:
response = HtmlResponse(url=self.start_url,
status=200,
body=source.read(),
encoding='utf-8')
# when
results = [
_trim_item(item, 'date', 'installation_id', 'seller')
for item in self.spider.parse(response)
]
# then
self.assertCountEqual(results, [{
'date': '2019-10-07',
'installation_id': 3430,
'seller': 'FLUXYSLNG'
}])
def test_uk_port_operator_sentout():
fixture_path = fixtures_path('operator', 'uk_sentout', 'response.csv')
mock_data_path = Mock(return_value="test.json")
with patch('kp_scrapers.spiders.bases.persist_data_manager.data_path',
new=mock_data_path):
spider = UKOperatorHistoricalSendOutSpider()
with open(fixture_path) as f:
body = f.read()
grouped_results = {}
for e in RESULTS_TABLE:
grouped_results.setdefault(e['date'],
{}).setdefault(e['city'], e['output_o'])
spider.parse_csv(body)
computed = list(spider.parse_csv(body))
nt.assert_equal(len(computed), len(RESULTS_TABLE))
for e in computed:
nt.assert_equal(float(e['output_o']),
grouped_results[e['date']][e['city']])
from unittest import TestCase
from unittest.mock import patch
from scrapy import Selector
from kp_scrapers.models.base import strip_meta_fields
from kp_scrapers.spiders.ais.marinetraffic import MarineTrafficSpider
from tests._helpers.mocks import (
FakeResponse,
FakeXmlResponse,
fixtures_path,
ForcedCollection,
response_factory,
)
FIXTURE_PATH = fixtures_path('ais', 'marinetraffic')
VESSELS_MODULE = 'kp_scrapers.lib.static_data.vessels'
class MarineTrafficTestCase(TestCase):
def setUp(self):
self.vessel = {
'imo': '0000000',
'name': 'Some Vessel',
'mmsi': '111111111',
'status': 'Active',
'status_detail': 'In Service/Commission',
}
self.api_key = '123456'
def test_parse_response(self):
# -*- coding: utf-8; -*-
from __future__ import absolute_import, unicode_literals
from datetime import datetime
from unittest import TestCase
from unittest.mock import patch, PropertyMock
from kp_scrapers.models.base import strip_meta_fields
from kp_scrapers.spiders.ais.vesselfinder import KEY_MAP, VesselFinderApi
from tests._helpers.date import DateTimeWithChosenNow
from tests._helpers.mocks import fixtures_path, inject_fixture, response_factory
FIXTURE_PATH = fixtures_path('ais', 'vesselfinder')
class VesselFinderResponseTestCase(TestCase):
def test_key_map_haeding_remove_bad_value(self):
_, transformer = KEY_MAP['HEADING']
# we're good, give us as is
self.assertEqual(transformer(200), 200)
# we're good, give as an int
self.assertEqual(transformer('200'), 200)
# above 360°, remove that
self.assertIsNone(transformer(4000))
class VesselFinderApiTestCase(TestCase):
def setUp(self):
self.response = {
'provider_id': 'VF API',
'aisType': 'T-AIS',
# -*- coding: utf-8; -*-
from __future__ import absolute_import, unicode_literals
import os
import unittest
from unittest.mock import patch
from kp_scrapers.models.base import strip_meta_fields
from kp_scrapers.spiders.ais.vesseltracker import VesselTrackerSpider
from tests._helpers.mocks import fixtures_path
FIXTURE_PATH = fixtures_path('ais', 'vesseltracker')
class FakeSoapResponse(object):
def __init__(self, filename):
"""Mocks soap method call response.
Parameters
----------
filename (str): a file from which load the XML data to use
as the response from a SOAP method call.
"""
with open(os.path.join(FIXTURE_PATH, filename), 'r') as f:
self._xml_response = f.read()
def as_xml(self):
return self._xml_response
import datetime as dt
import os
from unittest import TestCase
from scrapy import Selector
import six
from kp_scrapers.spiders.ais.exactearth import api, constants, parser, spider
from tests._helpers.mocks import FakeXmlResponse, fixtures_path, MockStaticData
MODULE_PATH = os.path.dirname(os.path.dirname(__file__))
FIXTURES_PATH = fixtures_path('ais', 'exactearth')
FIXTURE_RESPONSE_PATH = os.path.join(FIXTURES_PATH, 'latest-vessel-info.xml')
FIXTURE_VESSEL = {
"ais_type":
"T-AIS",
"message_type":
"1",
"next_destination_eta":
'{y}-06-17T02:00:00'.format(y=dt.datetime.utcnow().year),
"next_destination_ais_type":
"T-AIS",
"next_destination_destination":
"SINGAPORE",
"position": {
"ais_type": "T-AIS",
"course": 207.4,
"heading": 206.0,
"lat": 29.678498333333334,
"lon": 126.184325,
"nav_state": 0,
def test_1(self):
name = "VPR-08-04-2020.PDF"
filename = fixtures_path('port_authorities', 'benlines', name)
content = Path(filename).read_bytes()
spider = BenlinesSpider()
port_names = spider.extract_port_names(content)
rows = spider.extract_port_data(content)
data = {}
for row in rows:
data[row["vessel"]["name"]] = row
print(data)
# port_name=row[0]
# table_id=row[1]
# info=row[2]
# if data.get(port_name,None) is None:
# data[port_name] = {}
# if data[port_name].get(table_id,None) is None:
#self.assertListEqual(sorted(port_names),sorted(list(data.keys())))
self.assertEqual(566, len(data.keys()))
d = data["CERVIA"]
self.assertEqual("SANDHEAD ANCHORAGE", d["berth"])
d = data["WEI LUN JU LONG"]
self.assertEqual("8", d["berth"])
self.assertEqual("discharge", d["cargoes"][0]["movement"])
self.assertEqual("COKING COAL", d["cargoes"][0]["product"])
self.assertEqual(26803, float(d["cargoes"][0]["volume"]))
d = data["JAG PRABHA"]
self.assertEqual("Q4", d["berth"])
self.assertEqual(2, len(d["cargoes"]))
self.assertEqual("discharge", d["cargoes"][1]["movement"])
self.assertEqual("HIGH SPEED DIESEL (HSD )",
d["cargoes"][1]["product"])
self.assertEqual(8000, float(d["cargoes"][1]["volume"]))
d = data["FRANCESCO CORRADO"]
self.assertEqual("discharge", d["cargoes"][0]["movement"])
self.assertEqual("STEAM COAL", d["cargoes"][0]["product"])
self.assertEqual(71619.540, float(d["cargoes"][0]["volume"]))
d = data["JAOHAR RANIM"]
self.assertEqual("CJ-2", d["berth"])
self.assertEqual(2, len(d["cargoes"]))
self.assertEqual("load", d["cargoes"][1]["movement"])
self.assertEqual("SUGAR", d["cargoes"][1]["product"])
self.assertEqual(0, float(d["cargoes"][1]["volume"]))
d = data["LPG/C BERLIAN EKUATOR"]
self.assertEqual(2, len(d["cargoes"]))
self.assertEqual("discharge", d["cargoes"][1]["movement"])
self.assertEqual("BUTANE", d["cargoes"][1]["product"])
self.assertEqual(9982, float(d["cargoes"][1]["volume"]))
d = data["STOLT PONDO"]
self.assertEqual(1, len(d["cargoes"]))
self.assertEqual("discharge", d["cargoes"][0]["movement"])
self.assertEqual("PALM FATTY ACID", d["cargoes"][0]["product"])
self.assertEqual(29499, float(d["cargoes"][0]["volume"]))
# -*- coding: utf-8; -*-
from __future__ import absolute_import, unicode_literals
from datetime import datetime
import os
from unittest import skip, TestCase
from unittest.mock import patch, PropertyMock
import pytz
from kp_scrapers.models.base import strip_meta_fields
from kp_scrapers.spiders.ais.shipfinder import ShipFinderSpider
from tests._helpers.mocks import FakeTextResponse, fixtures_path, ForcedCollection, response_factory
FIXTURE_PATH = fixtures_path('ais', 'shipfinder')
VESSELS_MODULE = 'kp_scrapers.lib.static_data.vessels'
@skip('shipfinder source is deprecated')
class ShipFinderTestCase(TestCase):
def setUp(self):
self._vessel_list = [
{'name': 'KIKYO', 'imo': '9415703', 'mmsi': '477744900', 'callsign': 'VRGR7'}
]
self._vessel_dict = {v['mmsi']: v for v in self._vessel_list}
# The content of the item expected to be yielded by the parse function.
self._item = {
'master_name': 'KIKYO',
'master_imo': '9415703',
'master_mmsi': '477744900',
def _get_table(self, name):
filename = fixtures_path('port_authorities', 'ferrol', name)
with open(filename, 'r') as f:
table = FerrolTable(f.read(), filename, Mock()) # Spider
return table
def setUp(self):
fake_file = fixtures_path('weather', 'jtwc_doc.kml')
self.mock_cyclones_index = FakeXmlResponse(fake_file)
DateTimeWithChosenNow.choosen_now = dt.datetime(2017, 6, 12, 0, 0)