import datetime as dt
from schematics.types import DateTimeType, IntType, StringType
from kp_scrapers.models.enum import Enum
from kp_scrapers.models.normalize import BaseEvent
from kp_scrapers.models.validators import is_valid_rate
CommodityType = Enum(lng='lng', lpg='lpg', oil='oil', butane='butane', propane='propane', cpp='cpp')
class StockExchange(BaseEvent):
"""StockExchange model describes the predicted price for a certain commodity in a specific
zone.
REQUIRES ALL of the following fields:
- commodity
- index
- month
- provider_name (as defined in `BaseEvent`)
- raw_unit
- raw_value
- reported_date
- ticker
- zone
Optional fields:
- converted_value
- difference_value
- source_power
from schematics.models import Model
from schematics.types import StringType, UUIDType
import semver
from kp_scrapers import __version__ as _package_version
from kp_scrapers.models.enum import Enum
DataTypes = Enum(
Ais='ais',
BillOfLading='bill_of_lading',
BunkerFuelCost='bunker_fuel_cost',
Cargo='cargo',
CargoMovement='cargo_movement',
CustomsFigure='customs_figure',
CustomsPortCall='customs_portcall',
ExchangeRate='exchange_rate',
MarketFigure='market_figure',
PortCall='portcall',
SpotCharter='spotcharter',
StockExchange='stock_exchange',
Vessel='vessel',
TropicalStorm='tropical_storm',
)
class BaseEvent(Model):
"""Initialise meta fields for an event.
NOTE this model should ideally also document the presence of `scrapy-magicfields`,
since they are required by the ETL:
- sh_item_time : str('%Y-%m-%dT%H:%M:%S.%f')
Unit = Enum(
# Dimension
meter='meter',
kilometer='kilometer',
# Mass
gram='gram',
kilogram='kilogram',
tons='tons',
kilotons='kilotons',
megatons='megatons',
# Volume
liter='liter',
kiloliter='kiloliter',
cubic_meter=
'cubic_meter', # `cm` is not used to avoid namespace conflict with `centimeter`
cubic_feet='cubic_feet',
barrel='barrel',
kilobarrel='kilobarrel',
megabarrel='megabarrel',
# Time
second='second',
minute='minute',
hour='hour',
day='day',
week='week',
month='month',
year='year',
# Power
watt='W',
# Energy
joule='joule',
Btu='Btu', # british thermal unit
therm='therm', # equal to 100000 Btu
watt_hour='Wh',
kilowatt_hour='kWh',
megawatt_hour='MWh',
# Force
newton='N',
# Pressure
pascal='Pa',
# Temperature
celsius='C',
kelvin='K',
# Mass Flow Rate
tons_per_annum='tpa',
# Speed
knots='knots',
)
from schematics.types import DateTimeType, FloatType, StringType
from kp_scrapers.lib.date import ISO8601_FORMAT
from kp_scrapers.models.enum import Enum
from kp_scrapers.models.normalize import BaseEvent
FuelType = Enum(
IFO380='IFO380',
MGO='MGO',
)
class BunkerFuelCost(BaseEvent):
"""Bunker fuel oil prices
It is used by the voyage-calculator to estimate the total fuel cost during a
voyage. There is two type of fuel, IF0380 and MGO. Due to regulation, vessels
have to use one or the other, depending on the water they sails. We use the
price of "Global 4 Ports Average" as an average of the world bunker price.
REQUIRES ALL of the following fields:
- reported_date
- price
- type
- zone
"""
fuel_type = StringType(metadata='Type of fuel',
required=True,
choices=[value for _, value in FuelType])
price = FloatType(
from schematics.exceptions import ValidationError
from schematics.types import DateTimeType, FloatType, StringType
from kp_scrapers.lib.date import ISO8601_FORMAT
from kp_scrapers.models.enum import Enum
from kp_scrapers.models.normalize import BaseEvent
from kp_scrapers.models.units import Unit
BalanceType = Enum(
import_='Import',
export='Export',
stock_change='Stock change',
refinery_intake='Refinery intake',
direct_use='Direct use',
production='Production',
ending_stocks='Ending stocks',
)
CountryType = Enum(country_checkpoint='country_checkpoint',
country='country',
region='region',
custom='custom')
class MarketFigure(BaseEvent):
"""Describe a market figure schema.
REQUIRES ALL of the following fields:
- balance
- country
- country_type
from kp_scrapers.models.normalize import DataTypes
from kp_scrapers.spiders.tropical_storm import TropicalStormSpider
from kp_scrapers.spiders.tropical_storm.jtwc import normalize
from kp_scrapers.spiders.tropical_storm.jtwc.parser import CycloneKMLParser
# NOTE sounds like something in utils/date.py
def utc_epoch(past_days=0):
target_dt = dt.datetime.utcnow() - dt.timedelta(days=past_days)
return int(time.mktime(target_dt.timetuple()) * 1000)
# supported doc type
# as defined on http://www.metoc.navy.mil/jtwc/jtwc.html
JTWCReportNames = Enum(
tcfa='tcfa_doc.kml',
tc='doc.kml' # Tropical Cyclone Formation # Tropical Depression
)
# TODO Inherit from an RSS parser
# NOTE there is a storm report as well: http://www.metoc.navy.mil/fwcn/fwcn.html#!/geospatial_kml.html # noqa
class JtwcSpider(TropicalStormSpider):
name = 'JTWC'
version = '2.0.0'
provider = 'JTWC'
produces = [DataTypes.TropicalStorm]
BASE_URL = 'https://www.metoc.navy.mil/jtwc/rss/jtwc.rss'
reported_date = (dt.datetime.utcnow().replace(
hour=0, minute=0, second=0).isoformat(timespec='seconds'))
def setUp(self):
self.earth = Enum(
age='4.5 billion years',
continents=7,
oceans=['Atlantic', 'Arctic', 'Indian', 'Pacific', 'Southern'],
)
from schematics.exceptions import ValidationError
from schematics.types import DateTimeType, ListType, ModelType, StringType
from kp_scrapers.lib.date import ISO8601_FORMAT
from kp_scrapers.models.cargo import Cargo
from kp_scrapers.models.enum import Enum
from kp_scrapers.models.normalize import BaseEvent
from kp_scrapers.models.vessel import Vessel
# synced with `etl.orm.db_scheme.charter.(Raw)SpotCharterStatus`
SpotCharterStatus = Enum(
on_subs='On Subs', # charter is under negotiation
fully_fixed='Fully Fixed', # charter has been agreed upon on both parties
in_progress='In Progress', # charter has started
finished='Finished', # charter has been executed and is no longer in effect
cancelled='Cancelled', # charter has been cancelled
failed='Failed', # charter was never agreed by both players
replaced='Replaced', # charter will happen but for another vessel
updated='Updated', # charter is the same but some data changed
)
class SpotCharter(BaseEvent):
"""Describe a spot charter schema.
REQUIRES ALL of the following fields:
- charterer
- departure_zone
- lay_can_start
- provider_name (as defined in `BaseEvent`)
- reported_date
IntType,
ListType,
ModelType,
StringType,
)
from kp_scrapers.lib.date import ISO8601_FORMAT
from kp_scrapers.models.enum import Enum
from kp_scrapers.models.normalize import BaseEvent
from kp_scrapers.models.validators import is_positive_number, is_valid_build_year, is_valid_imo
# TODO it deserves to be elsewhere
PlayerRole = Enum(
builder='Builder',
insurer='Insurer',
ism_manager='ISM Manager',
operator='Operator',
owner='Registered owner',
ship_manager='Ship manager/Commercial manager',
)
VesselStatus = Enum(
broken_up='Broken Up',
cancelled_order='Cancelled Order',
converting='Converting/Rebuilding',
existence_in_doubt='Continued Existence In Doubt',
in_casualty='In Casualty Or Repairing',
in_service='In Service/Commission',
laid_up='Laid-Up',
launched='Launched',
on_order='On Order/Under Construction',
to_be_broken_up='To Be Broken Up',
from schematics.models import Model
from schematics.types import DateTimeType, FloatType, IntType, ModelType, StringType
from kp_scrapers.lib.date import ISO8601_FORMAT
from kp_scrapers.models.enum import Enum
from kp_scrapers.models.normalize import BaseEvent
from kp_scrapers.models.vessel import Vessel
AisType = Enum(dynamic='D-AIS', satellite='S-AIS', terrestrial='T-AIS')
class Position(Model):
"""Describe an position model for AIS message.
REQUIRES ALL of the following fields:
- ais_type
- course
- lat
- lon
- received_time
- speed
Optional fields:
- draught
- draught_raw
- heading
- nav_state
"""
ais_type = StringType(