import datetime as dt from schematics.types import DateTimeType, IntType, StringType from kp_scrapers.models.enum import Enum from kp_scrapers.models.normalize import BaseEvent from kp_scrapers.models.validators import is_valid_rate CommodityType = Enum(lng='lng', lpg='lpg', oil='oil', butane='butane', propane='propane', cpp='cpp') class StockExchange(BaseEvent): """StockExchange model describes the predicted price for a certain commodity in a specific zone. REQUIRES ALL of the following fields: - commodity - index - month - provider_name (as defined in `BaseEvent`) - raw_unit - raw_value - reported_date - ticker - zone Optional fields: - converted_value - difference_value - source_power
from schematics.models import Model from schematics.types import StringType, UUIDType import semver from kp_scrapers import __version__ as _package_version from kp_scrapers.models.enum import Enum DataTypes = Enum( Ais='ais', BillOfLading='bill_of_lading', BunkerFuelCost='bunker_fuel_cost', Cargo='cargo', CargoMovement='cargo_movement', CustomsFigure='customs_figure', CustomsPortCall='customs_portcall', ExchangeRate='exchange_rate', MarketFigure='market_figure', PortCall='portcall', SpotCharter='spotcharter', StockExchange='stock_exchange', Vessel='vessel', TropicalStorm='tropical_storm', ) class BaseEvent(Model): """Initialise meta fields for an event. NOTE this model should ideally also document the presence of `scrapy-magicfields`, since they are required by the ETL: - sh_item_time : str('%Y-%m-%dT%H:%M:%S.%f')
Unit = Enum( # Dimension meter='meter', kilometer='kilometer', # Mass gram='gram', kilogram='kilogram', tons='tons', kilotons='kilotons', megatons='megatons', # Volume liter='liter', kiloliter='kiloliter', cubic_meter= 'cubic_meter', # `cm` is not used to avoid namespace conflict with `centimeter` cubic_feet='cubic_feet', barrel='barrel', kilobarrel='kilobarrel', megabarrel='megabarrel', # Time second='second', minute='minute', hour='hour', day='day', week='week', month='month', year='year', # Power watt='W', # Energy joule='joule', Btu='Btu', # british thermal unit therm='therm', # equal to 100000 Btu watt_hour='Wh', kilowatt_hour='kWh', megawatt_hour='MWh', # Force newton='N', # Pressure pascal='Pa', # Temperature celsius='C', kelvin='K', # Mass Flow Rate tons_per_annum='tpa', # Speed knots='knots', )
from schematics.types import DateTimeType, FloatType, StringType from kp_scrapers.lib.date import ISO8601_FORMAT from kp_scrapers.models.enum import Enum from kp_scrapers.models.normalize import BaseEvent FuelType = Enum( IFO380='IFO380', MGO='MGO', ) class BunkerFuelCost(BaseEvent): """Bunker fuel oil prices It is used by the voyage-calculator to estimate the total fuel cost during a voyage. There is two type of fuel, IF0380 and MGO. Due to regulation, vessels have to use one or the other, depending on the water they sails. We use the price of "Global 4 Ports Average" as an average of the world bunker price. REQUIRES ALL of the following fields: - reported_date - price - type - zone """ fuel_type = StringType(metadata='Type of fuel', required=True, choices=[value for _, value in FuelType]) price = FloatType(
from schematics.exceptions import ValidationError from schematics.types import DateTimeType, FloatType, StringType from kp_scrapers.lib.date import ISO8601_FORMAT from kp_scrapers.models.enum import Enum from kp_scrapers.models.normalize import BaseEvent from kp_scrapers.models.units import Unit BalanceType = Enum( import_='Import', export='Export', stock_change='Stock change', refinery_intake='Refinery intake', direct_use='Direct use', production='Production', ending_stocks='Ending stocks', ) CountryType = Enum(country_checkpoint='country_checkpoint', country='country', region='region', custom='custom') class MarketFigure(BaseEvent): """Describe a market figure schema. REQUIRES ALL of the following fields: - balance - country - country_type
from kp_scrapers.models.normalize import DataTypes from kp_scrapers.spiders.tropical_storm import TropicalStormSpider from kp_scrapers.spiders.tropical_storm.jtwc import normalize from kp_scrapers.spiders.tropical_storm.jtwc.parser import CycloneKMLParser # NOTE sounds like something in utils/date.py def utc_epoch(past_days=0): target_dt = dt.datetime.utcnow() - dt.timedelta(days=past_days) return int(time.mktime(target_dt.timetuple()) * 1000) # supported doc type # as defined on http://www.metoc.navy.mil/jtwc/jtwc.html JTWCReportNames = Enum( tcfa='tcfa_doc.kml', tc='doc.kml' # Tropical Cyclone Formation # Tropical Depression ) # TODO Inherit from an RSS parser # NOTE there is a storm report as well: http://www.metoc.navy.mil/fwcn/fwcn.html#!/geospatial_kml.html # noqa class JtwcSpider(TropicalStormSpider): name = 'JTWC' version = '2.0.0' provider = 'JTWC' produces = [DataTypes.TropicalStorm] BASE_URL = 'https://www.metoc.navy.mil/jtwc/rss/jtwc.rss' reported_date = (dt.datetime.utcnow().replace( hour=0, minute=0, second=0).isoformat(timespec='seconds'))
def setUp(self): self.earth = Enum( age='4.5 billion years', continents=7, oceans=['Atlantic', 'Arctic', 'Indian', 'Pacific', 'Southern'], )
from schematics.exceptions import ValidationError from schematics.types import DateTimeType, ListType, ModelType, StringType from kp_scrapers.lib.date import ISO8601_FORMAT from kp_scrapers.models.cargo import Cargo from kp_scrapers.models.enum import Enum from kp_scrapers.models.normalize import BaseEvent from kp_scrapers.models.vessel import Vessel # synced with `etl.orm.db_scheme.charter.(Raw)SpotCharterStatus` SpotCharterStatus = Enum( on_subs='On Subs', # charter is under negotiation fully_fixed='Fully Fixed', # charter has been agreed upon on both parties in_progress='In Progress', # charter has started finished='Finished', # charter has been executed and is no longer in effect cancelled='Cancelled', # charter has been cancelled failed='Failed', # charter was never agreed by both players replaced='Replaced', # charter will happen but for another vessel updated='Updated', # charter is the same but some data changed ) class SpotCharter(BaseEvent): """Describe a spot charter schema. REQUIRES ALL of the following fields: - charterer - departure_zone - lay_can_start - provider_name (as defined in `BaseEvent`) - reported_date
IntType, ListType, ModelType, StringType, ) from kp_scrapers.lib.date import ISO8601_FORMAT from kp_scrapers.models.enum import Enum from kp_scrapers.models.normalize import BaseEvent from kp_scrapers.models.validators import is_positive_number, is_valid_build_year, is_valid_imo # TODO it deserves to be elsewhere PlayerRole = Enum( builder='Builder', insurer='Insurer', ism_manager='ISM Manager', operator='Operator', owner='Registered owner', ship_manager='Ship manager/Commercial manager', ) VesselStatus = Enum( broken_up='Broken Up', cancelled_order='Cancelled Order', converting='Converting/Rebuilding', existence_in_doubt='Continued Existence In Doubt', in_casualty='In Casualty Or Repairing', in_service='In Service/Commission', laid_up='Laid-Up', launched='Launched', on_order='On Order/Under Construction', to_be_broken_up='To Be Broken Up',
from schematics.models import Model from schematics.types import DateTimeType, FloatType, IntType, ModelType, StringType from kp_scrapers.lib.date import ISO8601_FORMAT from kp_scrapers.models.enum import Enum from kp_scrapers.models.normalize import BaseEvent from kp_scrapers.models.vessel import Vessel AisType = Enum(dynamic='D-AIS', satellite='S-AIS', terrestrial='T-AIS') class Position(Model): """Describe an position model for AIS message. REQUIRES ALL of the following fields: - ais_type - course - lat - lon - received_time - speed Optional fields: - draught - draught_raw - heading - nav_state """ ais_type = StringType(