def portcall_mapping():
return {
'DERIVED$01': (ignore_key('detail button')),
'VESSEL': ('vessel_name', None),
'ETA_PILOT': ('eta', normalize_date),
'ETA_FIRST_BERTH': ('berthed', normalize_date),
'ARRIVAL': ('arrival', normalize_date),
'ETD': ('etd', normalize_date),
'DEPARTURE': ('departure', normalize_date),
'STATUS': (ignore_key('portcall status')),
'INTERNET_REMARK': (ignore_key('remarks')),
'URL': (ignore_key('detail link, always None')),
# call detail
'Status': (ignore_key('status')),
'Authorities ref': (ignore_key('ignore')),
'Port': (ignore_key('port name, duplicate')),
'Remarks': (ignore_key('remarks, duplicate')),
'Last updated on': ('reported_date', normalize_date),
# terminal detail
'Terminal': ('installation', normalize_berth_installation),
'Jetty': ('berth', normalize_berth_installation),
'Portnumber': (ignore_key('ignore')),
'(E)TA': ('actual_eta', normalize_date),
'(E)TD': ('actual_etd', normalize_date),
# meta info
'port_name': ('port_name', None),
'provider_name': ('provider_name', None),
}
def field_mapping():
return {
# NOTE since we are only scraping scheduled/future portcalls, we hardcode 'eta'
'ship_name': ('vessel_name', None),
'pier_Assigned': ('berth', None),
'ship_type': ignore_key('vessel type'),
'date_arrival': ('eta', lambda x: to_isoformat(x, dayfirst=True)),
'date_departure':
('departure', lambda x: to_isoformat(x, dayfirst=True)),
'depth': ignore_key('depth'),
'Ship name': ignore_key('Ship name'),
'Indicative Radius:': ignore_key('Indicative Radius:'),
'IMO Number:': ('imo', None),
'GT London 1969:': ignore_key('irrelevant'),
'GT:': ('gross_tonnage', None),
'Country:': ('flag_code', lambda x: x.partition('-')[0]),
'Consignee:': ignore_key('consignee'),
'Shipowner:': ignore_key('companies'),
'Ship type:': ('vessel_type', None),
'Tanks:': ignore_key('Tanks:'),
'Beam:': ('beam', None),
'Length:': ('length', None),
'Draught:': ignore_key('Draught:'),
'provider_name': ('provider_name', None),
'port_name': ('port_name', None),
}
def grades_mapping():
return {
'Cargo': ('cargo_movement', lambda x: 'discharge' if 'import' in x.lower() else 'load'),
'Cargo No': ('cargo_movement', lambda x: 'discharge' if 'import' in x.lower() else 'load'),
'Cargo No.': ('cargo_movement', lambda x: 'discharge' if 'import' in x.lower() else 'load'),
'Charterer': ignore_key('charterer'),
'Charterers': ignore_key('charterer'),
'Date': ('arrival', lambda x: to_isoformat(x, dayfirst=True)),
'Dates': ('arrival', lambda x: to_isoformat(x, dayfirst=True)),
'Grade': ('cargo_product', None),
'Next Port': ignore_key('next port'),
'Notes': ignore_key('remarks'),
'provider_name': ('provider_name', None),
'port_name': ('port_name', None),
# cargo unit is in kilobarrels
'QTY': ('cargo_volume', lambda x: x + '000' if x else None),
'Quantity': ('cargo_volume', lambda x: x + '000' if x else None),
'reported_date': ('reported_date', None),
'Shipper': ignore_key('irrelevant'),
'Shipper/Receiver': ignore_key('irrelevant'),
'Shipper/Receivers': ignore_key('irrelevant'),
'Supp/Rcvr': ignore_key('irrelevant'),
'Supplier': ignore_key('irrelevant'),
'Vessel': ('vessel', lambda x: {'name': x} if 'TBN' not in x else None),
}
def current_week_mapping(self):
return {
'product': ('product', lambda x: PRODUCT_MAPPING.get(x, x)),
'volume_current':
('volume_current', lambda x: float(x.replace(',', ''))),
'current_week': ('current_week', None),
'volume_prev':
ignore_key('volume_prev'),
'prev_week':
ignore_key('prev_week'),
}
def field_mapping():
return {
'0': ('vessel', normalize_vessel),
'1': ignore_key('size'),
'2': ignore_key('cargo'),
'3': ('lay_can', None),
'4': ('voyage', None),
'5': ('rate_value', lambda x: RATE_MAPPING.get(x, x)),
'6': ('charterer_status', None),
'provider_name': ('provider_name', None),
'reported_date': ('reported_date', None),
}
def field_mapping():
return {
# common to all tables
'AGENT': (ignore_key('not used in model')),
'CARGO': ('cargoes', lambda x: [{
'product': x
}] if x else []),
'NAME OF SHIP': ('vessel_name', None),
'Name of the ship / voyage': ('vessel_name', None),
'Type of Cargo': ('cargoes', lambda x: [{
'product': x
}] if x else []),
'Type of Cargo ': ('cargoes', lambda x: [{
'product': x
}] if x else []),
# static info
'event': ('event', None),
'port_name': ('port_name', None),
'provider_name': ('provider_name', None),
'reported_date': ('reported_date', None),
# for eta/arrival tables
'DRAFT': (ignore_key('not used in model')),
'ETA / ATA': ('matching_date', to_isoformat),
'No': (ignore_key('not used in model')),
# for berthed tables
'ARR. TEMA': (ignore_key('not used in model')),
'BOOKINGS': (ignore_key('not used in model')),
'ETB/ATB': ('matching_date', to_isoformat),
'ETD': (ignore_key('not used in model')),
'L.O.A': ('vessel_length', lambda x: try_apply(x, float, int, str)),
'SHIP STATUS': (ignore_key('not used in model')),
'ST\'DORE': (ignore_key('not used in model')),
}
def portcall_mapping():
return {
# ships expected tables
'VESSEL NAME': ('vessel_name', normalize_vessel_name),
'VES.SCHEDULE': (ignore_key('vessel schedule')),
'CALL SIGN': ('call_sign', None),
'VOYAGE NO.': (ignore_key('voyage number')),
'ETA': ('eta', lambda x: to_isoformat(x, dayfirst=True)),
'LOA': ('vessel_length', normalize_numeric_value),
'DRAFT': (ignore_key('draught')),
'AGENT': ('shipping_agent', None),
'DISCH': ('disch_volume', normalize_numeric_value),
'LOAD': ('load_volume', normalize_numeric_value),
'FBW': (ignore_key('unknown')),
'BER': (ignore_key('unknown')),
'BOOKED': (ignore_key('booked date')),
'REMARKS': ('remarks', None),
# waiters
'0': ('arrival', lambda x: to_isoformat(x, dayfirst=True)),
'1': ('vessel_name', None),
'2': ('vessel_length', normalize_numeric_value),
'3': (ignore_key('draught')),
'4': (ignore_key('unknown')),
'5': ('remarks', None),
# meta info
'port_name': ('port_name', None),
'provider_name': ('provider_name', None),
'reported_date': ('reported_date', None),
}
def portcall_mapping():
return {
'Call ID': ignore_key('irrelevant'),
'Vessel Name': ('vessel_name', None),
'IMO': ('vessel_imo', None),
'Agent': ('shipping_agent', None),
'Estimated Time of Arrival': ('eta', to_isoformat),
'Last Port': ignore_key('previous port of call'),
'Next Port': ignore_key('next port of call'),
'Status': ignore_key('irrelevant'),
'port_name': ('port_name', None),
'provider_name': ('provider_name', None),
'reported_date': ('reported_date', None),
}
def field_mapping():
return {
'0': ignore_key('arrival_time_or_status'),
'1': ('vessel', None),
'2': ignore_key('volume_product_combined'),
'3': ('lay_can', None),
'4': ('departure_zone', None),
'5': ('rate_value', None),
'6': ('charterer', lambda x: x.replace('?', '')),
'7': ('volume', None),
'8': ('product', None),
'provider_name': ('provider_name', None),
'reported_date': ('reported_date', None),
}
def portcall_mapping():
return {
'Berth From': ignore_key('from berth'),
'Berth Name': ('berth', None),
'Cargo Code': ('cargo_product', lambda x: PRODUCT_MAPPING.get(x, x)),
'Commence Work Date': ('berthed_date', None),
'Commence Work Time': ('berthed_time', None),
'Finish Work Date': ignore_key('finish date'),
'Finish Work Time': ignore_key('finish time'),
'Haul In Date': ('arrival_date', None),
'Haul In Time': ('arrival_time', None),
'Haul Out Date': ignore_key('departure date'),
'Haul Out Time': ignore_key('departure time'),
'port_name': ('port_name', None),
'provider_name': ('provider_name', None),
'reported_date': ('reported_date', None),
'S.No': ignore_key('serial number'),
'To Be LD/SH': ('cargo_volume_handled', None),
'Total Quantity (MT)': ('cargo_volume_leftover', None),
'Vessel Due': ignore_key('vessel due'),
'Vessel Name': ('vessel', lambda x: {
'name': normalize_vessel_name(x)
}),
'Vessel Shifting': ignore_key('vessel shifting'),
}
def field_mapping():
return {
'0': ('vessel', lambda x: {
'name': x
} if 'TBN' not in x.split() else None),
'1': ignore_key('size'),
'2': ignore_key('cargo'),
'3': ('voyage', None),
'4': ('lay_can', None),
'5': ('rate_value', None),
'6': ('charterer_status', None),
'provider_name': ('provider_name', None),
'reported_date': ('reported_date', None),
}
def field_mapping():
return {
'CALLNO':
ignore_key('redundant'),
'Charterer Name':
ignore_key('report does not provide movement or origin port'),
'Responsible Party':
ignore_key('redundant'),
'Owner - Operator (Principal Name)':
ignore_key('redundant'),
'Vessel': ('vessel', lambda x: {
'name': x
}),
'Arrival': ('arrival', None),
'Departure': ('departure', None),
'Cargo Type':
ignore_key('redundant'),
'Commodity': ('cargo_product', None),
'Cargo Description': ('provider_name', None),
'Quantity(Cargo)': ('cargo_volume', lambda x: None if x == '' else x),
'Measurement Unit': ('cargo_unit', lambda x: None
if x == '' else UNIT_MAPPING.get(x, x)),
'Country Name':
ignore_key('redundant'),
'Port Name': ('port_name', None),
'Next Port Name':
ignore_key('redundant'),
'Prev Port Name':
ignore_key('redundant'),
'provider_name': ('provider_name', None),
'reported_date':
('reported_date', lambda x: to_isoformat(x, dayfirst=True)),
}
def eta_mapping():
return {
'0': ('eta_date', None),
'1': ('eta_time', lambda x: x.split('/')[0]),
'2': ignore_key('irrelevant place'),
'3': ('vessel_name', remove_diacritics),
'4': ('cargoes', lambda x: [{'product': x}]),
'5': ignore_key('irrelevant event'),
'6': ignore_key('irrelevant voyage'),
'7': ignore_key('irrelevant duv'),
'port_name': ('port_name', None),
'provider_name': ('provider_name', None),
'reported_date': ('reported_date', lambda x: to_isoformat(x, dayfirst=True)),
'url': ignore_key('url no longer needed'),
}
def portcall_mapping():
return {
'N° Escale': ignore_key('internal stop number'),
'Nom Navire': ('vessel', lambda x: {'name': may_strip(x.replace('*', ''))}),
'Marchandise': ('cargoes', lambda x: normalize_cargo(x)),
'D-Arrivée': ('arrival', to_isoformat),
'D-Accostage': ('berthed', to_isoformat),
'D-Sortie': ignore_key('no need to scrape estimated time of departure'),
'Quai': ignore_key('berth is not needed since there is only one installation'),
'Observation': ignore_key('comments by port authority'),
'port_name': ('port_name', None),
'provider_name': ('provider_name', None),
'reported_date': ('reported_date', None),
'DUMMY': ignore_key('just a temporary header column to match the empty values'),
}
def grades_mapping():
return {
'Date': ('departure', to_isoformat),
'Dock': ('berth', None),
'Vessel': ('vessel', lambda x: {'name': x}),
'Charterer': ignore_key('charters spider'),
'Grade': ('cargo_product', lambda x: PRODUCT_MAPPING.get(x.upper(), x)),
'Qty Nominated': ('cargo_volume_nominated', lambda x: try_apply(x, float, int, str)),
'Qty Loaded': ('cargo_volume', lambda x: try_apply(x, float, int, str)),
'Loading Rate': ignore_key('irrelevant'),
'Status': ignore_key('irrelevant'),
'provider_name': ('provider_name', None),
'reported_date': ('reported_date', None),
'sheet_name': ('sheet_name', None),
}
def portcall_mapping():
return {
'Cargo': ('cargoes', normalize_cargoes),
'Consignee': ignore_key('shipping agent'),
'Date': ('pc_date', lambda x: to_isoformat(x, dayfirst=False)),
'Dockside': ignore_key('berth'),
'port_name': ('port_name', None),
'Port of call number': ignore_key('call number'),
'provider_name': ('provider_name', None),
'reported_date': ('reported_date', None),
'Type': ('event', lambda x: EVENT_MAPPING.get(x)),
'Vessel': ('vessel', lambda x: {
'name': x
}),
}
def portcall_mapping():
return {
'0': ('eta', None),
'1': ('vessel_name', None),
'2': ('vessel_type', lambda x: x.title()),
'3': ('vessel_gt', lambda x: x is None if x == '' else x),
'4': ('vessel_length', lambda x: x is None if x == '' else x),
'5': ignore_key('type'),
'6': ('vessel_flagcode', lambda x: FLAGS.get(x, x)),
'7': ignore_key('previous portcall'),
'8': ('port_name', clean_zone_name),
'9': ignore_key('warning ship'),
'10': ignore_key('pilot'),
'provider_name': ('provider_name', None),
'reported_date': ('reported_date', None),
}
def field_mapping():
return {
'reported_date': ('reported_date', None),
'port_name': ('port_name', None),
'provider_name': ('provider_name', None),
'Vessel Name': ('vessel_name', None),
'Expected To Arrival': ('arrival', lambda x: to_isoformat(x, dayfirst=True)),
'SCN NO': ignore_key('SCN NO'),
'LOA': ('length', None),
'Voyage No': ignore_key('Voyage No'),
'Shipping Agent': ignore_key('Shipping Agent'),
'Cargo Detail': (
'cargoes',
lambda x: [{'product': x} for x in x.split(',') if x and x not in INVALID_PRODUCTS],
),
}
def berthed_mapping():
return {
'0': ('berth', None),
'1': ('vessel_name', remove_diacritics),
'2': ignore_key('morning'),
'3': ignore_key('afternoon'),
'4': ignore_key('evening'),
'5': ignore_key('night'),
'6': ('cargo', None),
'7': ('discharge', None),
'8': ('load', None),
'port_name': ('port_name', None),
'provider_name': ('provider_name', None),
'reported_date': ('reported_date', lambda x: to_isoformat(x, dayfirst=True)),
'url': ignore_key('url no longer needed'),
}
def field_mapping():
return {
'Cargo Type': ('cargo', lambda x: {
'product': may_strip(x)
}),
'Cargo Type Name': ('cargo', lambda x: {
'product': may_strip(x)
}),
'Cgo': ('cargo', lambda x: {
'product': may_strip(x)
}),
'Charterer': ('charterer', may_strip),
'Date':
('reported_date', lambda x: parse_date(x).strftime('%d %b %Y')),
'From': ('departure_zone', may_strip),
'From Port': ('departure_zone', may_strip),
'From Zn': (ignore_key('redundant departure zone')),
'From Zone Code': (ignore_key('redundant departure zone')),
'IMO': ('vessel', lambda x: {
'imo': x
}),
'IMO ': ('vessel', lambda x: {
'imo': x
}),
'IMO #': ('vessel', lambda x: {
'imo': x
}),
'IMONumber': ('vessel', lambda x: {
'imo': x
}),
'Lay Can From': ('lay_can_start', lambda x: normalize_lay_can(x)),
'Lay Can To': ('lay_can_end', lambda x: normalize_lay_can(x)),
'LayCan From': ('lay_can_start', lambda x: normalize_lay_can(x)),
'LayCan To': ('lay_can_end', lambda x: normalize_lay_can(x)),
'provider_name': ('provider_name', may_strip),
'Reported Date':
('reported_date', lambda x: parse_date(x).strftime('%d %b %Y')),
'To':
('arrival_zone', lambda x: [may_strip(zone) for zone in x.split('-')]
if x else []),
'To Port': (
'arrival_zone',
lambda x: [may_strip(zone) for zone in x.split('-')] if x else [],
),
'To Zn': (ignore_key('redundant arrival zone')),
'To Zone Code': (ignore_key('redundant arrival zone')),
}
def portcall_mapping() -> Dict[str, Tuple[str, Optional[Callable]]]:
return {
'Date':
('eta', lambda x: to_isoformat(x, dayfirst=False, yearfirst=True)),
'Ship': ('vessel_name', None),
'GT': ('vessel_gt', lambda x: validate_weight(x)),
'DWT': ('vessel_dwt', lambda x: validate_weight(x)),
'Move Type': ('event', lambda x: may_strip(x.lower())),
'Remarks': ignore_key('handwritten notes from port authority'),
'From': ignore_key('irrelevant'),
'To': ('installation', map_berth_to_installation),
'ACTION_STATUS_ID': ignore_key('internal ID used by port'),
'Tug': ignore_key('irrelevant'),
'port_name': ('port_name', None),
'provider_name': ('provider_name', None),
'reported_date': ('reported_date', None),
}
def field_mapping():
return {
'N◦': (ignore_key('order number')),
'VESSEL': ('vessel', normalize_vessel),
'ETA': ('eta', normalize_date),
'ARRIVED': ('arrival', normalize_date),
'ETC/ETS': ('berthed', normalize_date),
'PRODUCT': ('cargo_product', None),
'SBCO': ('cargo_product', None),
'QUANTITY': ('cargo_volume', None),
'JETTY': (ignore_key('jetty')),
'DATE': ('departure', normalize_date),
'DEST': ('next_zone', None),
'provider_name': ('provider_name', None),
'reported_date': ('reported_date', None),
'port_name': ('port_name', None),
}
def field_mapping():
return {
'SR': (ignore_key('irrelevant')),
'Date': ('berthed', None),
'Jetty': ('berth', lambda x: try_apply(x, str).replace('.0', '')),
'Operation': ('cargo_movement', lambda x: re.sub(r'[\W]', '', x.lower())),
'Product': ('cargo_product', None),
'Vessel Name': ('vessel', lambda x: {'name': may_strip(x)}),
# TODO include buyer and seller
'Customer / Terminal': (ignore_key('buyer and seller, cannot be used currently')),
'Latest Value b/w CT inform & Berth Available': (ignore_key('irrelevant')),
'Parcel size M3': ('cargo_volume', lambda x: try_apply(x, str).replace(',', '')),
'port_name': ('port_name', None),
'installation': ('installation', None),
'provider_name': ('provider_name', None),
'reported_date': ('reported_date', None),
}
def field_mapping():
return {
'VESSEL': ('vessel', lambda x: {'name': may_strip(x)}),
'ETA': ('eta', normalize_date),
'ETB': ('berthed', normalize_date),
'ETS': ('departure', normalize_date),
'GRADE': ('cargo_product', lambda x: None if 'TBI' in x else x),
'QTTY': ('cargo_quantity', lambda x: try_apply(x.replace('.', '', 1), int)),
'TTL QTTY': ignore_key('redundant'),
'LD PORT': ('load_cargo_movement', None),
'DISC PORT': ('dis_cargo_movement', None),
'SH / REC': ignore_key('redundant'),
'RMK': ignore_key('redundant'),
'port_name': ('port_name', None),
'provider_name': ('provider_name', None),
'reported_date': ('reported_date', None),
}
def field_mapping():
return {
'country': ignore_key('country'),
'location': ('port_name', may_strip),
'vessel': ('vessel', lambda x: {
'name': x
}),
'facility/terminal': ignore_key('redundant'),
'cargo': ('cargo_product', None),
'volume/mt': ('cargo_volume', may_strip),
'eta': ('eta', None),
'load/discharge': ('cargo_movement', lambda x: x.lower()),
'receiver': ('cargo_seller', None),
'origin': ignore_key('origin'),
'provider_name': ('provider_name', None),
'reported_date': ('reported_date', None),
}
def field_mapping():
return {
'Carrier': ignore_key('redundant'),
'Commodity': ('cargo_product', None),
'Departure Date': ('departure', None),
'Destination Country': ignore_key('redundant'),
'Destination Region': ignore_key('redundant'),
'Destination Port': ignore_key('redundant'),
'Port of Departure': ('port_name', lambda x: PORT_MAPPING.get(x, x)),
'provider_name': ('provider_name', None),
'reported_date': ('reported_date', None),
'Shipper': ignore_key('redundant'),
'Vessel Name': ('vessel', lambda x: {
'name': x
}),
'Weight (MT)': ('cargo_volume', None),
}
def field_mapping():
return {
'FILLER': ignore_key('irrelevant'),
'NO.': ignore_key('irrelevant'),
'VESSELS NAME': ('vessel', lambda x: {
'name': x
}),
'QUANTITY': ('cargo_quantity', lambda x: x.replace(',', '')),
'ETA': ('eta', lambda x: normalize_dates(x)),
'LOAD': ('berthed', lambda x: normalize_dates(x)),
'ETC / D': ('departure', lambda x: normalize_dates(x)),
'DESTINATION': ignore_key('irrelevant'),
'SHIPPER': ('cargo_seller', None),
'provider_name': ('provider_name', None),
'port_name': ('port_name', lambda x: normalize_port_name(x)),
'reported_date': ('reported_date', lambda x: to_isoformat(x)),
}
def field_mapping():
# declarative mapping for ease of developement/maintenance
return {
'SHIP NAME': ('vessel_name', None),
'ETA': ('eta', None),
'hh.mm': ignore_key('hh.mm'),
'PROSP.': ignore_key('PROSP.'),
'POB': ('berthed', None),
'BERTH': ignore_key('BERTH'),
'ETS': ('departure', None),
'LOAD. PORT': ignore_key('LOAD. PORT'),
'GRADE': ('cargo_product', None),
'M. TONS': ('cargo_volume', None),
'RECEIVER': ('cargo_receiver', None),
'provider_name': ('provider_name', None),
'reported_date': ('reported_date', None),
}
def field_mapping(**kwargs):
return {
'Vessel Name': ('vessel_name', None),
'Flag': ignore_key('vessel flag'),
'Vessel Details Disch.Agent/ Load. Agent':
ignore_key('shipping agent'),
'Arrival Date': ('arrival', lambda x: normalize_date(x, **kwargs)),
'Sailing Date': ('departure', lambda x: normalize_date(x, **kwargs)),
'Comm. D/L': ('berthed', lambda x: normalize_date(x, **kwargs)),
'Operations Berth No.': ignore_key('berth of portcall'),
'M/T TONS': ('ton', None),
'Cargo Disch .Details Commodity': ('commodity_d', None),
'Cargo Load.Details Commodity': ('commodity_l', None),
'reported_date': ('reported_date', None),
'port_name': ('port_name', None),
'provider_name': ('provider_name', None),
}
def charters_mapping():
return {
'0': ('status', lambda x: STATUS_MAPPING.get(x.partition(' ')[0])),
'1': ('vessel', lambda x: {'name': x}),
'2': ignore_key('unknown value, possibly cargo volume'),
'3': ('cargo', lambda x: {'product': x}),
'4': ('departure_zone', None),
'5': ignore_key('slash /, irrelevant'),
'6': ('arrival_zone', lambda x: x.split('-') if x else None),
'7': ('lay_can', None),
'8': ('rate_value', None),
'9': ('charterer', None),
'10': ignore_key('irrelevant'),
'11': ignore_key('irrelevant'),
'provider_name': ('provider_name', None),
'reported_date': ('reported_date', None),
}
