def portcall_mapping(): return { 'DERIVED$01': (ignore_key('detail button')), 'VESSEL': ('vessel_name', None), 'ETA_PILOT': ('eta', normalize_date), 'ETA_FIRST_BERTH': ('berthed', normalize_date), 'ARRIVAL': ('arrival', normalize_date), 'ETD': ('etd', normalize_date), 'DEPARTURE': ('departure', normalize_date), 'STATUS': (ignore_key('portcall status')), 'INTERNET_REMARK': (ignore_key('remarks')), 'URL': (ignore_key('detail link, always None')), # call detail 'Status': (ignore_key('status')), 'Authorities ref': (ignore_key('ignore')), 'Port': (ignore_key('port name, duplicate')), 'Remarks': (ignore_key('remarks, duplicate')), 'Last updated on': ('reported_date', normalize_date), # terminal detail 'Terminal': ('installation', normalize_berth_installation), 'Jetty': ('berth', normalize_berth_installation), 'Portnumber': (ignore_key('ignore')), '(E)TA': ('actual_eta', normalize_date), '(E)TD': ('actual_etd', normalize_date), # meta info 'port_name': ('port_name', None), 'provider_name': ('provider_name', None), }
def field_mapping(): return { # NOTE since we are only scraping scheduled/future portcalls, we hardcode 'eta' 'ship_name': ('vessel_name', None), 'pier_Assigned': ('berth', None), 'ship_type': ignore_key('vessel type'), 'date_arrival': ('eta', lambda x: to_isoformat(x, dayfirst=True)), 'date_departure': ('departure', lambda x: to_isoformat(x, dayfirst=True)), 'depth': ignore_key('depth'), 'Ship name': ignore_key('Ship name'), 'Indicative Radius:': ignore_key('Indicative Radius:'), 'IMO Number:': ('imo', None), 'GT London 1969:': ignore_key('irrelevant'), 'GT:': ('gross_tonnage', None), 'Country:': ('flag_code', lambda x: x.partition('-')[0]), 'Consignee:': ignore_key('consignee'), 'Shipowner:': ignore_key('companies'), 'Ship type:': ('vessel_type', None), 'Tanks:': ignore_key('Tanks:'), 'Beam:': ('beam', None), 'Length:': ('length', None), 'Draught:': ignore_key('Draught:'), 'provider_name': ('provider_name', None), 'port_name': ('port_name', None), }
def grades_mapping(): return { 'Cargo': ('cargo_movement', lambda x: 'discharge' if 'import' in x.lower() else 'load'), 'Cargo No': ('cargo_movement', lambda x: 'discharge' if 'import' in x.lower() else 'load'), 'Cargo No.': ('cargo_movement', lambda x: 'discharge' if 'import' in x.lower() else 'load'), 'Charterer': ignore_key('charterer'), 'Charterers': ignore_key('charterer'), 'Date': ('arrival', lambda x: to_isoformat(x, dayfirst=True)), 'Dates': ('arrival', lambda x: to_isoformat(x, dayfirst=True)), 'Grade': ('cargo_product', None), 'Next Port': ignore_key('next port'), 'Notes': ignore_key('remarks'), 'provider_name': ('provider_name', None), 'port_name': ('port_name', None), # cargo unit is in kilobarrels 'QTY': ('cargo_volume', lambda x: x + '000' if x else None), 'Quantity': ('cargo_volume', lambda x: x + '000' if x else None), 'reported_date': ('reported_date', None), 'Shipper': ignore_key('irrelevant'), 'Shipper/Receiver': ignore_key('irrelevant'), 'Shipper/Receivers': ignore_key('irrelevant'), 'Supp/Rcvr': ignore_key('irrelevant'), 'Supplier': ignore_key('irrelevant'), 'Vessel': ('vessel', lambda x: {'name': x} if 'TBN' not in x else None), }
def current_week_mapping(self): return { 'product': ('product', lambda x: PRODUCT_MAPPING.get(x, x)), 'volume_current': ('volume_current', lambda x: float(x.replace(',', ''))), 'current_week': ('current_week', None), 'volume_prev': ignore_key('volume_prev'), 'prev_week': ignore_key('prev_week'), }
def field_mapping(): return { '0': ('vessel', normalize_vessel), '1': ignore_key('size'), '2': ignore_key('cargo'), '3': ('lay_can', None), '4': ('voyage', None), '5': ('rate_value', lambda x: RATE_MAPPING.get(x, x)), '6': ('charterer_status', None), 'provider_name': ('provider_name', None), 'reported_date': ('reported_date', None), }
def field_mapping(): return { # common to all tables 'AGENT': (ignore_key('not used in model')), 'CARGO': ('cargoes', lambda x: [{ 'product': x }] if x else []), 'NAME OF SHIP': ('vessel_name', None), 'Name of the ship / voyage': ('vessel_name', None), 'Type of Cargo': ('cargoes', lambda x: [{ 'product': x }] if x else []), 'Type of Cargo ': ('cargoes', lambda x: [{ 'product': x }] if x else []), # static info 'event': ('event', None), 'port_name': ('port_name', None), 'provider_name': ('provider_name', None), 'reported_date': ('reported_date', None), # for eta/arrival tables 'DRAFT': (ignore_key('not used in model')), 'ETA / ATA': ('matching_date', to_isoformat), 'No': (ignore_key('not used in model')), # for berthed tables 'ARR. TEMA': (ignore_key('not used in model')), 'BOOKINGS': (ignore_key('not used in model')), 'ETB/ATB': ('matching_date', to_isoformat), 'ETD': (ignore_key('not used in model')), 'L.O.A': ('vessel_length', lambda x: try_apply(x, float, int, str)), 'SHIP STATUS': (ignore_key('not used in model')), 'ST\'DORE': (ignore_key('not used in model')), }
def portcall_mapping(): return { # ships expected tables 'VESSEL NAME': ('vessel_name', normalize_vessel_name), 'VES.SCHEDULE': (ignore_key('vessel schedule')), 'CALL SIGN': ('call_sign', None), 'VOYAGE NO.': (ignore_key('voyage number')), 'ETA': ('eta', lambda x: to_isoformat(x, dayfirst=True)), 'LOA': ('vessel_length', normalize_numeric_value), 'DRAFT': (ignore_key('draught')), 'AGENT': ('shipping_agent', None), 'DISCH': ('disch_volume', normalize_numeric_value), 'LOAD': ('load_volume', normalize_numeric_value), 'FBW': (ignore_key('unknown')), 'BER': (ignore_key('unknown')), 'BOOKED': (ignore_key('booked date')), 'REMARKS': ('remarks', None), # waiters '0': ('arrival', lambda x: to_isoformat(x, dayfirst=True)), '1': ('vessel_name', None), '2': ('vessel_length', normalize_numeric_value), '3': (ignore_key('draught')), '4': (ignore_key('unknown')), '5': ('remarks', None), # meta info 'port_name': ('port_name', None), 'provider_name': ('provider_name', None), 'reported_date': ('reported_date', None), }
def portcall_mapping(): return { 'Call ID': ignore_key('irrelevant'), 'Vessel Name': ('vessel_name', None), 'IMO': ('vessel_imo', None), 'Agent': ('shipping_agent', None), 'Estimated Time of Arrival': ('eta', to_isoformat), 'Last Port': ignore_key('previous port of call'), 'Next Port': ignore_key('next port of call'), 'Status': ignore_key('irrelevant'), 'port_name': ('port_name', None), 'provider_name': ('provider_name', None), 'reported_date': ('reported_date', None), }
def field_mapping(): return { '0': ignore_key('arrival_time_or_status'), '1': ('vessel', None), '2': ignore_key('volume_product_combined'), '3': ('lay_can', None), '4': ('departure_zone', None), '5': ('rate_value', None), '6': ('charterer', lambda x: x.replace('?', '')), '7': ('volume', None), '8': ('product', None), 'provider_name': ('provider_name', None), 'reported_date': ('reported_date', None), }
def portcall_mapping(): return { 'Berth From': ignore_key('from berth'), 'Berth Name': ('berth', None), 'Cargo Code': ('cargo_product', lambda x: PRODUCT_MAPPING.get(x, x)), 'Commence Work Date': ('berthed_date', None), 'Commence Work Time': ('berthed_time', None), 'Finish Work Date': ignore_key('finish date'), 'Finish Work Time': ignore_key('finish time'), 'Haul In Date': ('arrival_date', None), 'Haul In Time': ('arrival_time', None), 'Haul Out Date': ignore_key('departure date'), 'Haul Out Time': ignore_key('departure time'), 'port_name': ('port_name', None), 'provider_name': ('provider_name', None), 'reported_date': ('reported_date', None), 'S.No': ignore_key('serial number'), 'To Be LD/SH': ('cargo_volume_handled', None), 'Total Quantity (MT)': ('cargo_volume_leftover', None), 'Vessel Due': ignore_key('vessel due'), 'Vessel Name': ('vessel', lambda x: { 'name': normalize_vessel_name(x) }), 'Vessel Shifting': ignore_key('vessel shifting'), }
def field_mapping(): return { '0': ('vessel', lambda x: { 'name': x } if 'TBN' not in x.split() else None), '1': ignore_key('size'), '2': ignore_key('cargo'), '3': ('voyage', None), '4': ('lay_can', None), '5': ('rate_value', None), '6': ('charterer_status', None), 'provider_name': ('provider_name', None), 'reported_date': ('reported_date', None), }
def field_mapping(): return { 'CALLNO': ignore_key('redundant'), 'Charterer Name': ignore_key('report does not provide movement or origin port'), 'Responsible Party': ignore_key('redundant'), 'Owner - Operator (Principal Name)': ignore_key('redundant'), 'Vessel': ('vessel', lambda x: { 'name': x }), 'Arrival': ('arrival', None), 'Departure': ('departure', None), 'Cargo Type': ignore_key('redundant'), 'Commodity': ('cargo_product', None), 'Cargo Description': ('provider_name', None), 'Quantity(Cargo)': ('cargo_volume', lambda x: None if x == '' else x), 'Measurement Unit': ('cargo_unit', lambda x: None if x == '' else UNIT_MAPPING.get(x, x)), 'Country Name': ignore_key('redundant'), 'Port Name': ('port_name', None), 'Next Port Name': ignore_key('redundant'), 'Prev Port Name': ignore_key('redundant'), 'provider_name': ('provider_name', None), 'reported_date': ('reported_date', lambda x: to_isoformat(x, dayfirst=True)), }
def eta_mapping(): return { '0': ('eta_date', None), '1': ('eta_time', lambda x: x.split('/')[0]), '2': ignore_key('irrelevant place'), '3': ('vessel_name', remove_diacritics), '4': ('cargoes', lambda x: [{'product': x}]), '5': ignore_key('irrelevant event'), '6': ignore_key('irrelevant voyage'), '7': ignore_key('irrelevant duv'), 'port_name': ('port_name', None), 'provider_name': ('provider_name', None), 'reported_date': ('reported_date', lambda x: to_isoformat(x, dayfirst=True)), 'url': ignore_key('url no longer needed'), }
def portcall_mapping(): return { 'N° Escale': ignore_key('internal stop number'), 'Nom Navire': ('vessel', lambda x: {'name': may_strip(x.replace('*', ''))}), 'Marchandise': ('cargoes', lambda x: normalize_cargo(x)), 'D-Arrivée': ('arrival', to_isoformat), 'D-Accostage': ('berthed', to_isoformat), 'D-Sortie': ignore_key('no need to scrape estimated time of departure'), 'Quai': ignore_key('berth is not needed since there is only one installation'), 'Observation': ignore_key('comments by port authority'), 'port_name': ('port_name', None), 'provider_name': ('provider_name', None), 'reported_date': ('reported_date', None), 'DUMMY': ignore_key('just a temporary header column to match the empty values'), }
def grades_mapping(): return { 'Date': ('departure', to_isoformat), 'Dock': ('berth', None), 'Vessel': ('vessel', lambda x: {'name': x}), 'Charterer': ignore_key('charters spider'), 'Grade': ('cargo_product', lambda x: PRODUCT_MAPPING.get(x.upper(), x)), 'Qty Nominated': ('cargo_volume_nominated', lambda x: try_apply(x, float, int, str)), 'Qty Loaded': ('cargo_volume', lambda x: try_apply(x, float, int, str)), 'Loading Rate': ignore_key('irrelevant'), 'Status': ignore_key('irrelevant'), 'provider_name': ('provider_name', None), 'reported_date': ('reported_date', None), 'sheet_name': ('sheet_name', None), }
def portcall_mapping(): return { 'Cargo': ('cargoes', normalize_cargoes), 'Consignee': ignore_key('shipping agent'), 'Date': ('pc_date', lambda x: to_isoformat(x, dayfirst=False)), 'Dockside': ignore_key('berth'), 'port_name': ('port_name', None), 'Port of call number': ignore_key('call number'), 'provider_name': ('provider_name', None), 'reported_date': ('reported_date', None), 'Type': ('event', lambda x: EVENT_MAPPING.get(x)), 'Vessel': ('vessel', lambda x: { 'name': x }), }
def portcall_mapping(): return { '0': ('eta', None), '1': ('vessel_name', None), '2': ('vessel_type', lambda x: x.title()), '3': ('vessel_gt', lambda x: x is None if x == '' else x), '4': ('vessel_length', lambda x: x is None if x == '' else x), '5': ignore_key('type'), '6': ('vessel_flagcode', lambda x: FLAGS.get(x, x)), '7': ignore_key('previous portcall'), '8': ('port_name', clean_zone_name), '9': ignore_key('warning ship'), '10': ignore_key('pilot'), 'provider_name': ('provider_name', None), 'reported_date': ('reported_date', None), }
def field_mapping(): return { 'reported_date': ('reported_date', None), 'port_name': ('port_name', None), 'provider_name': ('provider_name', None), 'Vessel Name': ('vessel_name', None), 'Expected To Arrival': ('arrival', lambda x: to_isoformat(x, dayfirst=True)), 'SCN NO': ignore_key('SCN NO'), 'LOA': ('length', None), 'Voyage No': ignore_key('Voyage No'), 'Shipping Agent': ignore_key('Shipping Agent'), 'Cargo Detail': ( 'cargoes', lambda x: [{'product': x} for x in x.split(',') if x and x not in INVALID_PRODUCTS], ), }
def berthed_mapping(): return { '0': ('berth', None), '1': ('vessel_name', remove_diacritics), '2': ignore_key('morning'), '3': ignore_key('afternoon'), '4': ignore_key('evening'), '5': ignore_key('night'), '6': ('cargo', None), '7': ('discharge', None), '8': ('load', None), 'port_name': ('port_name', None), 'provider_name': ('provider_name', None), 'reported_date': ('reported_date', lambda x: to_isoformat(x, dayfirst=True)), 'url': ignore_key('url no longer needed'), }
def field_mapping(): return { 'Cargo Type': ('cargo', lambda x: { 'product': may_strip(x) }), 'Cargo Type Name': ('cargo', lambda x: { 'product': may_strip(x) }), 'Cgo': ('cargo', lambda x: { 'product': may_strip(x) }), 'Charterer': ('charterer', may_strip), 'Date': ('reported_date', lambda x: parse_date(x).strftime('%d %b %Y')), 'From': ('departure_zone', may_strip), 'From Port': ('departure_zone', may_strip), 'From Zn': (ignore_key('redundant departure zone')), 'From Zone Code': (ignore_key('redundant departure zone')), 'IMO': ('vessel', lambda x: { 'imo': x }), 'IMO ': ('vessel', lambda x: { 'imo': x }), 'IMO #': ('vessel', lambda x: { 'imo': x }), 'IMONumber': ('vessel', lambda x: { 'imo': x }), 'Lay Can From': ('lay_can_start', lambda x: normalize_lay_can(x)), 'Lay Can To': ('lay_can_end', lambda x: normalize_lay_can(x)), 'LayCan From': ('lay_can_start', lambda x: normalize_lay_can(x)), 'LayCan To': ('lay_can_end', lambda x: normalize_lay_can(x)), 'provider_name': ('provider_name', may_strip), 'Reported Date': ('reported_date', lambda x: parse_date(x).strftime('%d %b %Y')), 'To': ('arrival_zone', lambda x: [may_strip(zone) for zone in x.split('-')] if x else []), 'To Port': ( 'arrival_zone', lambda x: [may_strip(zone) for zone in x.split('-')] if x else [], ), 'To Zn': (ignore_key('redundant arrival zone')), 'To Zone Code': (ignore_key('redundant arrival zone')), }
def portcall_mapping() -> Dict[str, Tuple[str, Optional[Callable]]]: return { 'Date': ('eta', lambda x: to_isoformat(x, dayfirst=False, yearfirst=True)), 'Ship': ('vessel_name', None), 'GT': ('vessel_gt', lambda x: validate_weight(x)), 'DWT': ('vessel_dwt', lambda x: validate_weight(x)), 'Move Type': ('event', lambda x: may_strip(x.lower())), 'Remarks': ignore_key('handwritten notes from port authority'), 'From': ignore_key('irrelevant'), 'To': ('installation', map_berth_to_installation), 'ACTION_STATUS_ID': ignore_key('internal ID used by port'), 'Tug': ignore_key('irrelevant'), 'port_name': ('port_name', None), 'provider_name': ('provider_name', None), 'reported_date': ('reported_date', None), }
def field_mapping(): return { 'N◦': (ignore_key('order number')), 'VESSEL': ('vessel', normalize_vessel), 'ETA': ('eta', normalize_date), 'ARRIVED': ('arrival', normalize_date), 'ETC/ETS': ('berthed', normalize_date), 'PRODUCT': ('cargo_product', None), 'SBCO': ('cargo_product', None), 'QUANTITY': ('cargo_volume', None), 'JETTY': (ignore_key('jetty')), 'DATE': ('departure', normalize_date), 'DEST': ('next_zone', None), 'provider_name': ('provider_name', None), 'reported_date': ('reported_date', None), 'port_name': ('port_name', None), }
def field_mapping(): return { 'SR': (ignore_key('irrelevant')), 'Date': ('berthed', None), 'Jetty': ('berth', lambda x: try_apply(x, str).replace('.0', '')), 'Operation': ('cargo_movement', lambda x: re.sub(r'[\W]', '', x.lower())), 'Product': ('cargo_product', None), 'Vessel Name': ('vessel', lambda x: {'name': may_strip(x)}), # TODO include buyer and seller 'Customer / Terminal': (ignore_key('buyer and seller, cannot be used currently')), 'Latest Value b/w CT inform & Berth Available': (ignore_key('irrelevant')), 'Parcel size M3': ('cargo_volume', lambda x: try_apply(x, str).replace(',', '')), 'port_name': ('port_name', None), 'installation': ('installation', None), 'provider_name': ('provider_name', None), 'reported_date': ('reported_date', None), }
def field_mapping(): return { 'VESSEL': ('vessel', lambda x: {'name': may_strip(x)}), 'ETA': ('eta', normalize_date), 'ETB': ('berthed', normalize_date), 'ETS': ('departure', normalize_date), 'GRADE': ('cargo_product', lambda x: None if 'TBI' in x else x), 'QTTY': ('cargo_quantity', lambda x: try_apply(x.replace('.', '', 1), int)), 'TTL QTTY': ignore_key('redundant'), 'LD PORT': ('load_cargo_movement', None), 'DISC PORT': ('dis_cargo_movement', None), 'SH / REC': ignore_key('redundant'), 'RMK': ignore_key('redundant'), 'port_name': ('port_name', None), 'provider_name': ('provider_name', None), 'reported_date': ('reported_date', None), }
def field_mapping(): return { 'country': ignore_key('country'), 'location': ('port_name', may_strip), 'vessel': ('vessel', lambda x: { 'name': x }), 'facility/terminal': ignore_key('redundant'), 'cargo': ('cargo_product', None), 'volume/mt': ('cargo_volume', may_strip), 'eta': ('eta', None), 'load/discharge': ('cargo_movement', lambda x: x.lower()), 'receiver': ('cargo_seller', None), 'origin': ignore_key('origin'), 'provider_name': ('provider_name', None), 'reported_date': ('reported_date', None), }
def field_mapping(): return { 'Carrier': ignore_key('redundant'), 'Commodity': ('cargo_product', None), 'Departure Date': ('departure', None), 'Destination Country': ignore_key('redundant'), 'Destination Region': ignore_key('redundant'), 'Destination Port': ignore_key('redundant'), 'Port of Departure': ('port_name', lambda x: PORT_MAPPING.get(x, x)), 'provider_name': ('provider_name', None), 'reported_date': ('reported_date', None), 'Shipper': ignore_key('redundant'), 'Vessel Name': ('vessel', lambda x: { 'name': x }), 'Weight (MT)': ('cargo_volume', None), }
def field_mapping(): return { 'FILLER': ignore_key('irrelevant'), 'NO.': ignore_key('irrelevant'), 'VESSELS NAME': ('vessel', lambda x: { 'name': x }), 'QUANTITY': ('cargo_quantity', lambda x: x.replace(',', '')), 'ETA': ('eta', lambda x: normalize_dates(x)), 'LOAD': ('berthed', lambda x: normalize_dates(x)), 'ETC / D': ('departure', lambda x: normalize_dates(x)), 'DESTINATION': ignore_key('irrelevant'), 'SHIPPER': ('cargo_seller', None), 'provider_name': ('provider_name', None), 'port_name': ('port_name', lambda x: normalize_port_name(x)), 'reported_date': ('reported_date', lambda x: to_isoformat(x)), }
def field_mapping(): # declarative mapping for ease of developement/maintenance return { 'SHIP NAME': ('vessel_name', None), 'ETA': ('eta', None), 'hh.mm': ignore_key('hh.mm'), 'PROSP.': ignore_key('PROSP.'), 'POB': ('berthed', None), 'BERTH': ignore_key('BERTH'), 'ETS': ('departure', None), 'LOAD. PORT': ignore_key('LOAD. PORT'), 'GRADE': ('cargo_product', None), 'M. TONS': ('cargo_volume', None), 'RECEIVER': ('cargo_receiver', None), 'provider_name': ('provider_name', None), 'reported_date': ('reported_date', None), }
def field_mapping(**kwargs): return { 'Vessel Name': ('vessel_name', None), 'Flag': ignore_key('vessel flag'), 'Vessel Details Disch.Agent/ Load. Agent': ignore_key('shipping agent'), 'Arrival Date': ('arrival', lambda x: normalize_date(x, **kwargs)), 'Sailing Date': ('departure', lambda x: normalize_date(x, **kwargs)), 'Comm. D/L': ('berthed', lambda x: normalize_date(x, **kwargs)), 'Operations Berth No.': ignore_key('berth of portcall'), 'M/T TONS': ('ton', None), 'Cargo Disch .Details Commodity': ('commodity_d', None), 'Cargo Load.Details Commodity': ('commodity_l', None), 'reported_date': ('reported_date', None), 'port_name': ('port_name', None), 'provider_name': ('provider_name', None), }
def charters_mapping(): return { '0': ('status', lambda x: STATUS_MAPPING.get(x.partition(' ')[0])), '1': ('vessel', lambda x: {'name': x}), '2': ignore_key('unknown value, possibly cargo volume'), '3': ('cargo', lambda x: {'product': x}), '4': ('departure_zone', None), '5': ignore_key('slash /, irrelevant'), '6': ('arrival_zone', lambda x: x.split('-') if x else None), '7': ('lay_can', None), '8': ('rate_value', None), '9': ('charterer', None), '10': ignore_key('irrelevant'), '11': ignore_key('irrelevant'), 'provider_name': ('provider_name', None), 'reported_date': ('reported_date', None), }