verify_datasets(
official_files_location + "/Official-ContractStandingData",
"ContractStandingData-for-test-contract-standing-data.parquet", {
"MDSeqNum": "mkt_data_seq_num",
"RebroadcastIndicator": "rebroadcast_ind",
"SymbolIndex": "symbol_index",
"OptiqSegment": "optiq_segment",
"PartitionID": "partition_id",
"ContractEventDate": "contract_event_date",
"ExchangeCode": "exchange_code",
"ExerStyle": "exercise_style",
"FlexInd": "flex_indicator",
"ContractName": "contract_name",
"ContractType": "contract_type",
"UnderlyingType": "underlying_type",
"PriceDecimals": "price_index_level_decimals",
"QuantityDecimals": "quantity_decimals",
"AmountDecimals": "amount_decimals",
"RatioDecimals": "ratio_multiplier_decimals",
"MainDepositary": "main_depositary",
"MIC": "MIC",
"CountryOfExchange": "country_of_exchange",
"ProductCode": "product_code",
"UnderlyingMIC": "underlying_MIC",
"UnderlyingISINCode": "underlying_isin_code",
"UnderlyingExpiry": "underlying_expiry",
"OrderTypeRules": "order_type_rules",
"SettlementMethod": "settlement_method",
"TradingCurrency": "trading_currency",
"WhRFCDaysBeforeExpiry": "whrfc_days_before_expiry",
"WhRFCMinutesBeforeClosing": "whrfc_minutes_before_closing",
"MinimumQuantityForInitiator": "min_quantity_for_initiator",
"MinimumQuantityForReactor": "min_quantity_for_reactor",
"WhRFCPickUpPerc": "whrfc_pick_up_perc",
"WhRFCImprovementPeriod": "whrfc_improvement_period",
"AvailableWholesaleTradeType": "available_wholesale_trade_type",
"InstrumentDecimalsRatio": "instrument_decimals_ratio",
"InstrumentTickSize": "instrument_tick_size",
"InstrumentSettlementTickSize": "instrument_settlement_tick_size",
"InstrumentEDSPTickSize": "instrument_edsp_tick_size",
"StrikePriceDecimalsRatio": "strike_price_decimals_ratio",
"DeltaProtectforMM": "delta_protect_for_mm",
"VegaProtectforMM": "vega_protect_for_mm",
"VolumeProtectforMM": "volume_protect_for_mm",
"ContractTradingType": "contract_trading_type",
"ThrottleforIncomingOrders": "throttle_for_incoming_orders",
"StrikePriceFlexIncrement": "strike_price_flex_increment",
"PremiumPricingTickSize": "premium_pricing_tick_size",
"PremiumPricingThreshold": "premium_pricing_threshold",
"TickValue": "tick_value",
"OutrightLISTradeThreshold": "outright_lis_trade_threshold",
"StrategyLISTradeThreshold": "strategy_lis_trade_threshold",
"OutrightGCrossTradeThreshold": "outright_gcross_threshold",
"StrategyGCrossTradeThreshold": "strategy_gcross_threshold",
"LotSize": "lot_size",
"InstUnitExp": "intrument_unit_expression",
"TickValueDecimals": "tick_value_decimals",
"PricingAlgorithm": "pricing_algorithm",
"UnderlyingSubtype": "underlying_subtype",
"MotherStockISIN": "mother_stock_isin",
"ReferenceFutureContractSecGrp": "reference_future_contract_secgrp",
"StrategyCode": "strategy_code"
})
from test_util import build_parquet_data, verify_datasets
import sys
official_application = sys.argv[1]
official_files_location = sys.argv[2]
cmdline_params = [
"-p", "TechnicalNotification-for-test", "-o", "1106",
official_files_location + "/20190114/020000/224.0.64.104_11104.pcap.gz"
]
print("Building TechnicalNotification")
build_parquet_data(official_application, cmdline_params)
print("Checking TechnicalNotification")
verify_datasets(
official_files_location + "/Official-TechnicalNotification",
"TechnicalNotification-for-test-technical-notification.parquet", {
"MDSeqNum": "mkt_data_seq_num",
"TechnicalNotificationType": "technical_notification_type",
"RebroadcastIndicator": "rebroadcast_indicator",
"RetransmissionStartTime": "retransmission_start_time",
"RetransmissionEndTime": "retransmission_end_time",
"SymbolIndex": "symbol_index"
})
cmdline_params = [
"-p", "StrategyStandingData-for-test", "-o", "1012",
official_files_location + "/20190104/020000/224.0.65.190_11190.pcap.gz",
official_files_location + "/20190104/020000/224.0.64.200_11200.pcap.gz",
official_files_location + "/20190104/020000/224.0.64.190_11190.pcap.gz"
]
print("Building StrategyStandingData")
build_parquet_data(official_application, cmdline_params)
print("Checking StrategyStandingData")
verify_datasets(
official_files_location + "/Official-StrategyStandingData",
"StrategyStandingData-for-test-strategy-standing-data.parquet", {
"MDSeqNum": "mkt_data_seq_num",
"RebroadcastIndicator": "rebroadcast_ind",
"EMM": "emm",
"SymbolIndex": "symbol_index",
"InstrumentTradingCode": "instrument_trading_code",
"ExchangeCode": "exchange_code",
"MaturityDate": "maturity_date",
"StrategyCode": "strategy_code",
"ContractSymbolIndex": "contract_symbol_index",
"CFI": "cfi",
"LegSymbolIndex": "leg_symbol_index",
"LegPrice": "leg_price",
"LegRatio": "leg_ratio",
"LegBuySell": "leg_buy_or_sell"
})
official_application = sys.argv[1]
official_files_location = sys.argv[2]
cmdline_params = [
"-p", "Timetable-for-test", "-o", "1006",
official_files_location + "/20190114/020000/224.0.64.110_10110.pcap.gz",
official_files_location + "/20190114/020000/224.0.64.120_12120.pcap.gz",
official_files_location + "/20190114/020000/224.0.65.170_10170.pcap.gz"
]
print("Building Timetable")
build_parquet_data(official_application, cmdline_params)
print("Checking Timetable")
verify_datasets(
official_files_location + "/Official-Timetable",
"Timetable-for-test-timetable.parquet", {
"MDSeqNum": "mkt_data_seq_num",
"RebroadcastIndicator": "rebroadcast_indicator",
"EMM": "emm",
"PatternID": "pattern_id",
"SymbolIndex": "symbol_index",
"PhaseTime": "phase_time",
"PhaseId": "phase_id",
"PhaseQualifier": "phase_qualifier",
"TradingPeriod": "trading_period",
"OrderEntryQualifier": "order_entry_qualifier",
"Session": "session"
})
from test_util import build_parquet_data, verify_datasets
import sys
official_application = sys.argv[1]
official_files_location = sys.argv[2]
cmdline_params = [
"-p", "Statistics-for-test", "-o", "1009",
official_files_location + "/20190104/160000/224.0.65.200_11200.pcap.gz",
official_files_location + "/20190104/170000/224.0.64.200_11200.pcap.gz",
official_files_location + "/20190104/183000/224.0.64.190_11190.pcap.gz"
]
print("Building Statistics")
build_parquet_data(official_application, cmdline_params)
print("Checking Statistics")
verify_datasets(
official_files_location + "/Official-Statistics",
"Statistics-for-test-statistics.parquet", {
"MDSeqNum": "mkt_data_seq_num",
"RebroadcastIndicator": "rebroadcast_indicator",
"SymbolIndex": "symbol_index",
"StatsUpdateType": "stats_update_type",
"StatsUpdateValue": "stats_update_value"
})
cmdline_params = ["-p",
"OrderUpdate-for-test",
"-o",
"1002",
official_files_location + "/20190114/114500/224.0.66.123_20123.pcap.gz",
official_files_location + "/20190114/114500/224.0.65.125_10125.pcap.gz",
official_files_location + "/20190114/114500/224.0.65.135_10135.pcap.gz",
official_files_location + "/20190114/020000/224.0.64.104_11104.pcap.gz"]
print("Building OrderUpdate")
build_parquet_data(official_application, cmdline_params)
print("Checking OrderUpdate")
verify_datasets(official_files_location + "/Official-OrderUpdate", "OrderUpdate-for-test-order-update.parquet", {
"MDSeqNum": "mkt_data_seq_num",
"RebroadcastIndicator": "rebroadcast_indicator",
"EMM": "emm",
"EventTime": "event_time",
"SymbolIndex": "symbol_index",
"ActionType": "mkt_data_action_type",
"OrderPriority": "order_priority",
"PreviousPriority": "previous_priority",
"OrderType": "order_type",
"OrderPx": "order_price",
"OrderSide": "order_side",
"OrderQuantity": "order_qty",
"PegOffset": "peg_offset"
})
verify_datasets(official_files_location + "/Official-StandingData", "StandingData-for-test-standing-data.parquet", {
"MDSeqNum": "mkt_data_seq_num",
"RebroadcastIndicator": "rebroadcast_ind",
"SymbolIndex": "symbol_index",
"OptiqSegment": "optiq_segment",
"PartitionID": "partition_id",
"FullInstrumentName": "full_instr_name",
"InstrumentName": "instr_name",
"InstrumentTradingCode": "instr_trdng_code",
"InstrumentGroupCode": "instr_group_code",
"ISINCode": "isin_code",
"PriceDecimals": "price_lvl_decimals",
"QuantityDecimals": "qty_decimals",
"AmountDecimals": "amount_decimals",
"RatioDecimals": "ratio_decimals",
"CFI": "cfi",
"InstrumentEventDate": "instr_event_date",
"StrikePrice": "strike_price",
"DarkEligibility": "dark_eligibility",
"DarkLISThreshold": "dark_lis_thres",
"DarkMinQuantity": "dark_min_qty",
"DateOfLastTrade": "date_of_last_trade",
"DepositaryList": "depositary_list",
"MainDepositary": "main_depositary",
"FirstSettlementDate": "first_settlement_date",
"GuaranteeIndicator": "guarantee_indicator",
"ICB": "icb",
"IssuingCountry": "issuing_country",
"LastAdjustedClosingPrice": "last_adj_close_price",
"LotSize": "lot_size",
"MaturityDate": "maturity_date",
"MaximumDecimalsInQuantity": "max_decimals_qty",
"MIC": "mic",
"MICList": "mic_list",
"CountryOfExchange": "country_of_exchange",
"Mnemonic": "mnemonic",
"UnderlyingMIC": "underlying_mic",
"UnderlyingISINCode": "underlying_isin",
"TradingCurrency": "trading_currency",
"CurrencyCoefficient": "currency_coeff",
"TradingCurrencyIndicator": "trading_currency_indicator",
"StrikeCurrencyIndicator": "strike_currency_indicator",
"NumberInstrumentCirculating": "num_instr_circulating",
"ParValue": "par_value",
"QuantityNotation": "qty_notation",
"InstUnitExp": "instr_unit_expression",
"SettlementDelay": "settlement_delay",
"StrikeCurrency": "strike_currency",
"TaxCode": "tax_code",
"TypeOfCorporateEvent": "type_of_corporate_event",
"TypeOfMarketAdmission": "type_of_mkt_admission",
"RepoIndicator": "repo_indicator",
"IssuePrice": "issue_price",
"NominalCurrency": "nominal_currency",
"IssuePriceDecimals": "issue_price_decimals",
"StrikePriceDecimals": "strike_price_decimals",
"LiquidInstrumentIndicator": "liquid_instrument_indicator",
"MarketOfReferenceMIC": "mkt_of_reference_mic",
"ICBCode": "icb_code",
"EMM": "emm",
"PatternID": "pattern_id",
"TickSizeIndexID": "tick_size_index_id",
"MarketModel": "mkt_model"
},{
'ParValue': lambda v: np.uint64(v) if v else v,
'TypeOfMarketAdmission': convertTypeOfMarketAdmission_enum})
official_files_location + "/20190104//020000/224.0.64.190_11190.pcap.gz",
official_files_location + "/20190104//020000/224.0.65.210_11210.pcap.gz",
official_files_location + "/20190104//020000/224.0.64.200_11200.pcap.gz"
]
print("Building OutrightStandingData")
build_parquet_data(official_application, cmdline_params)
print("Checking OutrightStandingData")
verify_datasets(
official_files_location + "/Official-OutrightStandingData",
"OutrightStandingData-for-test-outright-standing-data.parquet", {
"MDSeqNum": "mkt_data_seq_num",
"RebroadcastIndicator": "rebroadcast_ind",
"SymbolIndex": "symbol_index",
"ContractSymbolIndex": "contract_symbol_index",
"InstrumentEventDate": "instrument_event_date",
"ISINCode": "isin_code",
"CFI": "cfi",
"MaturityDate": "maturity_date",
"OptionType": "option_type",
"InstrumentTradingCode": "instrument_trading_code",
"LotSize": "lot_size",
"StrikePrice": "strike_price",
"LastTradingDate": "last_trading_date",
"UnderlyingInstrumentTradingCode":
"underlying_instrument_trading_code",
"DaysToExpiry": "days_to_expiry",
"EMM": "emm"
})
from test_util import build_parquet_data, verify_datasets
import sys
official_application = sys.argv[1]
official_files_location = sys.argv[2]
cmdline_params = ["-p",
"PriceUpdate-for-test",
"-o",
"1003",
official_files_location + "/20190104/221500/224.0.66.193_21193.pcap.gz",
official_files_location + "/20190104/170000/224.0.64.200_11200.pcap.gz",
official_files_location + "/20190104/084500/224.0.67.193_21193.pcap.gz"]
print("Building PriceUpdate")
build_parquet_data(official_application, cmdline_params)
print("Checking PriceUpdate")
verify_datasets(official_files_location + "/Official-PriceUpdate", "PriceUpdate-for-test-price-update.parquet", {
"MDSeqNum": "mkt_data_seq_num",
"RebroadcastIndicator": "rebroadcast_indicator",
"EMM": "emm",
"EventTime": "event_time",
"PriceType": "mkt_data_price_type",
"SymbolIndex": "symbol_index",
"Price": "price",
"Quantity": "quantity",
"ImbalanceQty": "imbalance_quantity",
"ImbalanceQtySide": "imbalance_qty_side"
})
verify_datasets(
official_files_location + "/Official-FullTradeInformation",
"FullTradeInformation-for-test-full-trade-information.parquet", {
"MDSeqNum": "mkt_data_seq_num",
"RebroadcastIndicator": "rebroadcast_indicator",
"EMM": "emm",
"EventTime": "event_time",
"SymbolIndex": "symbol_index",
"TradingDateTime": "trading_date_time",
"PublicationDateTime": "publication_date_time",
"TradeType": "trade_type",
"MifidInstrumentIdType": "MiFID_instrument_id_type",
"MifidInstrumentID": "MiFID_instrument_id",
"MifidExecutionID": "MiFID_execution_id",
"MifidPrice": "MiFID_price",
"MifidQuantity": "MiFID_quantity",
"MifidPriceNotation": "MiFID_price_notation",
"MifidCurrency": "MiFID_currency",
"MiFIDQtyinMsrmtUnitNotation": "MiFID_qty_in_msrmnt_unit_notation",
"MifidQuantityMeasurementUnit": "MiFID_msrmnt_unit",
"MiFIDNotionalAmount": "MiFID_notional_amount",
"NotionalCurrency": "notional_currency",
"MiFIDClearingFlag": "MiFID_clearing_flag",
"MMTMarketMechanism": "MMT_mkt_mechanism",
"MMTTradingMode": "MMT_trading_mode",
"MMTTransactionCategory": "MMT_transaction_category",
"MMTNegotiationIndicator": "MMT_negotiation_indicator",
"MMTAgencyCrossTradeIndicator": "MMT_agency_cross_trade_indicator",
"MMTModificationIndicator": "MMT_modification_indicator",
"MMTBenchmarkIndicator": "MMT_benchmark_indicator",
"MMTSpecialDividendIndicator": "MMT_special_dividend_indicator",
"MMTOffBookAutomatedIndicator": "MMT_off_book_auto_indicator",
"MMTContributiontoPrice": "MMT_contribution_to_price",
"MMTAlgorithmicIndicator": "MMT_algo_indicator",
"MMTPublicationMode": "MMT_publication_mode",
"MMTPostTradeDeferral": "MMT_post_trade_deferral",
"MMTDuplicativeIndicator": "MMT_duplicative_indicator",
"TradeQualifier": "trade_qualifier",
"TransactionType": "transaction_type",
"EffectiveDateIndicator": "effective_date_indicator",
"BlockTradeCode": "block_trade_code",
"TradeReference": "trade_reference",
"OriginalReportTimestamp": "original_report_timestamp",
"TransparencyIndicator": "transparency_indicator",
"CurrencyCoefficient": "currency_coefficient",
"PriceMultiplier": "price_multiplier",
"PriceMultiplierDecimals": "price_multiplier_decimals",
"Venue": "venue",
"StartTimeVwap": "start_time_vwap",
"EndTimeVwap": "end_time_vwap",
"MiFIDEmissionAllowanceType": "MiFID_emission_allowance_type",
"MarketOfReferenceMIC": "mkt_of_reference_MIC"
})
from test_util import build_parquet_data, verify_datasets
import sys
official_application = sys.argv[1]
official_files_location = sys.argv[2]
cmdline_params = [
"-p", "RealTimeIndex-for-test", "-o", "1008",
official_files_location + "/20190114/114500/224.0.64.170_10170.pcap.gz"
]
print("Building RealTimeIndex")
build_parquet_data(official_application, cmdline_params)
print("Checking RealTimeIndex")
verify_datasets(
official_files_location + "Official-RealTimeIndex",
"RealTimeIndex-for-test-real-time-index.parquet", {
"MDSeqNum": "mkt_data_seq_num",
"RebroadcastIndicator": "rebroadcast_indicator",
"EMM": "emm",
"EventTime": "event_time",
"SymbolIndex": "symbol_index",
"IndexLevel": "index_level",
"PctgOfCapitalization": "percentage_of_capitalization",
"PrctVarfromPrevClose": "percentage_var_from_prev_close",
"NumTradedInstruments": "number_of_traded_instruments_in_index",
"IndexLevelType": "index_level_type",
"IndexPriceCode": "index_price_code"
})
official_files_location + "/20190104/221500/224.0.66.213_21213.pcap.gz",
official_files_location + "/20190104/221500/224.0.66.193_21193.pcap.gz",
official_files_location + "/20190104/084500/224.0.67.193_21193.pcap.gz"]
print("Building MarketStatusChange")
build_parquet_data(official_application, cmdline_params)
print("Checking MarketStatusChange")
verify_datasets(official_files_location + "/Official-MarketStatusChange", "MarketStatusChange-for-test-market-status-change.parquet", {
"MDSeqNum": "mkt_data_seq_num",
"RebroadcastIndicator": "rebroadcast_indicator",
"EMM": "emm",
"ChangeType": "market_data_change_type",
"SymbolIndex": "symbol_index",
"EventTime": "event_time",
"BookState": "book_state",
"StatusReason": "status_reason",
"PhaseQualifier": "phase_qualifier",
"TradingPeriod": "trading_period",
"TradingSide": "trading_side",
"PriceLimits": "price_limits",
"QuoteSpreadMultiplier": "quote_spread_multiplier",
"OrderEntryQualifier": "order_entry_qualifier",
"Session": "session",
"ScheduledEvent": "scheduled_event",
"ScheduledEventTime": "scheduled_event_time"
})
from test_util import build_parquet_data, verify_datasets
import sys
official_application = sys.argv[1]
official_files_location = sys.argv[2]
cmdline_params = [
"-p", "MarketUpdate-for-test", "-o", "1001",
official_files_location + "/20190104/084500/224.0.67.193_21193.pcap.gz"
]
print("Building MarketUpdate")
build_parquet_data(official_application, cmdline_params)
print("Checking MarketUpdate")
verify_datasets(
official_files_location + "/Official-MarketUpdate",
"MarketUpdate-for-test-market-update.parquet", {
"MDSeqNum": "mkt_data_seq_num",
"RebroadcastIndicator": "rebroadcast_indicator",
"EMM": "emm",
"EventTime": "event_time",
"UpdateType": "mkt_data_update_type",
"SymbolIndex": "symbol_index",
"NumberOfOrders": "number_of_orders",
"Price": "price",
"Quantity": "quantity"
})