lpriceOB = []
lsizeOB = []
lsizeOS = []
ldelete = []
ltimeEB = []
ltimeES = []
lpriceES = []
lpriceEB = []
ltimeEP = [] # Ordenes ocultas
lpriceEP = []
lsizeEB = []
lsizeES = []
lsizeEP = []
sorders = OrdersProcessor()
i = 0
norders = 0
rfile = gzip.open(
datapath + 'Messages/' + ITCH_days[year][day] + '-' + stock +
'-MESSAGES.csv.gz', 'rt')
rfile = ITCHMessages(year, day, stock)
rfile.open()
sorders = OrdersProcessor()
# for mess in rfile:
for order in rfile.get_order():
# data = mess.split(',')
from FSociety.ITCH import ITCHv5, ITCHRecord, ITCHtime, ITCHMessages
from FSociety.Util import now, nanoseconds_to_time
from FSociety.Data import Stock, OrdersProcessor, Company, OrdersCounter
from FSociety.Config import datapath, ITCH_days
__author__ = 'bejar'
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--year', help="Anyo del analisis", default='2017G')
parser.add_argument('--day', help="dia del anyo", type=int, default=0)
parser.add_argument('--stock', help="Stock del analisis", default='GOOGL')
args = parser.parse_args()
year = args.year
stock = args.stock
day = args.day
# if 'G' in year:
# datapath = datapath + '/GIS/'
rfile = ITCHMessages(year, day, stock)
rfile.open()
sorders = OrdersProcessor()
for order in rfile.get_order():
# print(order.to_string())
sorders.insert_order(order)
sorders.list_executed(mode='exec', hft=True)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--year', help="Anyo del analisis", default='2017G')
parser.add_argument('--day', help="dia del anyo", type=int, default=0)
parser.add_argument('--stock', help="Stock del analisis", default='GOOGL')
args = parser.parse_args()
year = args.year
stock = args.stock
day = args.day
if 'G' in year:
datapath = datapath + '/GIS/'
rfile = ITCHMessages(year, day, stock)
sorders = OrdersProcessor()
ocounter = OrdersCounter(select=['A', 'F', 'E', 'U', 'C', 'D', 'X'], granularity='m')
rfile.open()
i = 0
for order in rfile.get_order():
print(order.to_string())
sorders.insert_order(order)
ocounter.process_order(order)
i += 1
if i == 100000:
i = 0
ocounter.plot_counter(['A', 'D', 'E'])
ocounter.plot_counter(['A', 'D', 'E'])
if year is None:
year = '2015'
sstock = Stock()
cpny = Company()
day = ITCH_days[year][0]
dhistoD = {}
dhistoS = {}
dhistoB = {}
mxhval = 16 # Max possible time interval
lstocks = sorted(sstock.get_list_stocks())
for stock in lstocks:
print(stock)
sorders = OrdersProcessor()
ldelete = []
lexecutionsS = []
lexecutionsB = []
rfile = open(datapath + 'Messages/' + day + '-' + stock + '-MESSAGES.csv', 'r')
for mess in rfile:
data = mess.split(',')
timestamp = ITCHtime(int(data[1].strip()))
order = data[2].strip()
ORN = data[3].strip()
if order in ['F', 'A']:
if order == 'A':
price = float(data[7].strip())
else:
price = float(data[8].strip())
def order_exec_analysis(year, day, stock, logging=False, market=False):
"""
:param year:
:param day:
:param stock:
:return:
"""
## Structure for collecting statistics
statistics = {v: {} for v in timelines}
for v in statistics:
statistics[v]['buy'] = {s: [] for s in stat}
statistics[v]['sell'] = {s: [] for s in stat}
rfile = ITCHMessages(year, day, stock)
rfile.open()
sorders = OrdersProcessor()
for order in rfile.get_order():
sorders.insert_order(order)
lopen = sorders.sorted_orders(otype='open')
lexecuted = sorders.sorted_orders(otype='executed')
lcancelled = sorders.sorted_orders(otype='cancelled')
# list for storing all the orders in chonological order
lorders = []
# Add to the list an open order with its time and all the partial executions
for o in lopen:
if not market or (time_to_nanoseconds(9, 30) < o.otime <
time_to_nanoseconds(16)):
lorders.append((o.otime, 'O', o.id))
# Orders still open buy maybe partially executed
for xo in range(1, len(o.history)):
if o.history[xo].type in ['C', 'E']:
if not market or (time_to_nanoseconds(
9, 30) < o.history[xo].otime <
time_to_nanoseconds(16)):
lorders.append((o.history[xo].otime, f'OF{xo}', o.id))
# Add to the list an executed order with the time of all the partial
# executions and the last execution
for o in lexecuted:
if not market or (time_to_nanoseconds(9, 30) < o.otime <
time_to_nanoseconds(16)):
lorders.append((o.otime, 'XI', o.id))
# Partial executions
for xo in range(1, len(o.history) - 1):
if o.history[xo].type in ['C', 'E']:
if not market or (time_to_nanoseconds(
9, 30) < o.history[xo].otime <
time_to_nanoseconds(16)):
lorders.append((o.history[xo].otime, f'XF{xo}', o.id))
# Final execution
if not market or (time_to_nanoseconds(9, 30) < o.history[-1].otime
< time_to_nanoseconds(16)):
lorders.append((o.history[-1].otime, f'XF', o.id))
# Add to the list a cancelled order with the time of the initial order
# all the possible partial executions
# and the time of the final cancellation
for o in lcancelled:
if not market or (time_to_nanoseconds(9, 30) < o.otime <
time_to_nanoseconds(16)):
lorders.append((o.otime, 'CI', o.id))
for xo in range(1, len(o.history) - 1):
if o.history[xo].type in ['C', 'E']:
if not market or (time_to_nanoseconds(
9, 30) < o.history[xo].otime <
time_to_nanoseconds(16)):
lorders.append((o.history[xo].otime, f'UF{xo}', o.id))
# Last item should be a cancelation (X) or a cancel/replace (U)
if not market or (time_to_nanoseconds(9, 30) < o.history[-1].otime
< time_to_nanoseconds(16)):
lorders.append((o.history[-1].otime, 'CF', o.id))
lorders = sorted(lorders)
# Processes all the itervals for the orders and registers for all the executions a
# some statistics
cbuy = Counter()
csell = Counter()
weird = 0
texec = 0
for _, op, orderid in lorders:
if op == 'O':
if sorders.orders[orderid].buy_sell == 'B':
cbuy[sorders.orders[orderid].price] += 1
else:
csell[sorders.orders[orderid].price] += 1
# sopen.append(orderid)
elif op == 'CI':
if sorders.cancelled[orderid].buy_sell == 'B':
cbuy[sorders.cancelled[orderid].price] += 1
else:
csell[sorders.cancelled[orderid].price] += 1
# scancel.append(orderid)
elif op == 'CF':
if sorders.cancelled[orderid].buy_sell == 'B':
cbuy[sorders.cancelled[orderid].price] -= 1
else:
csell[sorders.cancelled[orderid].price] -= 1
# scancel.remove(orderid)
elif op == 'XI':
if sorders.executed[orderid].buy_sell == 'B':
cbuy[sorders.executed[orderid].price] += 1
else:
csell[sorders.executed[orderid].price] += 1
# sexec.append(orderid)
else: # it is an execution
# If is a final execution the code is 'XF', else it has a number attached
# The final execution eliminates the price from the order book so the first now is the second best price
exorder = sorders.query_id(orderid)
if len(op) == 2:
if exorder.buy_sell == 'B':
cbuy[exorder.price] -= 1
else:
csell[exorder.price] -= 1
pendingbuy = sorted([v for v in cbuy.items() if v[1] > 0],
reverse=True)
pendingsell = sorted([v for v in csell.items() if v[1] > 0])
# Checks if the queues are empty
bestbuy = pendingbuy[0][0] if len(pendingbuy) > 0 else -1
bestsell = pendingsell[0][0] if len(pendingsell) > 0 else -1
if len(op) == 2:
timeline = in_timeline(exorder.history_time_length())
else:
timeline = in_timeline(
exorder.history_time_length(dist=int(op[2:])))
# print(f'{sorders.executed[orderid].price} {pendingbuy[0]} {pendingsell[0]}')
# If any of the queues is empty the statistics make no sense so they are not computed
if (bestsell != -1) and (bestbuy != -1):
texec += 1
if logging:
print(
f'******************************************** {op[2:]}'
)
print(f'ID: {orderid}')
print(exorder.to_string(history=True))
# Get the execution order number
if len(op) == 2:
hist_exorder = exorder.history[-1]
else:
hist_exorder = exorder.history[int(op[2:])]
# Get the price of the executed order (if the type is C then it is an execution with price)
exprice = hist_exorder.price if exorder.type == 'C' else exorder.price
# gap - the difference between the price of the execution and the best price of the other side
# diff - the difference between the price of the execution and the second best price
if exorder.buy_sell == 'B':
buy_sell = 'buy'
gap = bestsell - exprice
diff = exprice - bestbuy
if logging:
print(
f'BUY: {exprice} / GAP: {gap:3.2f} / DIFF: {diff:3.2f}'
)
else:
buy_sell = 'sell'
gap = exprice - bestbuy
diff = bestsell - exprice
if logging:
print(
f'SELL: {exprice} / GAP: {gap:3.2f} / DIFF: {diff:3.2f}'
)
if logging:
print(f'QSELL5={pendingsell[:5]}')
print(f'QBUY5={pendingbuy[:5]}')
print(f'BBUY={bestbuy} BSELL={bestsell}')
print(
f'LQBUY={sum_count(pendingbuy)} LQSELL={sum_count(pendingbuy)}'
)
if gap < 0 or diff < 0:
weird += 1
pass
# print(f'?????????????????????????????????????????????????????????????')
# print(f'OP: {buy_sell} OTHER= {gap} SECOND= {diff}')
# print(f'ID: {orderid}')
# print(exorder.to_string(history=True))
# print(f'P:{exprice} BS: {bestsell} BB: {bestbuy}')
# print(f'QSELL5={pendingsell[:5]}')
# print(f'QBUY5={pendingbuy[:5]}')
# print(f'BBUY={bestbuy} BSELL={bestsell}')
# print(f'LQBUY={sum_count(pendingbuy)} LQSELL={sum_count(pendingbuy)}')
else:
statistics[timelines[timeline]][buy_sell]['price'].append(
exprice)
statistics[timelines[timeline]][buy_sell]['lenbuy'].append(
sum_count(pendingbuy))
statistics[
timelines[timeline]][buy_sell]['lensell'].append(
sum_count(pendingsell))
statistics[
timelines[timeline]][buy_sell]['lenbuy5'].append(
sum_count(pendingbuy, lim=5))
statistics[
timelines[timeline]][buy_sell]['lensell5'].append(
sum_count(pendingsell, lim=5))
statistics[
timelines[timeline]][buy_sell]['lenbuy10'].append(
sum_count(pendingbuy, lim=10))
statistics[
timelines[timeline]][buy_sell]['lensell10'].append(
sum_count(pendingsell, lim=10))
statistics[timelines[timeline]][buy_sell][
'otherprice'].append(diff)
statistics[timelines[timeline]][buy_sell]['gap'].append(
gap)
statistics[timelines[timeline]][buy_sell]['size'].append(
hist_exorder.size)
print(f"W={weird} TEX={texec}")
for st in stat:
for v in timelines:
statistics[v]['buy'][st] = np.array(statistics[v]['buy'][st])
statistics[v]['sell'][st] = np.array(statistics[v]['sell'][st])
return statistics
def order_statistics(year, day, stock):
"""
Computes statistics for the orders of a stock
:param year:
:param day:
:param stock:
:return:
"""
statistics = {
'buy': {
'ordersize': [],
'ordertime': [],
'orderprice': [],
'executionsize': [],
'executionprice': [],
'executiondeltatime': [],
'executiontime': [],
'deletedeltatime': []
},
'sell': {
'ordersize': [],
'ordertime': [],
'orderprice': [],
'executionsize': [],
'executionprice': [],
'executiontime': [],
'executiondeltatime': [],
'deletedeltatime': []
},
}
i = 0
norders = 0
rfile = ITCHMessages(year, day, stock)
sorders = OrdersProcessor()
rfile.open()
for order in rfile.get_order():
sorders.insert_order(order)
if not args.market or (time_to_nanoseconds(9, 30) < order.otime <
time_to_nanoseconds(16)):
if order.type in ['F', 'A', 'U']:
norders += 1
# if 0 < order.price < 5000:
if order.buy_sell == 'S':
statistics['sell']['ordersize'].append(order.size)
statistics['sell']['ordertime'].append(order.otime)
statistics['sell']['orderprice'].append(order.price)
else:
statistics['buy']['ordersize'].append(order.size)
statistics['buy']['ordertime'].append(order.otime)
statistics['buy']['orderprice'].append(order.price)
# If is a cancel/replace order consider also a deletion
if order.type in ['U']:
trans = sorders.query_id(order.oid)
if order.buy_sell == 'S':
statistics['sell']['deletedeltatime'].append(order.otime -
trans.otime)
else:
statistics['buy']['deletedeltatime'].append(order.otime -
trans.otime)
# Computes the time between placing and order and canceling it
if order.type == 'D':
trans = sorders.query_id(order.id)
if trans is not None:
if trans.buy_sell == 'S':
statistics['sell']['deletedeltatime'].append(
order.otime - trans.otime)
else:
statistics['buy']['deletedeltatime'].append(
order.otime - trans.otime)
else:
print('MISSING DELETED' + order.id)
# Computes the time between placing and order and its execution
if order.type in ['E', 'C']:
trans = sorders.query_id(order.id)
if trans.buy_sell == 'S':
statistics['sell']['executiondeltatime'].append(
order.otime - trans.otime)
statistics['sell']['executiontime'].append(order.otime)
if order.type == 'E':
statistics['sell']['executionprice'].append(
trans.price)
else: # Execution with price
statistics['sell']['executionprice'].append(
order.price)
statistics['sell']['executionsize'].append(order.size)
else:
statistics['buy']['executiondeltatime'].append(
order.otime - trans.otime)
statistics['buy']['executiontime'].append(order.otime)
if order.type == 'E':
statistics['buy']['executionprice'].append(trans.price)
else: # Execution with price
statistics['buy']['executionprice'].append(order.price)
statistics['buy']['executionsize'].append(order.size)
# Convert everything to numpy arrays
for v in statistics:
for att in statistics[v]:
statistics[v][att] = np.array(statistics[v][att])
return statistics