def readOurFillsFile(datestr):
ourFillsFilename = os.environ[
'RUN_DIR'] + '/../' + datestr + '/fills.' + datestr + '.txt'
ourFillsFile = open(ourFillsFilename, 'r')
fillsLines = util.csvdict(ourFillsFile)
orderid2fills = {}
for line in fillsLines:
# type|date|strat|seqnum|secid|ticker|ts_received|ts_exchange|shares|price|exchange|liquidity|orderID|tactic
orderid = long(line['orderID'])
fillIndex = int(line['seqnum'])
ts_exchange = long(line['ts_exchange'])
if orderid not in orderid2fills:
orderid2fills[orderid] = []
orderid2fills[orderid].append({
'fillIndex': fillIndex,
'secid': int(line['secid']),
'ticker': line['ticker'],
'ts_received': long(line['ts_received']),
'ts_exchange': ts_exchange,
'qty': int(line['shares']),
'price': float(line['price']),
'exch': line['exchange'],
'liq': line['liquidity'],
'orderid': orderid,
'tactic': line['tactic'].strip(),
'matched': False
})
# util.info('Returning from readOurFillsFile')
return orderid2fills
def loadFillsFile(filepath):
secid2orders = {}
with open(filepath, "r") as file:
for row in util.csvdict(file):
secid = int(row["secid"])
shares = int(row["shares"])
price = float(row["price"])
orders = secid2orders.get(secid, None)
if orders is None:
orders = []
secid2orders[secid] = orders
orders.append((shares, price))
return secid2orders
def process(filepath, source):
info = datafiles.read_info_file(filepath)
if info["date_last_absent"] is not None:
backfill = 0
timestamp = util.convert_date_to_millis(info["date_first_present"])
else:
backfill = 1
timestamp = util.convert_date_to_millis(info["date_modified"])
database.setAttributeAutoCreate(True)
bad = 0
data = util.csvdict(open(filepath))
for row in data:
ticker = row["Symbol"]
secid = database.getSecidFromXref("TIC", ticker, timestamp,
"compustat_idhist",
newdb.xrefsolve.preferUS)
if secid is None:
continue
try:
date = util.convert_date_to_millis(row["Record_Date"])
except:
util.warning("Bad date for row: " + str(row))
bad += 1
if bad > 20:
util.error(
str(bad) +
" bad lines found. Raising excpeption. Go check file " +
filepath)
raise Exception(
str(bad) +
" bad lines found. Raising excpeption. Go check file " +
filepath)
for sqAtt, ourAtt in attributeMap.iteritems():
name = ourAtt[0]
compareWithRecent = ourAtt[1]
value = row[sqAtt]
if value == '': value = None
database.insertAttribute("sec", "n", secid, date, source, name,
value, timestamp, None, backfill, False,
compareWithRecent, approximatelyEqual)
print('Unknown liquidity type ' + liq)
return ''
def check_add_liquidity(input, liq, value):
category = get_liq_str(liq)
check_add_category(input, 'liq' + category, value)
# Read the orders files in RUN_DIR, and create an orderID to aggressiveness mapping
fs = file_source.FileSource(os.environ['RUN_DIR'] + '/orders/')
listing = fs.list('orders\.[0-9_]+\.txt')
oid2aggr = {}
for item in listing:
orderfile = open(os.environ['RUN_DIR'] + '/orders/' + item[0], 'r')
reader = util.csvdict(orderfile)
for row in reader:
orderid = long(row['orderid'])
aggr = float(row['aggr'])
oid2aggr[orderid] = aggr
orderfile.close()
# Download fills file from each server
REMOTE_LOG_DIR = os.environ['EXEC_LOG_DIR']
exchanges = set()
algos = set()
aggrs = set()
liqs = set()
trade_cfg['servers'].extend(moc_trade_cfg['servers'])
for (host, port) in trade_cfg['servers']:
server_num = re.findall('[0-9]+', host)[0]
def idle(self):
if MKT_CLOSE_TIME / 1000 < time.time():
util.debug("Exceeded market close time.")
self.close()
return
now = datetime.datetime.utcnow()
if now - self.last_idle_time < IDLE_TIME:
return
self.last_idle_time = now
#sleep the first few times to make sure that guillotine initializes
if self.guillotine_wait_times < NUM_WAIT_TIMES_IN_IDLE:
self.guillotine_wait_times += 1
util.log('waiting for guillotine to initialize')
return
if len(self.gtc.symmap) == 0:
util.log('not submitting, not connected')
return
#self._check_trading()
listing = self.fs.list('orders\.[0-9_]+\.txt')
if len(listing) == 0:
return
listing.sort(key=lambda x: x[0], reverse=True)
orderfilename = listing[0][0]
orderfileStat = os.stat(os.environ['RUN_DIR'] + '/orders/' +
orderfilename)
if math.trunc(orderfileStat.st_ctime) <= self.last_submitted_time:
return
util.log("Submitting trades in {}".format(orderfilename))
orderfile = open(os.environ['RUN_DIR'] + '/orders/' + orderfilename,
'r')
reader = util.csvdict(orderfile)
trades = set()
allSymbols = filter(lambda key: self.gtc.symmap[key] != None,
self.gtc.symmap.keys())
tradeSymbols = []
#gather trades
currentSecid = None
currentTicker = None
buyDollars = 0
sellDollars = 0
try:
for row in reader:
#save this info for error messages
currentSecid = None
currentTicker = None
qty = int(float(row['shares']))
if qty == 0:
continue
sec = int(row['secid'])
currentSecid = sec
ticker = sec2tic[sec]
currentTicker = ticker
if qty > 0:
buyDollars += qty * float(row['oprice'])
else:
sellDollars += qty * float(row['oprice'])
aggr = float(row['aggr'])
orderID = long(row['orderid'])
self.gtc.test_trade(ticker, qty, aggr, orderID)
trades.add((sec, ticker, qty, aggr, orderID))
tradeSymbols.append(ticker)
except KeyError:
msg = 'submission failed, key error ' + str(
currentSecid
) + ' ' + str(
currentTicker
) + ". If the ticker is None, the problem happened when translating the secid to ticker. Else, the ticker was not in the guilottine symbol map"
util.log(msg)
util.email("[ERROR] livetrader " + orderfilename, msg)
return
#now actually send the trades. not that if we have a super=unexpected error, we will mark the file as done, so as not to screw things even more
#by resending
report = []
report.append("Buy $ = {:,.2f}".format(buyDollars))
report.append("Sell $ = {:,.2f}".format(sellDollars))
report.append("secid|ticker|qty|aggr|orderID")
setZeroQtySymbols = [
sym for sym in allSymbols if sym not in tradeSymbols
]
setZeroQtySymbols = filter(lambda key: key in tic2sec,
setZeroQtySymbols)
try:
for trade in trades:
report.append("{}|{}|{}|{}|{}".format(trade[0], trade[1],
trade[2], trade[3],
trade[4]))
self.gtc.trade(trade[1], trade[2], trade[3], trade[4])
for sym in setZeroQtySymbols:
report.append("{}|{}|{}|{}|{}".format(tic2sec[sym], sym, 0,
1.0, -1))
self.gtc.trade(sym, 0, 1.0, -1)
except KeyError:
msg = 'Submission half-done due to a super-unexpected key error. I have not clue why we reached this point'
util.log(msg)
util.email("[ERROR] livetrader " + orderfilename, msg)
else:
util.email(
"livetrader submitted successfully {} trades from file {}".
format(str(len(trades)), orderfilename), "\n".join(report))
# self.last_submitted = tradefilename
self.last_submitted_time = math.trunc(orderfileStat.st_ctime)
self.last_trade_time = datetime.datetime.utcnow()
util.log('submitted %s' % orderfilename)