def getCachedBacktrackingCandles(looper, dir, highSlice, lowSlice, since, till, lowAheadOfHigh=True):
""" use cached data to provide candles for a high-slice and low-slice, on a [since,till] time range
highSlice must be higher than lowSlice (e.g H1 for high slice, M1 for low slice)
What is returned is a list L of pairs ([0],[1]), where the 1st element of the pair L[n][0] is the single high candle,
and the 2nd element of the pair L[n][1] is the list of low candles contained under L[n][0].
when lowAheadOfHigh is true, then all elements in L[n][1] have a time that is before L[n][0].
When lowAheadOfHigh is false, then all elemetns in L[n+1][1] have a time that is before L[n][0]
Example times for M15 vs M5 slices:
lowAheadOfHigh: [ (01:15:00, [ 01:15:00, 01:20:00, 01:25:00 ] ]), (01:30:00, [01:30:00, 01:35:00, 01:40:00 ]), ...]
not lowAheadOfHigh: [ (01:15:00, [ 01:00:00, 01:05:00, 01:10:00 ] ]), (01:30:00, [01:15:00, 01:20:00, 01:25:00 ]), ...] """
from candlecache import SliceRowIterator
hIterator = SliceRowIterator(dir, looper.instrumentName, highSlice, since, till, looper.api)
lIterator = SliceRowIterator(dir, looper.instrumentName, lowSlice, since, till, looper.api)
highCandles = [c for c in hIterator]
if(len(highCandles)<2):
raise ValueError("cannot work on so few high slice candles...")
hi =0
backtracking = []
pouch = []
hitime = highCandles[hi].time
hntime = highCandles[hi+1].time
xoff = 1 if(not lowAheadOfHigh) else 0
if(not lowAheadOfHigh):
yup = (highCandles[0], [])
backtracking.append(yup)
for lc in lIterator:
ctime = lc.time
ci = cmp(ctime, hitime)
cn = cmp(ctime, hntime)
if(ci<0):
continue
elif(ci>=0 and cn<0):
pouch.append(lc)
elif(cn>=0):
if(hi+xoff<len(highCandles)):
yup = ( highCandles[hi+xoff], pouch )
backtracking.append(yup)
pouch = [ lc ]
hitime = hntime
hi += 1
if(hi+1>=len(highCandles)):
hntime = "9999-99-99T99:99:99.999999999Z"
else:
hntime = highCandles[hi+1].time
if(lowAheadOfHigh):
yup = ( highCandles[-1], pouch )
backtracking.append(yup)
return backtracking
def collectForMonth(pair, year, month, dir, granularity="S5", refresh=True):
kwargs = {"granularity":granularity, "price":"MBA"}
batches = ( ("00:00:00", "03:59:59"), ("04:00:00", "07:59:59"),
("08:00:00", "11:59:59"), ("12:00:00", "15:59:59"),
("16:00:00", "19:59:59"), ("20:00:00", "23:59:59"))
precache = []
daysOfMonth = (00,31,28,31,30,31,30,31,31,30,31,30,31)
days = daysOfMonth[month]
month = str(month)
if(len(month)==1): month = "0" + month
if(month=="02"):
if(year % 4 == 0 and (not day % 100 ==0 or day % 400 ==0)):
days+=1
if(not refresh):
from candlecache import SliceRowIterator
_since = "{}-{}-{}T00:00:00.000000000Z".format(year, month, "01")
_till = "{}-{}-{}T23:59:59:999999999Z".format(year, month, days)
precache = list( SliceRowIterator(dir,pair,granularity, _since, _till, api) )
allCandles = []
timeToBreak = False
if(len(precache)>0 and not refresh):
for c in precache:
row = [c.time, c.bid.o, c.ask.o, c.mid.o, c.bid.l, c.ask.l, c.mid.l, c.bid.h, c.ask.h, c.mid.h, c.bid.c, c.ask.c, c.mid.c, c.volume ]
allCandles.append(row)
header = ["time", "bid-o", "ask-o", "mid-o", "bid-l", "ask-l", "mid-l", "bid-h", "ask-h", "mid-h", "bid-c", "ask-c", "mid-c", "volume"]
for day in range(days):
d = str(day+1)
if(len(d)==1): d = "0" + d
dstamp = "{}-{}-{}".format(year,month,d)
for b in batches:
fromTs = "{}T{}.000000000Z".format(dstamp, b[0])
toTs = "{}T{}.000000000Z".format(dstamp, b[1])
kwargs["fromTime"]=fromTs
kwargs["toTime"]=toTs
if(not refresh):
if(len(allCandles)>0 and cmp(toTs, allCandles[-1][0])<0):
# no need to refresh what is way in the past...
continue
print kwargs
resp = api.instrument.candles(pair, **kwargs)
# import pdb; pdb.set_trace()
if(str(resp.status) != '200'):
if(resp.body.has_key("errorMessage") and resp.body["errorMessage"] == "Invalid value specified for 'to'. Time is in the future"):
del kwargs['toTime']
print ('future is not accepted', kwargs)
resp = api.instrument.candles(pair, **kwargs)
timeToBreak = True
else:
print resp.body
candles = resp.get('candles',200)
candles.sort(lambda a,b: cmp(a.time, b.time))
for c in candles:
if(len(allCandles)>0 and cmp(c.time, allCandles[-1][0])<0): continue
row = [c.time, c.bid.o, c.ask.o, c.mid.o, c.bid.l, c.ask.l, c.mid.l, c.bid.h, c.ask.h, c.mid.h, c.bid.c, c.ask.c, c.mid.c, c.volume ]
allCandles.append(row)
if(timeToBreak): break
if(timeToBreak): break
with open( "{}/{}-{}.{}.{}.csv".format(dir, year, month, pair, granularity), "wb") as outf:
csvwriter = csv.writer(outf)
for r in allCandles:
csvwriter.writerow(r)