def chopMonthToDays(bararray,sym,dur,secs): ### what does this do?
lines = bararray
print secs
preve = 0
prevfname = 0
ubars =[]
uubars =[]
diffprev = 1
daylist =[]
for l in lines:
if len(l) > 2:
timestring = l[1]
date = timestring[0:11]
daylist.append(date)
daylistu = rpu_rp.uniq(daylist)
for day in daylistu:
daysarray=[]
for l in bararray:
if day in l[1] :
timestring = l[1]
try:
e = TicksUtile.convertTime(timestring,'dashspace','timetoepoch')
except:
e = TicksUtile.convertTime(timestring,'dashspace','timetoepoch')
diffcur = e - preve
preve = e
## print diffcur,timestring
if diffcur == int(secs):
daysarray.append(l)
## print l
print day, sym,dur, len(daysarray)
fileoutname = DataDown + day +'.' + sym +'.' + dur+'.cleaned.csv'
rpu_rp.WriteArrayToCsvfile(fileoutname, daysarray)
def create_timelist(barfile):
fileoutname = 'barsout.csv'
lines = rpu_rp.CsvToLines(barfile)
preve = 0
prevfname = 0
ubars =[]
uubars =[]
diffprev = 1
timelist = []
for l in lines:
if len(l) > 2:
## print l
timestring = l[2]
## e = TicksUtile.time_to_epoch(timestring)
timelist.append(timestring)
timelistu = rpu_rp.uniq(timelist)
return timelistu
def show_by_fnum(fullarray, fnum):
c = 0
x = len(fullarray)
timelist = []
hourlist = []
for l in fullarray:
timelist.append(l[fnum])
c += 1
if c > (x - 8):
print l
timelistu = rpu_rp.uniq(timelist)
c = 0
lengtu = len(timelistu)
for t in timelistu:
totalsize = 0
c += 1
netdelta = 0
upticks = downticks = neutralticks = upticksize = downticksize = neutralticksize = 0
for l in fullarray:
## print l
tsize = l[2]
tprice = l[1]
if t == l[fnum]:
totalsize += tsize
if l[5] == 'uptick':
upticks += 1
upticksize += tsize
elif l[5] == 'downtick':
downticks += 1
downticksize += tsize
else:
neutralticks += 1
neutralticksize += tsize
if l[0] == 'HitBid':
netdelta = netdelta - l[2]
elif l[0] == 'LiftOffer':
netdelta = netdelta + l[2]
if c > (lengtu - 5):
print t, totalsize, netdelta, upticks, downticks, neutralticks, upticksize, downticksize, neutralticksize, tprice
def show_by_fnum(fullarray,fnum):
c=0
x = len(fullarray)
timelist =[]
hourlist =[]
for l in fullarray:
timelist.append(l[fnum])
c+=1
if c > (x - 8):
print l
timelistu = rpu_rp.uniq(timelist)
c=0
lengtu = len(timelistu)
for t in timelistu:
totalsize = 0
c+=1
netdelta =0
upticks = downticks = neutralticks = upticksize = downticksize = neutralticksize = 0
for l in fullarray:
## print l
tsize = l[2]
tprice = l[1]
if t == l[fnum]:
totalsize += tsize
if l[5] == 'uptick':
upticks +=1
upticksize += tsize
elif l[5] == 'downtick':
downticks +=1
downticksize += tsize
else:
neutralticks +=1
neutralticksize += tsize
if l[0] == 'HitBid':
netdelta =netdelta - l[2]
elif l[0] == 'LiftOffer':
netdelta =netdelta + l[2]
if c > (lengtu - 5):
print t, totalsize, netdelta, upticks,downticks,neutralticks,upticksize,downticksize,neutralticksize,tprice
def show_by_fnum(fullarray, fnum):
c = 0
x = len(fullarray)
timelist = []
hourlist = []
for l in fullarray:
timelist.append(l[fnum])
c += 1
if c > (x - 8):
print l
timelistu = rpu_rp.uniq(timelist)
c = 0
lengtu = len(timelistu)
for t in timelistu:
## print t
totalsize = 0
HitLift = 0
c += 1
netdelta = 0
upticks = downticks = HitLift = 0
sumtagval = totsize = 0
for l in fullarray:
bid = l[1]
ask = l[2]
tsize = l[5]
tprice = l[7]
sumtag = l[fnum]
if t == sumtag:
totsize += tsize
sumtagval += 1
if tprice == bid:
quoteflag = 'HitBid'
HitLift = HitLift - 1
else:
quoteflag = 'LIFT'
## print bid
HitLift = HitLift + 1
## print HitLift
print '%-8s %8s %8s %8s price,vsumtagval, totsize,vHitLift' % (
t, sumtagval, totsize, HitLift)
def show_by_fnum(fullarray,fnum):
c=0
x = len(fullarray)
timelist =[]
hourlist =[]
for l in fullarray:
timelist.append(l[fnum])
c+=1
if c > (x - 8):
print l
timelistu = rpu_rp.uniq(timelist)
c=0
lengtu = len(timelistu)
for t in timelistu:
## print t
totalsize = 0
HitLift = 0
c+=1
netdelta =0
upticks = downticks = HitLift = 0
sumtagval = totsize = 0
for l in fullarray:
bid = l[1]
ask = l[2]
tsize = l[5]
tprice = l[7]
sumtag = l[fnum]
if t == sumtag:
totsize += tsize
sumtagval +=1
if tprice == bid:
quoteflag = 'HitBid'
HitLift = HitLift -1
else:
quoteflag = 'LIFT'
## print bid
HitLift = HitLift +1
## print HitLift
print '%-8s %8s %8s %8s price,vsumtagval, totsize,vHitLift' %(t,sumtagval,totsize,HitLift)
prevpnew = low
pnew = low
while pnew <= high:
linearray =[]
pnew = prevpnew + incr
## print pnew,barid
prevpnew = float(pnew)
linearray.append(pnew)
linearray.append(barid)
linearray.append(volumeraw)
farray.append(linearray)
plist =[]
varray =[]
for p in farray:
plist.append(p[0])
pricelist = rpu_rp.uniq(plist)
for price in pricelist:
vlarray=[]
vlarray.append(price)
volumer = ''
totalhits =0
totalvolume =0
for l in farray:
if l[0] == price:
mark = l[1]
mark = '1'
if len(l) > 1:
volumer = str(l[2]).replace(' ','')
if volumer == 'full':
volumer = '100'
totalhits+=1
prevpnew = low
pnew = low
while pnew <= high:
linearray = []
pnew = prevpnew + incr
## print pnew,barid
prevpnew = float(pnew)
linearray.append(pnew)
linearray.append(barid)
linearray.append(volumeraw)
farray.append(linearray)
plist = []
varray = []
for p in farray:
plist.append(p[0])
pricelist = rpu_rp.uniq(plist)
for price in pricelist:
vlarray = []
vlarray.append(price)
volumer = ''
totalhits = 0
totalvolume = 0
for l in farray:
if l[0] == price:
mark = l[1]
mark = '1'
if len(l) > 1:
volumer = str(l[2]).replace(' ', '')
if volumer == 'full':
volumer = '100'
totalhits += 1
def printResults(dict1):
results = list(filter(lambda x: len(x) > 1, dict1.values()))
if len(results) > 0:
print('Duplicates Found:')
print(
'The following files are identical. The name could differ, but the content is identical'
)
print('___________________')
resultnum = 0
for result in results:
## print result
## resultnum +=1
## if resultnum == len(results):
## print 'this is the last result of the pair', len(result)
words = []
upperwords = []
subresultnum = 0
for subresult in result:
subresultnum += 1
lengsubresult = len(result)
## print lengsubresult
file1 = subresult
filenostem = file1.replace(maindir, '')
## print filenostem
fileline = filenostem.replace('\\', ' ').replace('_',
' ').replace(
'-', ' ')
justfilename = filenostem.split('\\')[
len(filenostem.split('\\')) - 1]
partfolder = filenostem.replace(justfilename, '')
print partfolder, justfilename
file1words = fileline.split()
#### print file1words
lenwords = len(file1words)
## print lenwords
c = 0
for wrd in file1words:
c += 1
## print c,wrd
if c == lenwords:
lastwrdsplit = wrd.upper().split('.')
## print lastwrdsplit
if len(lastwrdsplit) == 2:
lastwrd = lastwrdsplit[0]
else:
lastwrd = wrd.uppper()
exten = lastwrdsplit[len(lastwrdsplit) - 1]
wrd = lastwrd
pass
else:
wrd = wrd.upper()
upperwords.append(wrd.upper())
## print exten, lastwrd, 'extension'
if lengsubresult == subresultnum:
print 'lastdupe of pair'
newfname = ''
upperwordsu = rpu_rp.uniq(upperwords)
for wordup in upperwordsu:
newfname = newfname + ' ' + wordup
## newfnameNext = outputcopiesdir +partfolder+ newfname+'dupexx'+'.'+exten
newfnameNext = outputcopiesdir + newfname + 'dupexx' + '.' + exten
print newfnameNext
## print maindir,'maindir'
## fpath = os.path.join(maindir, newfnameNext)
## print newfnameNext
shutil.copyfile(subresult, newfnameNext)
## print 'copy',subresult,'to',newfnameNext
print('___________________')
else:
print('No duplicate files found.')
def get_dload_barsWbu(start_path,dur,barfileout,sym):
import TicksUtile
total_size = 0
alllines =[]
alltimes =[]
fileoutname = barfileout
for dirpath, dirnames, filenames in os.walk(start_path):
## print dirpath
subdirsize = 0
for f in filenames:
## if 'ES.'+dur+'.ddload' in f :
if '.'+sym+'.'+dur+'.ddload' in f :
fp = os.path.join(dirpath, f)
total_size += os.path.getsize(fp)
############## print fp,'working'
lines = rpu_rp.CsvToLines(fp)
for l in lines:
newl =[]
if len(l) > 2:
timestring = l[1]
e = TicksUtile.convertTime(timestring,'dashspace','timetoepoch')
## fsourcedate = f.replace('.ES.'+dur+'.ddload.csv','')
fsourcedate = f.split('.')[0]
fsourcesym = f.split('.')[1]
fsourcedur = f.split('.')[2]
l.append(fsourcedate)
l.append(fsourcesym)
l.append(fsourcedur)
newl.append(l[8])
newl.append(l[9])
newl.append(l[1])
newl.append(l[7])
newl.append(l[2])
newl.append(l[3])
newl.append(l[4])
newl.append(l[5])
newl.append(l[6])
alllines.append(newl)
from operator import itemgetter, attrgetter, methodcaller
sortedall = sorted(alllines,key=itemgetter(0,1,2,3),reverse=True)
usort = rpu_rp.uniqnosort(sortedall)
prevdur = time = ''
cleans =[]
for y in usort:
## print y
dur = y[2]
time = y[3]
if dur == prevdur and time == prevtime:
## print y
## print prevy,'prev'
pass
else:
cleans.append(y)
prevdur = dur
prevtime = time
prevy = y
## rpu_rp.WriteArrayToCsvfile(fileoutname+'news', cleans)
newalls=[]
for n in cleans:
## print n
newn =[]
sym = n[0]
dur = n[1]
time = n[2]
openp = n[4]
hip = n[5]
lowp = n[6]
closep = n[7]
volu = n[8]
newn.append(sym)
newn.append(time)
newn.append(openp)
newn.append(hip)
newn.append(lowp)
newn.append(closep)
newn.append(volu)
newalls.append(newn)
sortednewalls = sorted(newalls,key=itemgetter(0),reverse=False)
sortednewallsu = rpu_rp.uniq(sortednewalls)
sortednewallsuClean = []
preve = 0
for l in sortednewallsu:
timestring = l[1]
try:
e = TicksUtile.convertTime(timestring,'dashspace','timetoepoch')
except:
print timestring,' did not work, needs old format'
e = TicksUtile.convertTime(timestring,'dashspace','timetoepoch')
diffcur = e - preve
preve = e
if diffcur >= int(secs) or '22:30:00' in timestring:
sortednewallsuClean.append(l)
else:
## print diffcur,timestring,dur,'did not pass test'
pass
lenfull = len(sortednewallsuClean)
third1 = int(round(lenfull/3))
third2 = 2*third1
third3 = lenfull - third2 + 1
print lenfull,third1,third2,third3,dur,sym
part33 = rpu_rp.head_array_to_array(sortednewallsuClean,third1)
third2array = rpu_rp.head_array_to_array(sortednewallsuClean,third2)
part34 = rpu_rp.tail_array_to_array(third2array,third1)
part35 = rpu_rp.tail_array_to_array(sortednewallsuClean,third3)
rpu_rp.WriteArrayToCsvfile(fileoutname, part33)
third2name = fileoutname.replace('33','34')
rpu_rp.WriteArrayToCsvfile(third2name, part34)
third3name = fileoutname.replace('33','35')
rpu_rp.WriteArrayToCsvfile(third3name, part35)
def checkRTnonbarticksNEW(sym,fnums):
fullarray =[]
filein = DataDown +date+'.'+sym+ '.RTtickData.csv'
prevtotdiff = ask = bid =0
prevtime =''
string =''
prevtradeuid=''
time = ''
size = ''
tprice = 0.0
tsize = singletrade = totdaysize= vwap =''
totdowns = totups = 0
tottickups=0
nettickhits = 0
lastsize = lastprice = bidsize = asksize = 0
prevtick = nettickups = totHits = totLifts = 0
timeminarray =[]
for line in rpu_rp.tail_array_to_array(rpu_rp.CsvToLines(filein),5000):
## print line
if 'tickString' in str(line) and 'tickType=48' in str(line):
time = line[3]
timemin = time[0:8]
## print timemin # '22:58:36.067000'
string = line[2].split(';')
if len(string) > 2:
tprice = float(string[0].replace('value=',''))
tsize = float(string[1])
singletrade = string[5]
totdaysize = string[3]
vwaps = string[4]
if len(vwaps) > 0:
vwap = round(float(string[4]),2)
else:
vwap =0.0
## value=1984.50;1;1441796865858;207069;1982.49963539;true>
## print tsize,tprice,time,'trade', bid, ask, bidsize, asksize
if 'tickSize' in str(line):
## 0 BID_SIZE tickSize()
## 1 BID_PRICE tickPrice()
## 2 ASK_PRICE tickPrice()
## 3 ASK_SIZE tickSize()
## 4 LAST_PRICE tickPrice()
## 5 LAST_SIZE tickSize()
fs = line[1].replace(' field=','')
sizetime = line[3]
value = int((line[2].replace(' size=','')).replace('>',''))
size = value
######## print sizetime,size,fs,'size'
if fs == '0':
bidsize = value
elif fs == '3':
asksize = value
elif fs =='5':
lastsize = value
else:
pass
if 'tickPrice' in str(line):
pricetime = line[4]
value = float(line[2].replace(' price=',''))
fp = line[1].replace(' field=','')
if fp == '1':
bid= value
pass
elif fp == '2':
ask = value
else:
lastprice = value
## if bid == 0.0:
tradeuid = time + str(tsize) + str(tprice)
if 'tickString' in str(line) and len(string) > 2 and bid > 0 and ask > 0 and tradeuid != prevtradeuid:
if tprice > prevtick:
tickdir='uptick'
elif tprice < prevtick:
tickdir = 'downtick'
else:
tickdir = 'neutral'
if bid == tprice:
tickflag = 'HitBid'
else:
tickflag = 'LiftOffer'
aout =[]
prevtick = tprice
if tsize > 0 and time != 'xxx':
aout.append(tickflag)
aout.append(tickdir)
aout.append(tprice)
aout.append(tsize)
aout.append(bid)
aout.append(ask)
aout.append(time)
aout.append(timemin)
timeminarray.append([timemin])
## print aout
fullarray.append(aout) ### how many upticks in a row?
prevtime = time
prevtradeuid = tradeuid
## rpu_rp.WriteArrayToCsvfile(libarea +'blatmp', timeminarray)
## print fullarray
## timesecs = rpu_rp.CsvToLines(libarea +'blatmp')
utimesecs = rpu_rp.uniq(timeminarray)
fullscore =0
for t in utimesecs:
counter =0
countertick = 0
for l in fullarray:
if t[0] in str(l) and l[3] > 4:
price = l[2]
## print l
if l[0] == 'HitBid':
counter -=1
pass
else:
counter +=1
if l[1] == 'uptick':
countertick +=1
pass
elif l[1] == 'downtick':
countertick -=1
pass
else:
pass
fullscore +=counter+countertick
if counter != 7777:
print ('%4s %4s %4s %s %s' % (fullscore,counter,countertick, t[0], price))
## print timesecs
c=0
x = len(fullarray)
## print x
limiter = 90
f = x - limiter
## print f
netlifts = netticks = 0
lifthitscore = 0
## print 'netlifts',' netUPticks liftscore'
for l in fullarray:
## print l
## ['LiftOffer', 'uptick', 1995.5, 2.0, 1995.25, 1995.5, '15:49:02.881000']
## ['HitBid', 'neutral', 1995.5, 50.0, 1995.5, 1995.75, '15:49:02.894000']
c+=1
if c > f:
## print l
tprice = l[2]
lifttag=l[0]
tsize = l[3]
ticktag=l[1]
if lifttag == 'HitBid':
netlifts = netlifts -1
lifthitscore += (tsize * (-1))
pass
else:
netlifts +=1
lifthitscore += (tsize * (1))
if ticktag == 'downtick':
netticks = netticks -1
pass
elif ticktag == 'uptick':
netticks +=1
else:
pass
if c > (x - 8):
pass
## print l
print ('%4d %4d | %4d %4.1f %s netlifts netUPticks liftscore' % (netticks, lifthitscore, x, limiter,tprice))
def printResults(dict1):
results = list(filter(lambda x: len(x) > 1, dict1.values()))
if len(results) > 0:
print('Duplicates Found:')
print('The following files are identical. The name could differ, but the content is identical')
print('___________________')
resultnum =0
for result in results:
## print result
## resultnum +=1
## if resultnum == len(results):
## print 'this is the last result of the pair', len(result)
words = []
upperwords=[]
subresultnum = 0
for subresult in result:
subresultnum+=1
lengsubresult = len(result)
## print lengsubresult
file1 = subresult
filenostem = file1.replace(maindir,'')
## print filenostem
fileline = filenostem.replace('\\',' ').replace('_',' ').replace('-',' ')
justfilename = filenostem.split('\\')[len(filenostem.split('\\'))-1]
partfolder = filenostem.replace(justfilename,'')
print partfolder, justfilename
file1words = fileline.split()
#### print file1words
lenwords = len(file1words)
## print lenwords
c=0
for wrd in file1words:
c+=1
## print c,wrd
if c==lenwords:
lastwrdsplit = wrd.upper().split('.')
## print lastwrdsplit
if len(lastwrdsplit) == 2:
lastwrd = lastwrdsplit[0]
else:
lastwrd = wrd.uppper()
exten = lastwrdsplit[len(lastwrdsplit)-1]
wrd = lastwrd
pass
else:
wrd = wrd.upper()
upperwords.append(wrd.upper())
## print exten, lastwrd, 'extension'
if lengsubresult == subresultnum:
print 'lastdupe of pair'
newfname =''
upperwordsu = rpu_rp.uniq(upperwords)
for wordup in upperwordsu:
newfname = newfname +' '+ wordup
## newfnameNext = outputcopiesdir +partfolder+ newfname+'dupexx'+'.'+exten
newfnameNext = outputcopiesdir + newfname+'dupexx'+'.'+exten
print newfnameNext
## print maindir,'maindir'
## fpath = os.path.join(maindir, newfnameNext)
## print newfnameNext
shutil.copyfile(subresult,newfnameNext)
## print 'copy',subresult,'to',newfnameNext
print('___________________')
else:
print('No duplicate files found.')