def chopMonthToDays(bararray,sym,dur,secs): ### what does this do? lines = bararray print secs preve = 0 prevfname = 0 ubars =[] uubars =[] diffprev = 1 daylist =[] for l in lines: if len(l) > 2: timestring = l[1] date = timestring[0:11] daylist.append(date) daylistu = rpu_rp.uniq(daylist) for day in daylistu: daysarray=[] for l in bararray: if day in l[1] : timestring = l[1] try: e = TicksUtile.convertTime(timestring,'dashspace','timetoepoch') except: e = TicksUtile.convertTime(timestring,'dashspace','timetoepoch') diffcur = e - preve preve = e ## print diffcur,timestring if diffcur == int(secs): daysarray.append(l) ## print l print day, sym,dur, len(daysarray) fileoutname = DataDown + day +'.' + sym +'.' + dur+'.cleaned.csv' rpu_rp.WriteArrayToCsvfile(fileoutname, daysarray)
def create_timelist(barfile): fileoutname = 'barsout.csv' lines = rpu_rp.CsvToLines(barfile) preve = 0 prevfname = 0 ubars =[] uubars =[] diffprev = 1 timelist = [] for l in lines: if len(l) > 2: ## print l timestring = l[2] ## e = TicksUtile.time_to_epoch(timestring) timelist.append(timestring) timelistu = rpu_rp.uniq(timelist) return timelistu
def show_by_fnum(fullarray, fnum): c = 0 x = len(fullarray) timelist = [] hourlist = [] for l in fullarray: timelist.append(l[fnum]) c += 1 if c > (x - 8): print l timelistu = rpu_rp.uniq(timelist) c = 0 lengtu = len(timelistu) for t in timelistu: totalsize = 0 c += 1 netdelta = 0 upticks = downticks = neutralticks = upticksize = downticksize = neutralticksize = 0 for l in fullarray: ## print l tsize = l[2] tprice = l[1] if t == l[fnum]: totalsize += tsize if l[5] == 'uptick': upticks += 1 upticksize += tsize elif l[5] == 'downtick': downticks += 1 downticksize += tsize else: neutralticks += 1 neutralticksize += tsize if l[0] == 'HitBid': netdelta = netdelta - l[2] elif l[0] == 'LiftOffer': netdelta = netdelta + l[2] if c > (lengtu - 5): print t, totalsize, netdelta, upticks, downticks, neutralticks, upticksize, downticksize, neutralticksize, tprice
def show_by_fnum(fullarray,fnum): c=0 x = len(fullarray) timelist =[] hourlist =[] for l in fullarray: timelist.append(l[fnum]) c+=1 if c > (x - 8): print l timelistu = rpu_rp.uniq(timelist) c=0 lengtu = len(timelistu) for t in timelistu: totalsize = 0 c+=1 netdelta =0 upticks = downticks = neutralticks = upticksize = downticksize = neutralticksize = 0 for l in fullarray: ## print l tsize = l[2] tprice = l[1] if t == l[fnum]: totalsize += tsize if l[5] == 'uptick': upticks +=1 upticksize += tsize elif l[5] == 'downtick': downticks +=1 downticksize += tsize else: neutralticks +=1 neutralticksize += tsize if l[0] == 'HitBid': netdelta =netdelta - l[2] elif l[0] == 'LiftOffer': netdelta =netdelta + l[2] if c > (lengtu - 5): print t, totalsize, netdelta, upticks,downticks,neutralticks,upticksize,downticksize,neutralticksize,tprice
def show_by_fnum(fullarray, fnum): c = 0 x = len(fullarray) timelist = [] hourlist = [] for l in fullarray: timelist.append(l[fnum]) c += 1 if c > (x - 8): print l timelistu = rpu_rp.uniq(timelist) c = 0 lengtu = len(timelistu) for t in timelistu: ## print t totalsize = 0 HitLift = 0 c += 1 netdelta = 0 upticks = downticks = HitLift = 0 sumtagval = totsize = 0 for l in fullarray: bid = l[1] ask = l[2] tsize = l[5] tprice = l[7] sumtag = l[fnum] if t == sumtag: totsize += tsize sumtagval += 1 if tprice == bid: quoteflag = 'HitBid' HitLift = HitLift - 1 else: quoteflag = 'LIFT' ## print bid HitLift = HitLift + 1 ## print HitLift print '%-8s %8s %8s %8s price,vsumtagval, totsize,vHitLift' % ( t, sumtagval, totsize, HitLift)
def show_by_fnum(fullarray,fnum): c=0 x = len(fullarray) timelist =[] hourlist =[] for l in fullarray: timelist.append(l[fnum]) c+=1 if c > (x - 8): print l timelistu = rpu_rp.uniq(timelist) c=0 lengtu = len(timelistu) for t in timelistu: ## print t totalsize = 0 HitLift = 0 c+=1 netdelta =0 upticks = downticks = HitLift = 0 sumtagval = totsize = 0 for l in fullarray: bid = l[1] ask = l[2] tsize = l[5] tprice = l[7] sumtag = l[fnum] if t == sumtag: totsize += tsize sumtagval +=1 if tprice == bid: quoteflag = 'HitBid' HitLift = HitLift -1 else: quoteflag = 'LIFT' ## print bid HitLift = HitLift +1 ## print HitLift print '%-8s %8s %8s %8s price,vsumtagval, totsize,vHitLift' %(t,sumtagval,totsize,HitLift)
prevpnew = low pnew = low while pnew <= high: linearray =[] pnew = prevpnew + incr ## print pnew,barid prevpnew = float(pnew) linearray.append(pnew) linearray.append(barid) linearray.append(volumeraw) farray.append(linearray) plist =[] varray =[] for p in farray: plist.append(p[0]) pricelist = rpu_rp.uniq(plist) for price in pricelist: vlarray=[] vlarray.append(price) volumer = '' totalhits =0 totalvolume =0 for l in farray: if l[0] == price: mark = l[1] mark = '1' if len(l) > 1: volumer = str(l[2]).replace(' ','') if volumer == 'full': volumer = '100' totalhits+=1
prevpnew = low pnew = low while pnew <= high: linearray = [] pnew = prevpnew + incr ## print pnew,barid prevpnew = float(pnew) linearray.append(pnew) linearray.append(barid) linearray.append(volumeraw) farray.append(linearray) plist = [] varray = [] for p in farray: plist.append(p[0]) pricelist = rpu_rp.uniq(plist) for price in pricelist: vlarray = [] vlarray.append(price) volumer = '' totalhits = 0 totalvolume = 0 for l in farray: if l[0] == price: mark = l[1] mark = '1' if len(l) > 1: volumer = str(l[2]).replace(' ', '') if volumer == 'full': volumer = '100' totalhits += 1
def printResults(dict1): results = list(filter(lambda x: len(x) > 1, dict1.values())) if len(results) > 0: print('Duplicates Found:') print( 'The following files are identical. The name could differ, but the content is identical' ) print('___________________') resultnum = 0 for result in results: ## print result ## resultnum +=1 ## if resultnum == len(results): ## print 'this is the last result of the pair', len(result) words = [] upperwords = [] subresultnum = 0 for subresult in result: subresultnum += 1 lengsubresult = len(result) ## print lengsubresult file1 = subresult filenostem = file1.replace(maindir, '') ## print filenostem fileline = filenostem.replace('\\', ' ').replace('_', ' ').replace( '-', ' ') justfilename = filenostem.split('\\')[ len(filenostem.split('\\')) - 1] partfolder = filenostem.replace(justfilename, '') print partfolder, justfilename file1words = fileline.split() #### print file1words lenwords = len(file1words) ## print lenwords c = 0 for wrd in file1words: c += 1 ## print c,wrd if c == lenwords: lastwrdsplit = wrd.upper().split('.') ## print lastwrdsplit if len(lastwrdsplit) == 2: lastwrd = lastwrdsplit[0] else: lastwrd = wrd.uppper() exten = lastwrdsplit[len(lastwrdsplit) - 1] wrd = lastwrd pass else: wrd = wrd.upper() upperwords.append(wrd.upper()) ## print exten, lastwrd, 'extension' if lengsubresult == subresultnum: print 'lastdupe of pair' newfname = '' upperwordsu = rpu_rp.uniq(upperwords) for wordup in upperwordsu: newfname = newfname + ' ' + wordup ## newfnameNext = outputcopiesdir +partfolder+ newfname+'dupexx'+'.'+exten newfnameNext = outputcopiesdir + newfname + 'dupexx' + '.' + exten print newfnameNext ## print maindir,'maindir' ## fpath = os.path.join(maindir, newfnameNext) ## print newfnameNext shutil.copyfile(subresult, newfnameNext) ## print 'copy',subresult,'to',newfnameNext print('___________________') else: print('No duplicate files found.')
def get_dload_barsWbu(start_path,dur,barfileout,sym): import TicksUtile total_size = 0 alllines =[] alltimes =[] fileoutname = barfileout for dirpath, dirnames, filenames in os.walk(start_path): ## print dirpath subdirsize = 0 for f in filenames: ## if 'ES.'+dur+'.ddload' in f : if '.'+sym+'.'+dur+'.ddload' in f : fp = os.path.join(dirpath, f) total_size += os.path.getsize(fp) ############## print fp,'working' lines = rpu_rp.CsvToLines(fp) for l in lines: newl =[] if len(l) > 2: timestring = l[1] e = TicksUtile.convertTime(timestring,'dashspace','timetoepoch') ## fsourcedate = f.replace('.ES.'+dur+'.ddload.csv','') fsourcedate = f.split('.')[0] fsourcesym = f.split('.')[1] fsourcedur = f.split('.')[2] l.append(fsourcedate) l.append(fsourcesym) l.append(fsourcedur) newl.append(l[8]) newl.append(l[9]) newl.append(l[1]) newl.append(l[7]) newl.append(l[2]) newl.append(l[3]) newl.append(l[4]) newl.append(l[5]) newl.append(l[6]) alllines.append(newl) from operator import itemgetter, attrgetter, methodcaller sortedall = sorted(alllines,key=itemgetter(0,1,2,3),reverse=True) usort = rpu_rp.uniqnosort(sortedall) prevdur = time = '' cleans =[] for y in usort: ## print y dur = y[2] time = y[3] if dur == prevdur and time == prevtime: ## print y ## print prevy,'prev' pass else: cleans.append(y) prevdur = dur prevtime = time prevy = y ## rpu_rp.WriteArrayToCsvfile(fileoutname+'news', cleans) newalls=[] for n in cleans: ## print n newn =[] sym = n[0] dur = n[1] time = n[2] openp = n[4] hip = n[5] lowp = n[6] closep = n[7] volu = n[8] newn.append(sym) newn.append(time) newn.append(openp) newn.append(hip) newn.append(lowp) newn.append(closep) newn.append(volu) newalls.append(newn) sortednewalls = sorted(newalls,key=itemgetter(0),reverse=False) sortednewallsu = rpu_rp.uniq(sortednewalls) sortednewallsuClean = [] preve = 0 for l in sortednewallsu: timestring = l[1] try: e = TicksUtile.convertTime(timestring,'dashspace','timetoepoch') except: print timestring,' did not work, needs old format' e = TicksUtile.convertTime(timestring,'dashspace','timetoepoch') diffcur = e - preve preve = e if diffcur >= int(secs) or '22:30:00' in timestring: sortednewallsuClean.append(l) else: ## print diffcur,timestring,dur,'did not pass test' pass lenfull = len(sortednewallsuClean) third1 = int(round(lenfull/3)) third2 = 2*third1 third3 = lenfull - third2 + 1 print lenfull,third1,third2,third3,dur,sym part33 = rpu_rp.head_array_to_array(sortednewallsuClean,third1) third2array = rpu_rp.head_array_to_array(sortednewallsuClean,third2) part34 = rpu_rp.tail_array_to_array(third2array,third1) part35 = rpu_rp.tail_array_to_array(sortednewallsuClean,third3) rpu_rp.WriteArrayToCsvfile(fileoutname, part33) third2name = fileoutname.replace('33','34') rpu_rp.WriteArrayToCsvfile(third2name, part34) third3name = fileoutname.replace('33','35') rpu_rp.WriteArrayToCsvfile(third3name, part35)
def checkRTnonbarticksNEW(sym,fnums): fullarray =[] filein = DataDown +date+'.'+sym+ '.RTtickData.csv' prevtotdiff = ask = bid =0 prevtime ='' string ='' prevtradeuid='' time = '' size = '' tprice = 0.0 tsize = singletrade = totdaysize= vwap ='' totdowns = totups = 0 tottickups=0 nettickhits = 0 lastsize = lastprice = bidsize = asksize = 0 prevtick = nettickups = totHits = totLifts = 0 timeminarray =[] for line in rpu_rp.tail_array_to_array(rpu_rp.CsvToLines(filein),5000): ## print line if 'tickString' in str(line) and 'tickType=48' in str(line): time = line[3] timemin = time[0:8] ## print timemin # '22:58:36.067000' string = line[2].split(';') if len(string) > 2: tprice = float(string[0].replace('value=','')) tsize = float(string[1]) singletrade = string[5] totdaysize = string[3] vwaps = string[4] if len(vwaps) > 0: vwap = round(float(string[4]),2) else: vwap =0.0 ## value=1984.50;1;1441796865858;207069;1982.49963539;true> ## print tsize,tprice,time,'trade', bid, ask, bidsize, asksize if 'tickSize' in str(line): ## 0 BID_SIZE tickSize() ## 1 BID_PRICE tickPrice() ## 2 ASK_PRICE tickPrice() ## 3 ASK_SIZE tickSize() ## 4 LAST_PRICE tickPrice() ## 5 LAST_SIZE tickSize() fs = line[1].replace(' field=','') sizetime = line[3] value = int((line[2].replace(' size=','')).replace('>','')) size = value ######## print sizetime,size,fs,'size' if fs == '0': bidsize = value elif fs == '3': asksize = value elif fs =='5': lastsize = value else: pass if 'tickPrice' in str(line): pricetime = line[4] value = float(line[2].replace(' price=','')) fp = line[1].replace(' field=','') if fp == '1': bid= value pass elif fp == '2': ask = value else: lastprice = value ## if bid == 0.0: tradeuid = time + str(tsize) + str(tprice) if 'tickString' in str(line) and len(string) > 2 and bid > 0 and ask > 0 and tradeuid != prevtradeuid: if tprice > prevtick: tickdir='uptick' elif tprice < prevtick: tickdir = 'downtick' else: tickdir = 'neutral' if bid == tprice: tickflag = 'HitBid' else: tickflag = 'LiftOffer' aout =[] prevtick = tprice if tsize > 0 and time != 'xxx': aout.append(tickflag) aout.append(tickdir) aout.append(tprice) aout.append(tsize) aout.append(bid) aout.append(ask) aout.append(time) aout.append(timemin) timeminarray.append([timemin]) ## print aout fullarray.append(aout) ### how many upticks in a row? prevtime = time prevtradeuid = tradeuid ## rpu_rp.WriteArrayToCsvfile(libarea +'blatmp', timeminarray) ## print fullarray ## timesecs = rpu_rp.CsvToLines(libarea +'blatmp') utimesecs = rpu_rp.uniq(timeminarray) fullscore =0 for t in utimesecs: counter =0 countertick = 0 for l in fullarray: if t[0] in str(l) and l[3] > 4: price = l[2] ## print l if l[0] == 'HitBid': counter -=1 pass else: counter +=1 if l[1] == 'uptick': countertick +=1 pass elif l[1] == 'downtick': countertick -=1 pass else: pass fullscore +=counter+countertick if counter != 7777: print ('%4s %4s %4s %s %s' % (fullscore,counter,countertick, t[0], price)) ## print timesecs c=0 x = len(fullarray) ## print x limiter = 90 f = x - limiter ## print f netlifts = netticks = 0 lifthitscore = 0 ## print 'netlifts',' netUPticks liftscore' for l in fullarray: ## print l ## ['LiftOffer', 'uptick', 1995.5, 2.0, 1995.25, 1995.5, '15:49:02.881000'] ## ['HitBid', 'neutral', 1995.5, 50.0, 1995.5, 1995.75, '15:49:02.894000'] c+=1 if c > f: ## print l tprice = l[2] lifttag=l[0] tsize = l[3] ticktag=l[1] if lifttag == 'HitBid': netlifts = netlifts -1 lifthitscore += (tsize * (-1)) pass else: netlifts +=1 lifthitscore += (tsize * (1)) if ticktag == 'downtick': netticks = netticks -1 pass elif ticktag == 'uptick': netticks +=1 else: pass if c > (x - 8): pass ## print l print ('%4d %4d | %4d %4.1f %s netlifts netUPticks liftscore' % (netticks, lifthitscore, x, limiter,tprice))
def printResults(dict1): results = list(filter(lambda x: len(x) > 1, dict1.values())) if len(results) > 0: print('Duplicates Found:') print('The following files are identical. The name could differ, but the content is identical') print('___________________') resultnum =0 for result in results: ## print result ## resultnum +=1 ## if resultnum == len(results): ## print 'this is the last result of the pair', len(result) words = [] upperwords=[] subresultnum = 0 for subresult in result: subresultnum+=1 lengsubresult = len(result) ## print lengsubresult file1 = subresult filenostem = file1.replace(maindir,'') ## print filenostem fileline = filenostem.replace('\\',' ').replace('_',' ').replace('-',' ') justfilename = filenostem.split('\\')[len(filenostem.split('\\'))-1] partfolder = filenostem.replace(justfilename,'') print partfolder, justfilename file1words = fileline.split() #### print file1words lenwords = len(file1words) ## print lenwords c=0 for wrd in file1words: c+=1 ## print c,wrd if c==lenwords: lastwrdsplit = wrd.upper().split('.') ## print lastwrdsplit if len(lastwrdsplit) == 2: lastwrd = lastwrdsplit[0] else: lastwrd = wrd.uppper() exten = lastwrdsplit[len(lastwrdsplit)-1] wrd = lastwrd pass else: wrd = wrd.upper() upperwords.append(wrd.upper()) ## print exten, lastwrd, 'extension' if lengsubresult == subresultnum: print 'lastdupe of pair' newfname ='' upperwordsu = rpu_rp.uniq(upperwords) for wordup in upperwordsu: newfname = newfname +' '+ wordup ## newfnameNext = outputcopiesdir +partfolder+ newfname+'dupexx'+'.'+exten newfnameNext = outputcopiesdir + newfname+'dupexx'+'.'+exten print newfnameNext ## print maindir,'maindir' ## fpath = os.path.join(maindir, newfnameNext) ## print newfnameNext shutil.copyfile(subresult,newfnameNext) ## print 'copy',subresult,'to',newfnameNext print('___________________') else: print('No duplicate files found.')