def extractAttributeFromDataMatrix(args): if args.n is None: print "N has not been specified" os._exit(-1) try: args.c colNumberWeAreWorkingWith = eval('colNumberOfData.'+args.c) except: print "C has not been specified" os._exit(-1) numberOfSecondsInFuture = int(args.n) currentRowCount = 0 totalNumberOfDataRows = len(dataFile.matrix) futureRowCount = 1 # Since we have a queue we do not need to initialize everytime inside the for loop totalOfRowsInFutureNSeconds = 0.0 queue = deque() for dataRow in dataFile.matrix: timeOfCurrentRow = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp) # lets remove all values from queue which have time <= timeOfCurrentRow if(len(queue) > 0): # When currentRowCount is 0 then there is nothing in the queue oldestElementinQueue = queue.popleft() while(oldestElementinQueue[0] < timeOfCurrentRow): oldestElementinQueue = queue.popleft() totalOfRowsInFutureNSeconds -= oldestElementinQueue[1] attribute.aList[currentRowCount][0] = timeOfCurrentRow while True: if(currentRowCount + futureRowCount < totalNumberOfDataRows): # If the totalNumberOfDataRows is 5 we can access index values 0 to 4. timeOfFutureRow = common.getTimeStamp(dataFile.matrix[currentRowCount + futureRowCount]) if(timeOfFutureRow - timeOfCurrentRow < numberOfSecondsInFuture): cellValue = float(dataFile.matrix[currentRowCount + futureRowCount][colNumberWeAreWorkingWith]) totalOfRowsInFutureNSeconds += cellValue futureRowCount = futureRowCount + 1 # add the data to queue for future needs queue.append([timeOfFutureRow,cellValue]) else: break else: break averageOfFutureRows = totalOfRowsInFutureNSeconds / futureRowCount currentCellValue = float(dataFile.matrix[currentRowCount][colNumberWeAreWorkingWith]) if( averageOfFutureRows > currentCellValue): attribute.aList[currentRowCount][1] = 1 else: attribute.aList[currentRowCount][1] = 0 attribute.aList[currentRowCount][2] = totalOfRowsInFutureNSeconds attribute.aList[currentRowCount][3] = float(dataFile.matrix[currentRowCount][colNumberOfData.LTP]) currentRowCount = currentRowCount + 1 if(currentRowCount % 1 == 0): print "Processed row number " + str(currentRowCount) lNameOfTarget = "tAverageOfCol" + args.c + "InFuture" + str(args.n) + "SecsIsGreaterThanCol" + args.c + "InCurrentRow" return ["TimeStamp",lNameOfTarget,"TotalOfFutureNSecs","CurrentRowLTPValue"]
def extractAttributeFromDataMatrix(args): queueOfCellValueInLastNRows = deque() totalOfLastNRows = 0.0 if args.n == None: N = 5 else: N = int(args.n) try: args.c except: print "Since -c has not been specified I cannot proceed" os._exit() currentRowCount = 0 if(args.cType == "synthetic"): colNumberOfAttribute = colNumberOfData.SysFeature else: colNumberOfAttribute = eval("colNumberOfData."+ args.c ) colNumberOfTimeStamp = colNumberOfData.TimeStamp for dataRow in dataFile.matrix: cellValue = float(dataFile.matrix[currentRowCount][colNumberOfAttribute]) queueOfCellValueInLastNRows.append(cellValue) totalOfLastNRows += cellValue if (currentRowCount < N): attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfTimeStamp) attribute.aList[currentRowCount][1] = totalOfLastNRows/(currentRowCount+1) # in 1st iteration currentRowCount = 0 currentRowCount = currentRowCount + 1 continue # Since we are going back 1 row from current we cannot get data from current row totalOfLastNRows -= queueOfCellValueInLastNRows.popleft() # In the next 2 rows we do not do -1 since this feature if for the current row. attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfTimeStamp) attribute.aList[currentRowCount][1] = totalOfLastNRows / N currentRowCount = currentRowCount + 1 if (currentRowCount%10000==0): print "Processed row number " + str(currentRowCount) lNameOfFeaturePrinted = "fMovingAverageOfCol" + args.c + "InLast" + str(args.n) + "Rows" return ["TimeStamp",lNameOfFeaturePrinted,"Zero1","Zero2"]
def extractAttributeFromDataMatrix(args): queueOfCellValueInFutureNRows = deque() totalOfFutureNRows = 0.0 if args.n is None: print "-n has not been specified" os._exit(-1) numberOfFutureRows = int(args.n) try: args.c except: print "-c has not been specified" os._exit(-1) """ lets get the total of futureNrows""" futureRowCount = 0 while (futureRowCount < numberOfFutureRows): codeString = 'float(dataFile.matrix[futureRowCount][colNumberOfData.' + args.c + '])' cellValue = eval(codeString) queueOfCellValueInFutureNRows.append(cellValue) totalOfFutureNRows += cellValue futureRowCount = futureRowCount + 1 currentRowCount = 0 for dataRow in dataFile.matrix: attribute.aList[currentRowCount][0] = common.getTimeStamp( dataFile.matrix[currentRowCount], colNumberOfData.TimeStamp) futureCellValue = 0 if (currentRowCount + numberOfFutureRows < len(dataFile.matrix)): codeString = 'float(dataFile.matrix[currentRowCount+numberOfFutureRows][colNumberOfData.' + args.c + '])' futureCellValue = eval(codeString) queueOfCellValueInFutureNRows.append(futureCellValue) divisor = numberOfFutureRows else: divisor = len(dataFile.matrix) - currentRowCount totalOfFutureNRows += futureCellValue totalOfFutureNRows -= queueOfCellValueInFutureNRows.popleft() averageOfFutureRows = totalOfFutureNRows / float(divisor) codeString = 'float(dataFile.matrix[currentRowCount][colNumberOfData.' + args.c + '])' valueInCurrentRow = eval(codeString) if (averageOfFutureRows > valueInCurrentRow): attribute.aList[currentRowCount][1] = 1 else: attribute.aList[currentRowCount][1] = 0 attribute.aList[currentRowCount][2] = totalOfFutureNRows attribute.aList[currentRowCount][3] = valueInCurrentRow currentRowCount = currentRowCount + 1 if (currentRowCount % 1000 == 0): print "Processed row number " + str(currentRowCount) lNameOfTarget = "tAverageOfCol" + args.c + "InFuture" + str( args.n) + "RowsIsGreaterThanCol" + args.c + "InCurrentRow" return ["TimeStamp", lNameOfTarget, "TotalOfFutureRows", "CurrentRowValue"]
def extractAttributeFromDataMatrix(args): """ In the next 100 order book entries is there a case where the bidP0 is > current askP0 """ currentRowCount = 0 while currentRowCount < len(dataFile.matrix) - 101: futureRowCount = 1 attribute.aList[currentRowCount][0] = common.getTimeStamp( dataFile.matrix[currentRowCount], colNumberOfData.TimeStamp) while (futureRowCount < 100): if (float( dataFile.matrix[currentRowCount + futureRowCount] [colNumberOfData.BidP0]) < float( dataFile.matrix[currentRowCount][colNumberOfData.AskP0])): attribute.aList[currentRowCount][1] = 1 futureRowCount = futureRowCount + 1 currentRowCount = currentRowCount + 1 if (currentRowCount % 1000 == 0): print "Processed row number " + str(currentRowCount) lNameOfTarget = "tBidLesserThanAskNext" + str(args.n) return ["TimeStamp", lNameOfTarget, "Zero1", "Zero2"]
def extractAttributeFromDataMatrix(args): try: args.n except: print "Since -n has not been specified I cannot proceed" os._exit(-1) try: args.c except: print "Since -c has not been specified I cannot proceed" os._exit(-1) list_of_price_array = [ eval('colNumberOfData.'+args.c+'P0') , eval('colNumberOfData.'+args.c+'P1') ,\ eval('colNumberOfData.'+args.c+'P2') , eval('colNumberOfData.'+args.c+'P3') , eval('colNumberOfData.'+args.c+'P4') ] list_of_qty_array = [ eval('colNumberOfData.'+args.c+'Q0') , eval('colNumberOfData.'+args.c+'Q1') ,\ eval('colNumberOfData.'+args.c+'Q2') , eval('colNumberOfData.'+args.c+'Q3') , eval('colNumberOfData.'+args.c+'Q4') ] currentRowCount = 0 qtyForCalculatingWeightedAverage = float(args.n) levelOfDataAvailable = 4 for dataRow in dataFile.matrix: qSum = 0 totalPrice = 0 totalPriceAtThisLevel = 0 i = 0 while (i <= levelOfDataAvailable and qSum < qtyForCalculatingWeightedAverage): priceAtThisLevel = float( dataFile.matrix[currentRowCount][list_of_price_array[i]]) qtyAtThisLevel = float( dataFile.matrix[currentRowCount][list_of_qty_array[i]]) qSum += qtyAtThisLevel if (qSum > qtyForCalculatingWeightedAverage): qtyToUseAtThisLevel = qtyAtThisLevel - ( qSum - qtyForCalculatingWeightedAverage) totalPriceAtThisLevel = qtyToUseAtThisLevel * priceAtThisLevel else: totalPriceAtThisLevel = qtyAtThisLevel * priceAtThisLevel totalPrice += totalPriceAtThisLevel i = i + 1 if ( qSum < qtyForCalculatingWeightedAverage ): # This implies that the current row does not have enough qty to fill our requirement. qtyToUseAtThisLevel = qtyForCalculatingWeightedAverage - qSum totalPriceAtThisLevel = qtyToUseAtThisLevel * priceAtThisLevel totalPrice += totalPriceAtThisLevel attribute.aList[currentRowCount][0] = common.getTimeStamp( dataFile.matrix[currentRowCount], colNumberOfData.TimeStamp) attribute.aList[currentRowCount][1] = float( totalPrice) / qtyForCalculatingWeightedAverage currentRowCount += 1 if (currentRowCount % 10000 == 0): print "Processed row number " + str(currentRowCount) lNameOfFeaturePrinted = "fWAPriceOfCol" + args.c + "InLast" + str( args.n) + "Qty" return ["TimeStamp", lNameOfFeaturePrinted, "Zero1", "Zero2"]
def extractAttributeFromDataMatrix(args): try: args.c except: print "Since -c has not been specified I cannot proceed" os._exit() if (args.cType == "synthetic"): colNumberOfAttribute = colNumberOfData.SysFeature else: colNumberOfAttribute = eval("colNumberOfData." + args.c) colNumberOfTimeStamp = colNumberOfData.TimeStamp currentRowCount = 0 for dataRow in dataFile.matrix: attribute.aList[currentRowCount][0] = common.getTimeStamp( dataFile.matrix[currentRowCount], colNumberOfTimeStamp) attribute.aList[currentRowCount][1] = float( dataFile.matrix[currentRowCount][colNumberOfAttribute]) currentRowCount = currentRowCount + 1 if (currentRowCount % 1000 == 0): print "Processed row number " + str(currentRowCount) lNameOfFeaturePrinted = "fCol" + args.c + "InCurrentRow" return ["TimeStamp", lNameOfFeaturePrinted, "Zero1", "Zero2"]
def extractAttributeFromDataMatrix(args): if args.n == None: N = 5 else: N = int(args.n) try: args.c except: print "Since -c has not been specified I cannot proceed" os._exit() currentRowCount = 0 codeString = 'float(dataFile.matrix[currentRowCount][colNumberOfData.' + args.c + '])' eParam = math.pow((.5), (1.0 / N)) for dataRow in dataFile.matrix: cellValue = eval(codeString) if currentRowCount == 0: attribute.aList[currentRowCount][0] = common.getTimeStamp( dataFile.matrix[currentRowCount], colNumberOfData.TimeStamp) attribute.aList[currentRowCount][1] = cellValue currentRowCount = currentRowCount + 1 continue attribute.aList[currentRowCount][0] = common.getTimeStamp( dataFile.matrix[currentRowCount], colNumberOfData.TimeStamp) firstPart = float(attribute.aList[currentRowCount - 1][1]) * eParam secondPart = (1 - eParam) * cellValue attribute.aList[currentRowCount][1] = firstPart + secondPart currentRowCount = currentRowCount + 1 if (currentRowCount % 10000 == 0): print "Processed row number " + str(currentRowCount) lNameOfFeaturePrinted = "fEMovingAverageOfCol" + args.c + "InLast" + str( args.n) + "Secs" return ["TimeStamp", lNameOfFeaturePrinted, "Zero1", "Zero2"]
def extractAttributeFromDataMatrix(args): if args.n == None: N = 5 else: N = int(args.n) try: args.c except: print "Since -c has not been specified I cannot proceed" os._exit() currentRowCount = 0 codeString = 'float(dataFile.matrix[currentRowCount][colNumberOfData.'+ args.c + '])' eParam = math.pow((.5),(1.0 / N)) for dataRow in dataFile.matrix: cellValue = eval(codeString) if currentRowCount == 0: attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp) attribute.aList[currentRowCount][1] = cellValue currentRowCount = currentRowCount + 1 continue attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp) firstPart = float(attribute.aList[currentRowCount - 1][1]) * eParam secondPart = (1- eParam) * cellValue attribute.aList[currentRowCount][1] = firstPart + secondPart currentRowCount = currentRowCount + 1 if (currentRowCount%10000==0): print "Processed row number " + str(currentRowCount) lNameOfFeaturePrinted = "fEMovingAverageOfCol" + args.c + "InLast" + str(args.n) + "Secs" return ["TimeStamp",lNameOfFeaturePrinted,"Zero1","Zero2"]
def extractAttributeFromDataMatrix(args): if args.n is None: print "-n has not been specified" os._exit(-1) totalOfLastLTQQty = int(args.n) colNumberOfLTP = colNumberOfData.LTP colNumberOfTTQ = colNumberOfData.TTQ prev_ttq = 0 ltq_ltp_list = [] sum_ltq = 0 prod_sum = 0 currentRowCount = 0 for dataRow in dataFile.matrix: ttq, ltp = (int(dataRow[colNumberOfTTQ]), float(dataRow[colNumberOfLTP])) attribute.aList[currentRowCount][0] = common.getTimeStamp( dataFile.matrix[currentRowCount], colNumberOfData.TimeStamp) if ttq > prev_ttq: ltq = ttq - prev_ttq ltq_ltp_list.append([ltq, ltp]) prod_sum += ltq * ltp sum_ltq += ltq if sum_ltq > totalOfLastLTQQty: index = 0 while sum_ltq <> totalOfLastLTQQty: if ltq_ltp_list[index][0] > sum_ltq - totalOfLastLTQQty: ltq_ltp_list[index][0] -= sum_ltq - totalOfLastLTQQty prod_sum -= (sum_ltq - totalOfLastLTQQty ) * ltq_ltp_list[index][1] sum_ltq = totalOfLastLTQQty else: sum_ltq -= ltq_ltp_list[index][0] prod_sum -= (ltq_ltp_list[index][0] * ltq_ltp_list[index][1]) ltq_ltp_list = ltq_ltp_list[1:] wtd_ltp = prod_sum / (max(sum_ltq, 1)) prev_ttq = ttq attribute.aList[currentRowCount][1] = max(wtd_ltp, 1) currentRowCount = currentRowCount + 1 if (currentRowCount % 1000 == 0): print "Processed row number " + str(currentRowCount) lNameOfFeature = "fWALTPInLast" + str(args.n) + "Qty" return ["TimeStamp", lNameOfFeature, "0", "1"]
def extractAttributeFromDataMatrix(args): currentRowCount = 0 lAPL = [ colNumberOfData.AskP0, colNumberOfData.AskP1, colNumberOfData.AskP2, colNumberOfData.AskP3, colNumberOfData.AskP4 ] lAQL = [ colNumberOfData.AskQ0, colNumberOfData.AskQ1, colNumberOfData.AskQ2, colNumberOfData.AskQ3, colNumberOfData.AskQ4 ] lBPL = [ colNumberOfData.BidP0, colNumberOfData.BidP1, colNumberOfData.BidP2, colNumberOfData.BidP3, colNumberOfData.BidP4 ] lBQL = [ colNumberOfData.BidQ0, colNumberOfData.BidQ1, colNumberOfData.BidQ2, colNumberOfData.BidQ3, colNumberOfData.BidQ4 ] if args.n == None: print "Since -n has not been specified I cannot proceed" os._exit() else: N = int(args.n) for dataRow in dataFile.matrix: askp = [] askq = [] bidp = [] bidq = [] lPriceMulQtySum = 0 lQtySum = 0 for i in range(0, N): askp.append(float(dataRow[lAPL[i]])) askq.append(float(dataRow[lAQL[i]])) bidp.append(float(dataRow[lBPL[i]])) bidq.append(float(dataRow[lBQL[i]])) for price, qty in zip(askp[:N] + bidp[:N], askq[:N] + bidq[:N]): lPriceMulQtySum += float(price) * float(qty) lQtySum += float(qty) qAverage = lPriceMulQtySum / lQtySum attribute.aList[currentRowCount][0] = common.getTimeStamp( dataFile.matrix[currentRowCount], colNumberOfData.TimeStamp) attribute.aList[currentRowCount][1] = qAverage currentRowCount = currentRowCount + 1 if (currentRowCount % 10000 == 0): print "Processed row number " + str(currentRowCount) lNameOfFeaturePrinted = "fInverseWAInLast" + str(N) + "Levels" return ["TimeStamp", lNameOfFeaturePrinted, "Zero1", "Zero2"]
def extractAttributeFromDataMatrix(args): queueOfCellValueInLastNRows = deque() totalOfLastNRows = 0.0 if args.n == None: N = 5 else: N = int(args.n) try: args.c except: print "Since -c has not been specified I cannot proceed" os._exit() currentRowCount = 0 if(args.cType == "synthetic"): colNumberOfAttribute = colNumberOfData.SysFeature else: colNumberOfAttribute = eval("colNumberOfData."+ args.c ) colNumberOfTimeStamp = colNumberOfData.TimeStamp for dataRow in dataFile.matrix: currentTickValue = float(dataFile.matrix[currentRowCount][colNumberOfAttribute]) attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfTimeStamp,args.cType) if currentTickValue >= UpperCutoff: attribute.aList[currentRowCount][1] = UpperValue elif currentTickValue <= LowerCutOff: attribute.aList[currentRowCount][1] = LowerValue else: if currentRowCount == 0: attribute.aList[currentRowCount][1] = MidValue elif attribute.aList[currentRowCount-1][1] == LowerValue : if currentTickValue <= MidCutOff: attribute.aList[currentRowCount][1] = LowerValue else: attribute.aList[currentRowCount][1] = MidValue elif attribute.aList[currentRowCount-1][1] == UpperValue: if currentTickValue >= MidCutOff: attribute.aList[currentRowCount][1] = UpperValue else: attribute.aList[currentRowCount][1] = MidValue else: attribute.aList[currentRowCount][1] = MidValue currentRowCount = currentRowCount + 1 if (currentRowCount%10000==0): print "Processed row number " + str(currentRowCount) lNameOfFeaturePrinted = "fMovingAverageOfCol" + args.c + "InLast" + str(args.n) + "Rows" return ["TimeStamp",lNameOfFeaturePrinted,"Zero1","Zero2"]
def extractAttributeFromDataMatrix(args): try: args.n except: print "Since -n has not been specified I cannot proceed" os._exit(-1) try: args.c except: print "Since -c has not been specified I cannot proceed" os._exit(-1) list_of_price_array = [ eval('colNumberOfData.'+args.c+'P0') , eval('colNumberOfData.'+args.c+'P1') ,\ eval('colNumberOfData.'+args.c+'P2') , eval('colNumberOfData.'+args.c+'P3') , eval('colNumberOfData.'+args.c+'P4') ] list_of_qty_array = [ eval('colNumberOfData.'+args.c+'Q0') , eval('colNumberOfData.'+args.c+'Q1') ,\ eval('colNumberOfData.'+args.c+'Q2') , eval('colNumberOfData.'+args.c+'Q3') , eval('colNumberOfData.'+args.c+'Q4') ] currentRowCount = 0 qtyForCalculatingWeightedAverage = float(args.n) levelOfDataAvailable = 4 for dataRow in dataFile.matrix: qSum = 0 totalPrice = 0 totalPriceAtThisLevel = 0 i = 0 while(i <= levelOfDataAvailable and qSum < qtyForCalculatingWeightedAverage): priceAtThisLevel = float(dataFile.matrix[currentRowCount][list_of_price_array[i]]) qtyAtThisLevel = float(dataFile.matrix[currentRowCount][list_of_qty_array[i]]) qSum += qtyAtThisLevel if(qSum > qtyForCalculatingWeightedAverage): qtyToUseAtThisLevel = qtyAtThisLevel - (qSum - qtyForCalculatingWeightedAverage) totalPriceAtThisLevel = qtyToUseAtThisLevel * priceAtThisLevel else: totalPriceAtThisLevel = qtyAtThisLevel * priceAtThisLevel totalPrice += totalPriceAtThisLevel i = i + 1 if(qSum < qtyForCalculatingWeightedAverage): # This implies that the current row does not have enough qty to fill our requirement. qtyToUseAtThisLevel = qtyForCalculatingWeightedAverage - qSum totalPriceAtThisLevel = qtyToUseAtThisLevel * priceAtThisLevel totalPrice += totalPriceAtThisLevel attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp) attribute.aList[currentRowCount][1] = float(totalPrice)/qtyForCalculatingWeightedAverage currentRowCount += 1 if (currentRowCount%10000==0): print "Processed row number " + str(currentRowCount) lNameOfFeaturePrinted = "fWAPriceOfCol" + args.c + "InLast" + str(args.n) + "Qty" return ["TimeStamp",lNameOfFeaturePrinted,"Zero1","Zero2"]
def extractAttributeFromDataMatrix(args): currentRowCount = 0 list_of_array = [ eval('colNumberOfData.'+args.c+'0') , eval('colNumberOfData.'+args.c+'4') ] for dataRow in dataFile.matrix: qSum = float(dataRow[list_of_array[0]]) + float(dataRow[list_of_array[1]]) qAverage = qSum / 2 attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp) attribute.aList[currentRowCount][1] = qAverage currentRowCount = currentRowCount + 1 if (currentRowCount%10000==0): print "Processed row number " + str(currentRowCount) lNameOfFeaturePrinted = "fCol" + args.c + "InCurrentRowAvg" return ["TimeStamp",lNameOfFeaturePrinted,"Zero1","Zero2"]
def extractAttributeFromDataMatrix(args): currentRowCount = 0 list_of_array = [ eval('colNumberOfData.'+args.c+'1') , eval('colNumberOfData.'+args.c+'2') ,eval('colNumberOfData.'+args.c+'3') , eval('colNumberOfData.'+args.c+'4') , eval('colNumberOfData.'+args.c+'5') ] for dataRow in dataFile.matrix: qSum = float(dataRow[list_of_array[0]]) + float(dataRow[list_of_array[1]]) + \ float(dataRow[list_of_array[2]]) + float(dataRow[list_of_array[3]]) + float(dataRow[list_of_array[4]]) attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp) attribute.aList[currentRowCount][1] = qSum currentRowCount = currentRowCount + 1 if (currentRowCount%10000==0): print "Processed row number " + str(currentRowCount) lNameOfFeaturePrinted = "fCol" + args.c + "InCurrentRowSum" return ["TimeStamp",lNameOfFeaturePrinted,"Zero1","Zero2"]
def extractAttributeFromDataMatrix(args): if args.n is None: print "-n has not been specified" os._exit(-1) totalOfLastLTQQty = int(args.n) colNumberOfLTP = colNumberOfData.LTP colNumberOfTTQ = colNumberOfData.TTQ prev_ttq = 0 ltq_ltp_list = [] sum_ltq = 0 prod_sum = 0 currentRowCount = 0 for dataRow in dataFile.matrix: ttq, ltp = (int(dataRow[colNumberOfTTQ]), float(dataRow[colNumberOfLTP])) attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp) if ttq > prev_ttq: ltq = ttq - prev_ttq ltq_ltp_list.append([ltq, ltp]) prod_sum += ltq * ltp sum_ltq += ltq if sum_ltq > totalOfLastLTQQty: index = 0 while sum_ltq <> totalOfLastLTQQty: if ltq_ltp_list[index][0] > sum_ltq - totalOfLastLTQQty: ltq_ltp_list[index][0] -= sum_ltq - totalOfLastLTQQty prod_sum -= (sum_ltq - totalOfLastLTQQty) * ltq_ltp_list[index][1] sum_ltq = totalOfLastLTQQty else: sum_ltq -= ltq_ltp_list[index][0] prod_sum -= (ltq_ltp_list[index][0] * ltq_ltp_list[index][1]) ltq_ltp_list = ltq_ltp_list[1:] wtd_ltp = prod_sum / (max(sum_ltq,1)) prev_ttq = ttq attribute.aList[currentRowCount][1] = max(wtd_ltp, 1) currentRowCount = currentRowCount + 1 if(currentRowCount % 1000 == 0): print "Processed row number " + str(currentRowCount) lNameOfFeature = "fWALTPInLast" + str(args.n) + "Qty" return ["TimeStamp",lNameOfFeature,"0", "1"]
def extractAttributeFromDataMatrix(args): """ In the next 100 order book entries is there a case where the bidP0 is > current askP0 """ currentRowCount = 0 while currentRowCount < len(dataFile.matrix)-101: futureRowCount = 1 attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp) while(futureRowCount < 100): if( float(dataFile.matrix[currentRowCount+futureRowCount][colNumberOfData.BidP0]) >= float(dataFile.matrix[currentRowCount][colNumberOfData.AskP0]) ): attribute.aList[currentRowCount][1] = 1 futureRowCount = futureRowCount + 1 currentRowCount = currentRowCount + 1 if(currentRowCount % 1000 == 0): print "Processed row number " + str(currentRowCount) lNameOfTarget = "tBidEqualsOrGreaterThanAskInNext" + str(args.n) return ["TimeStamp",lNameOfTarget,"Zero1","Zero2"]
def extractAttributeFromDataMatrix(args): currentRowCount = 0 list_of_price_array = [ eval('colNumberOfData.'+args.c+'P0') , eval('colNumberOfData.'+args.c+'P1') ,\ eval('colNumberOfData.'+args.c+'P2') , eval('colNumberOfData.'+args.c+'P3') , eval('colNumberOfData.'+args.c+'P4') ] list_of_qty_array = [ eval('colNumberOfData.'+args.c+'Q0') , eval('colNumberOfData.'+args.c+'Q1') ,\ eval('colNumberOfData.'+args.c+'Q2') , eval('colNumberOfData.'+args.c+'Q3') , eval('colNumberOfData.'+args.c+'Q4') ] for dataRow in dataFile.matrix: qPriceSum = 0.0 qQtySum = 0 for i in range(5): qPriceSum += int(dataRow[list_of_qty_array[i]]) * float(dataRow[list_of_price_array[i]]) qQtySum += int(dataRow[list_of_qty_array[i]]) qAverage = qPriceSum / qQtySum attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp) attribute.aList[currentRowCount][1] = qAverage currentRowCount = currentRowCount + 1 if (currentRowCount%10000==0): print "Processed row number " + str(currentRowCount) lNameOfFeaturePrinted = "fCol" + args.c + "InCurrentRowAvg" return ["TimeStamp",lNameOfFeaturePrinted,"Zero1","Zero2"]
def extractAttributeFromDataMatrix(args): currentRowCount = 0 lAPL = [ colNumberOfData.AskP0 , colNumberOfData.AskP1 , colNumberOfData.AskP2 , colNumberOfData.AskP3 , colNumberOfData.AskP4 ] lAQL = [ colNumberOfData.AskQ0 , colNumberOfData.AskQ1 , colNumberOfData.AskQ2 , colNumberOfData.AskQ3 , colNumberOfData.AskQ4 ] lBPL = [ colNumberOfData.BidP0 , colNumberOfData.BidP1 , colNumberOfData.BidP2 , colNumberOfData.BidP3 , colNumberOfData.BidP4 ] lBQL = [ colNumberOfData.BidQ0 , colNumberOfData.BidQ1 , colNumberOfData.BidQ2 , colNumberOfData.BidQ3 , colNumberOfData.BidQ4 ] if args.n == None: print "Since -n has not been specified I cannot proceed" os._exit() else: N = int(args.n) for dataRow in dataFile.matrix: askp = [] askq = [] bidp = [] bidq = [] lPriceMulQtySum = 0 lQtySum = 0 for i in range(0,N): askp.append(float(dataRow[lAPL[i]])) askq.append(float(dataRow[lAQL[i]])) bidp.append(float(dataRow[lBPL[i]])) bidq.append(float(dataRow[lBQL[i]])) for price, qty in zip(askp[:N] + bidp[:N], askq[:N] + bidq[:N]): lPriceMulQtySum += float(price) * float(qty) lQtySum += float(qty) qAverage = lPriceMulQtySum / lQtySum attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp) attribute.aList[currentRowCount][1] = qAverage currentRowCount = currentRowCount + 1 if (currentRowCount%10000==0): print "Processed row number " + str(currentRowCount) lNameOfFeaturePrinted = "fInverseWAInLast" + str(N) + "Levels" return ["TimeStamp",lNameOfFeaturePrinted,"Zero1","Zero2"]
def extractAttributeFromDataMatrix(args): try: args.c except: print "Since -c has not been specified I cannot proceed" os._exit() if(args.cType == "synthetic"): colNumberOfAttribute = colNumberOfData.SysFeature else: colNumberOfAttribute = eval("colNumberOfData."+ args.c ) colNumberOfTimeStamp = colNumberOfData.TimeStamp currentRowCount = 0 for dataRow in dataFile.matrix: attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfTimeStamp) attribute.aList[currentRowCount][1] = float(dataFile.matrix[currentRowCount][colNumberOfAttribute]) currentRowCount = currentRowCount + 1 if(currentRowCount % 1000 == 0): print "Processed row number " + str(currentRowCount) lNameOfFeaturePrinted = "fCol" + args.c + "InCurrentRow" return ["TimeStamp",lNameOfFeaturePrinted,"Zero1","Zero2"]
def extractAttributeFromDataMatrix(args): queueOfCellValueInFutureNRows = deque() totalOfFutureNRows = 0.0 if args.n is None: print "-n has not been specified" os._exit(-1) numberOfFutureRows = int(args.n) try: args.c except: print "-c has not been specified" os._exit(-1) """ lets get the total of futureNrows""" futureRowCount = 0 while(futureRowCount < numberOfFutureRows): codeString = 'float(dataFile.matrix[futureRowCount][colNumberOfData.'+args.c+'])' cellValue = eval(codeString) queueOfCellValueInFutureNRows.append(cellValue) totalOfFutureNRows += cellValue futureRowCount = futureRowCount + 1 currentRowCount = 0 for dataRow in dataFile.matrix: attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp) futureCellValue = 0 if(currentRowCount + numberOfFutureRows < len(dataFile.matrix)): codeString = 'float(dataFile.matrix[currentRowCount+numberOfFutureRows][colNumberOfData.'+args.c+'])' futureCellValue = eval(codeString) queueOfCellValueInFutureNRows.append(futureCellValue) divisor = numberOfFutureRows else: divisor = len(dataFile.matrix) - currentRowCount totalOfFutureNRows += futureCellValue totalOfFutureNRows -= queueOfCellValueInFutureNRows.popleft() averageOfFutureRows = totalOfFutureNRows / float(divisor) codeString = 'float(dataFile.matrix[currentRowCount][colNumberOfData.'+args.c+'])' valueInCurrentRow = eval(codeString) if( averageOfFutureRows > valueInCurrentRow): attribute.aList[currentRowCount][1] = 1 else: attribute.aList[currentRowCount][1] = 0 attribute.aList[currentRowCount][2] = totalOfFutureNRows attribute.aList[currentRowCount][3] = valueInCurrentRow currentRowCount = currentRowCount + 1 if(currentRowCount % 1000 == 0): print "Processed row number " + str(currentRowCount) lNameOfTarget = "tAverageOfCol" + args.c + "InFuture" + str(args.n) + "RowsIsGreaterThanCol" + args.c + "InCurrentRow" return ["TimeStamp",lNameOfTarget,"TotalOfFutureRows","CurrentRowValue"]
def extractAttributeFromDataMatrix(args): if args.n is None: print "N has not been specified" os._exit(-1) try: args.c colNumberWeAreWorkingWith = eval('colNumberOfData.' + args.c) except: print "C has not been specified" os._exit(-1) numberOfSecondsInFuture = int(args.n) currentRowCount = 0 totalNumberOfDataRows = len(dataFile.matrix) futureRowCount = 1 # Since we have a queue we do not need to initialize everytime inside the for loop totalOfRowsInFutureNSeconds = 0.0 queue = deque() for dataRow in dataFile.matrix: timeOfCurrentRow = common.getTimeStamp( dataFile.matrix[currentRowCount], colNumberOfData.TimeStamp) # lets remove all values from queue which have time <= timeOfCurrentRow if (len(queue) > 0 ): # When currentRowCount is 0 then there is nothing in the queue oldestElementinQueue = queue.popleft() while (oldestElementinQueue[0] < timeOfCurrentRow): oldestElementinQueue = queue.popleft() totalOfRowsInFutureNSeconds -= oldestElementinQueue[1] attribute.aList[currentRowCount][0] = timeOfCurrentRow while True: if ( currentRowCount + futureRowCount < totalNumberOfDataRows ): # If the totalNumberOfDataRows is 5 we can access index values 0 to 4. timeOfFutureRow = common.getTimeStamp( dataFile.matrix[currentRowCount + futureRowCount]) if (timeOfFutureRow - timeOfCurrentRow < numberOfSecondsInFuture): cellValue = float(dataFile.matrix[currentRowCount + futureRowCount] [colNumberWeAreWorkingWith]) totalOfRowsInFutureNSeconds += cellValue futureRowCount = futureRowCount + 1 # add the data to queue for future needs queue.append([timeOfFutureRow, cellValue]) else: break else: break averageOfFutureRows = totalOfRowsInFutureNSeconds / futureRowCount currentCellValue = float( dataFile.matrix[currentRowCount][colNumberWeAreWorkingWith]) if (averageOfFutureRows > currentCellValue): attribute.aList[currentRowCount][1] = 1 else: attribute.aList[currentRowCount][1] = 0 attribute.aList[currentRowCount][2] = totalOfRowsInFutureNSeconds attribute.aList[currentRowCount][3] = float( dataFile.matrix[currentRowCount][colNumberOfData.LTP]) currentRowCount = currentRowCount + 1 if (currentRowCount % 1 == 0): print "Processed row number " + str(currentRowCount) lNameOfTarget = "tAverageOfCol" + args.c + "InFuture" + str( args.n) + "SecsIsGreaterThanCol" + args.c + "InCurrentRow" return [ "TimeStamp", lNameOfTarget, "TotalOfFutureNSecs", "CurrentRowLTPValue" ]
def extractAttributeFromDataMatrix(args): queueOfCellValueInFutureNLTQs = deque() totalOfFutureNRows = 0.0 if args.n is None: print "-n has not been specified" os._exit(-1) totalOfFutureLTQQty = int(args.n) try: args.c except: print "-c has not been specified" os._exit(-1) try: args.m except: print "-m has not been specified" os._exit(-1) colNumberOfAttribute = eval("colNumberOfData."+ args.c ) print "attribute column number " , colNumberOfAttribute , args.c print args.m lDiffPip = float(args.m) lPipSize = int(args.tickSize) """ lets get the total of futureNrows""" futureLTQSum = 0 currentRowIndex = 0 lLastTradeIndex = 0 lNoMoreTradesFound = 0 currentLTPValue = 0.0 currentLTQValue = 0 WeightedLTPSum = 0.0 totalLTPQty = totalOfFutureLTQQty while(futureLTQSum < totalOfFutureLTQQty): if dataFile.matrix[currentRowIndex][colNumberOfData.MsgCode].upper() == "T": currentLTPValue = float(dataFile.matrix[currentRowIndex][colNumberOfData.LTP]) currentLTQValue = int(dataFile.matrix[currentRowIndex][colNumberOfData.NewQ]) queueOfCellValueInFutureNLTQs.append( [ (currentLTPValue * currentLTQValue) , currentLTQValue , currentLTPValue , 0 ] ) futureLTQSum = futureLTQSum + currentLTQValue WeightedLTPSum = WeightedLTPSum + queueOfCellValueInFutureNLTQs[-1][0] lLastTradeIndex = currentRowIndex currentRowIndex = currentRowIndex + 1 if futureLTQSum > totalOfFutureLTQQty : currentLTQValueToBeUsed = currentLTQValue - ( futureLTQSum - totalOfFutureLTQQty ) currentTickPriceToBeAdded = currentLTPValue * currentLTQValueToBeUsed LTQValueNotAdded = ( futureLTQSum - totalOfFutureLTQQty ) WeightedLTPSum = WeightedLTPSum - queueOfCellValueInFutureNLTQs[-1][0] + currentTickPriceToBeAdded queueOfCellValueInFutureNLTQs[-1] = [ currentTickPriceToBeAdded , currentLTQValueToBeUsed , currentLTPValue , LTQValueNotAdded ] currentRowCount = 0 for dataRow in dataFile.matrix: sum = 0 currentPrice = float(dataFile.matrix[currentRowCount][colNumberOfAttribute]) currentLTP = float(dataFile.matrix[currentRowCount][colNumberOfData.LTP]) currentMsgCode = dataFile.matrix[currentRowCount][colNumberOfData.MsgCode] currentLTQ = int(dataFile.matrix[currentRowCount][colNumberOfData.NewQ]) # try: attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp) # except: # print colNumberOfData.TimeStamp,currentRowCount,currentMsgCode # import pdb # pdb.set_trace() if (dataFile.matrix[currentRowCount][colNumberOfData.MsgCode].upper() == "T") : lZerothElementOfQueue = queueOfCellValueInFutureNLTQs.popleft() WeightedLTPSum = WeightedLTPSum - lZerothElementOfQueue[0] LTQSubtracted = lZerothElementOfQueue[1] if len(queueOfCellValueInFutureNLTQs) != 0: LTQWhichCanBeAddedWithoutNewTradeMessage = queueOfCellValueInFutureNLTQs[-1][3] LTQAddedWithoutNewTradeMessage = min( LTQSubtracted , LTQWhichCanBeAddedWithoutNewTradeMessage) currentTickPriceToBeAdded = ( LTQAddedWithoutNewTradeMessage * queueOfCellValueInFutureNLTQs[-1][2] ) + queueOfCellValueInFutureNLTQs[-1][0] LTQValueNotAddedInLastElementOfQueue = queueOfCellValueInFutureNLTQs[-1][3] - LTQAddedWithoutNewTradeMessage WeightedLTPSum = WeightedLTPSum - queueOfCellValueInFutureNLTQs[-1][0] + currentTickPriceToBeAdded queueOfCellValueInFutureNLTQs[-1][0] = currentTickPriceToBeAdded queueOfCellValueInFutureNLTQs[-1][1] = queueOfCellValueInFutureNLTQs[-1][1] + LTQAddedWithoutNewTradeMessage queueOfCellValueInFutureNLTQs[-1][3] = LTQValueNotAddedInLastElementOfQueue else: LTQAddedWithoutNewTradeMessage = 0 if LTQSubtracted > LTQAddedWithoutNewTradeMessage: LTQToBeAddedFromNewTrade = LTQSubtracted - LTQAddedWithoutNewTradeMessage if (lNoMoreTradesFound == 0): startIndexToFindNextTrade = lLastTradeIndex + 1 while( startIndexToFindNextTrade < len(dataFile.matrix)): if dataFile.matrix[startIndexToFindNextTrade][colNumberOfData.MsgCode].upper() == "T": lLastTradeIndex = startIndexToFindNextTrade futureLTPValue = float(dataFile.matrix[startIndexToFindNextTrade][colNumberOfData.LTP]) futureLTQValue = int(dataFile.matrix[startIndexToFindNextTrade][colNumberOfData.NewQ]) queueOfCellValueInFutureNLTQs.append( [ (futureLTPValue * futureLTQValue) , futureLTQValue , futureLTPValue , 0 ] ) LTQToBeAddedFromNewTrade = LTQToBeAddedFromNewTrade - futureLTQValue WeightedLTPSum = WeightedLTPSum + queueOfCellValueInFutureNLTQs[-1][0] if LTQToBeAddedFromNewTrade <= 0 : if LTQToBeAddedFromNewTrade < 0 : LTQValueToBeUsed = futureLTQValue + LTQToBeAddedFromNewTrade currentTickPriceToBeAdded = LTQValueToBeUsed * futureLTPValue LTQValueNotAdded = -1 * LTQToBeAddedFromNewTrade WeightedLTPSum = WeightedLTPSum - queueOfCellValueInFutureNLTQs[-1][0] + currentTickPriceToBeAdded queueOfCellValueInFutureNLTQs[-1] = [ currentTickPriceToBeAdded , LTQValueToBeUsed , futureLTPValue , LTQValueNotAdded ] break startIndexToFindNextTrade = startIndexToFindNextTrade + 1 if startIndexToFindNextTrade == len(dataFile.matrix): if LTQToBeAddedFromNewTrade > 0 : totalLTPQty = totalLTPQty - LTQToBeAddedFromNewTrade lNoMoreTradesFound = 1 else: totalLTPQty = totalLTPQty - LTQToBeAddedFromNewTrade for ele in queueOfCellValueInFutureNLTQs: sum = sum + (ele[1]*ele[2]) if (ele[1]*ele[2] != ele[0]): print "Error in Code" os.exit(-1) if sum != 0 and sum != WeightedLTPSum : print "Error in Code" os.exit(-1) if totalLTPQty != 0: WALTPOfFutureNQty = WeightedLTPSum / totalLTPQty if lPipSize== 25000: tTranCost = 0.000015 * 2 else: tTranCost = 0.00015 * 2 l_margin_ticks = ceil((lDiffPip*tTranCost*currentPrice)/lPipSize) if args.c == "BidP0" or args.c == "BestBidP": if WALTPOfFutureNQty >= (currentPrice + l_margin_ticks* lPipSize): lClassOfTargetVariable = 1 else: lClassOfTargetVariable = 0 else: if WALTPOfFutureNQty <= (currentPrice - l_margin_ticks*lPipSize): lClassOfTargetVariable = 1 else: lClassOfTargetVariable = 0 attribute.aList[currentRowCount][1] = lClassOfTargetVariable attribute.aList[currentRowCount][2] = currentPrice attribute.aList[currentRowCount][3] = ";".join([ str(WALTPOfFutureNQty) , str(currentLTP) , str(currentLTQ) ,\ str(currentMsgCode) , str(WeightedLTPSum) , str(totalLTPQty) ]) currentRowCount = currentRowCount + 1 if(currentRowCount % 1000 == 0): print "Processed row number " + str(currentRowCount) lNameOfTarget = "tWALTPComparedToCol" + str(args.c) + "InFuture" + str(args.n) + "Qty" return ["TimeStamp",lNameOfTarget,args.c,"WALTPWithWtAsLTQForFuture"+args.n+"LTQQty","CurrentLTP","CurrentLTQ","CurrentMsgCode",\ "WeightedLTPSum" , "totalLTPQtys"]
def extractAttributeFromDataMatrix(args): queueOfCellValueInFutureNLTQs = deque() totalOfFutureNRows = 0.0 if args.n is None: print "-n has not been specified" os._exit(-1) totalOfFutureLTQQty = int(args.n) try: args.c except: print "-c has not been specified" os._exit(-1) colNumberOfAttribute = eval("colNumberOfData." + args.c) lPipSize = int(args.tickSize) """ lets get the total of futureNrows""" futureLTQSum = 0 currentRowIndex = 0 lLastTradeIndex = 0 lNoMoreTradesFound = 0 currentLTPValue = 0.0 currentLTQValue = 0 WeightedLTPSum = 0.0 totalLTPQty = totalOfFutureLTQQty while (futureLTQSum < totalOfFutureLTQQty): try: dataFile.matrix[currentRowIndex][ colNumberOfData.MsgCode].upper() == "T" except: print "Data Line = ", dataFile.matrix[currentRowIndex] print "Current Row Index = ", currentRowIndex if dataFile.matrix[currentRowIndex][ colNumberOfData.MsgCode].upper() == "T": currentLTPValue = float( dataFile.matrix[currentRowIndex][colNumberOfData.LTP]) currentLTQValue = int( dataFile.matrix[currentRowIndex][colNumberOfData.NewQ]) queueOfCellValueInFutureNLTQs.append([ (currentLTPValue * currentLTQValue), currentLTQValue, currentLTPValue, 0 ]) futureLTQSum = futureLTQSum + currentLTQValue WeightedLTPSum = WeightedLTPSum + queueOfCellValueInFutureNLTQs[ -1][0] lLastTradeIndex = currentRowIndex currentRowIndex = currentRowIndex + 1 if futureLTQSum > totalOfFutureLTQQty: currentLTQValueToBeUsed = currentLTQValue - (futureLTQSum - totalOfFutureLTQQty) currentTickPriceToBeAdded = currentLTPValue * currentLTQValueToBeUsed LTQValueNotAdded = (futureLTQSum - totalOfFutureLTQQty) WeightedLTPSum = WeightedLTPSum - queueOfCellValueInFutureNLTQs[-1][ 0] + currentTickPriceToBeAdded queueOfCellValueInFutureNLTQs[-1] = [ currentTickPriceToBeAdded, currentLTQValueToBeUsed, currentLTPValue, LTQValueNotAdded ] currentRowCount = 0 for dataRow in dataFile.matrix: sum = 0 currentPrice = float( dataFile.matrix[currentRowCount][colNumberOfAttribute]) currentLTP = float( dataFile.matrix[currentRowCount][colNumberOfData.LTP]) currentMsgCode = dataFile.matrix[currentRowCount][ colNumberOfData.MsgCode] currentLTQ = int( dataFile.matrix[currentRowCount][colNumberOfData.NewQ]) attribute.aList[currentRowCount][0] = common.getTimeStamp( dataFile.matrix[currentRowCount], colNumberOfData.TimeStamp) if (dataFile.matrix[currentRowCount][colNumberOfData.MsgCode].upper() == "T"): lZerothElementOfQueue = queueOfCellValueInFutureNLTQs.popleft() WeightedLTPSum = WeightedLTPSum - lZerothElementOfQueue[0] LTQSubtracted = lZerothElementOfQueue[1] if len(queueOfCellValueInFutureNLTQs) != 0: LTQWhichCanBeAddedWithoutNewTradeMessage = queueOfCellValueInFutureNLTQs[ -1][3] LTQAddedWithoutNewTradeMessage = min( LTQSubtracted, LTQWhichCanBeAddedWithoutNewTradeMessage) currentTickPriceToBeAdded = ( LTQAddedWithoutNewTradeMessage * queueOfCellValueInFutureNLTQs[-1][2] ) + queueOfCellValueInFutureNLTQs[-1][0] LTQValueNotAddedInLastElementOfQueue = queueOfCellValueInFutureNLTQs[ -1][3] - LTQAddedWithoutNewTradeMessage WeightedLTPSum = WeightedLTPSum - queueOfCellValueInFutureNLTQs[ -1][0] + currentTickPriceToBeAdded queueOfCellValueInFutureNLTQs[-1][ 0] = currentTickPriceToBeAdded queueOfCellValueInFutureNLTQs[-1][ 1] = queueOfCellValueInFutureNLTQs[-1][ 1] + LTQAddedWithoutNewTradeMessage queueOfCellValueInFutureNLTQs[-1][ 3] = LTQValueNotAddedInLastElementOfQueue else: LTQAddedWithoutNewTradeMessage = 0 if LTQSubtracted > LTQAddedWithoutNewTradeMessage: LTQToBeAddedFromNewTrade = LTQSubtracted - LTQAddedWithoutNewTradeMessage if (lNoMoreTradesFound == 0): startIndexToFindNextTrade = lLastTradeIndex + 1 while (startIndexToFindNextTrade < len(dataFile.matrix)): if dataFile.matrix[startIndexToFindNextTrade][ colNumberOfData.MsgCode].upper() == "T": lLastTradeIndex = startIndexToFindNextTrade futureLTPValue = float( dataFile.matrix[startIndexToFindNextTrade][ colNumberOfData.LTP]) futureLTQValue = int( dataFile.matrix[startIndexToFindNextTrade][ colNumberOfData.NewQ]) queueOfCellValueInFutureNLTQs.append([ (futureLTPValue * futureLTQValue), futureLTQValue, futureLTPValue, 0 ]) LTQToBeAddedFromNewTrade = LTQToBeAddedFromNewTrade - futureLTQValue WeightedLTPSum = WeightedLTPSum + queueOfCellValueInFutureNLTQs[ -1][0] if LTQToBeAddedFromNewTrade <= 0: if LTQToBeAddedFromNewTrade < 0: LTQValueToBeUsed = futureLTQValue + LTQToBeAddedFromNewTrade currentTickPriceToBeAdded = LTQValueToBeUsed * futureLTPValue LTQValueNotAdded = -1 * LTQToBeAddedFromNewTrade WeightedLTPSum = WeightedLTPSum - queueOfCellValueInFutureNLTQs[ -1][0] + currentTickPriceToBeAdded queueOfCellValueInFutureNLTQs[-1] = [ currentTickPriceToBeAdded, LTQValueToBeUsed, futureLTPValue, LTQValueNotAdded ] break startIndexToFindNextTrade = startIndexToFindNextTrade + 1 if startIndexToFindNextTrade == len(dataFile.matrix): if LTQToBeAddedFromNewTrade > 0: totalLTPQty = totalLTPQty - LTQToBeAddedFromNewTrade lNoMoreTradesFound = 1 else: totalLTPQty = totalLTPQty - LTQToBeAddedFromNewTrade for ele in queueOfCellValueInFutureNLTQs: sum = sum + (ele[1] * ele[2]) if (ele[1] * ele[2] != ele[0]): print "Error in Code" os.exit(-1) if sum != 0 and sum != WeightedLTPSum: print "Error in Code" os.exit(-1) if totalLTPQty != 0: WALTPOfFutureNQty = WeightedLTPSum / totalLTPQty if args.c == "BidP0": if WALTPOfFutureNQty >= (currentPrice + lPipSize): lClassOfTargetVariable = 1 else: lClassOfTargetVariable = 0 else: if WALTPOfFutureNQty <= (currentPrice - lPipSize): lClassOfTargetVariable = 1 else: lClassOfTargetVariable = 0 attribute.aList[currentRowCount][1] = lClassOfTargetVariable attribute.aList[currentRowCount][2] = currentPrice attribute.aList[currentRowCount][3] = ";".join([ str(WALTPOfFutureNQty) , str(currentLTP) , str(currentLTQ) ,\ str(currentMsgCode) , str(WeightedLTPSum) , str(totalLTPQty) ]) currentRowCount = currentRowCount + 1 if (currentRowCount % 1000 == 0): print "Processed row number " + str(currentRowCount) lNameOfTarget = "tWALTPComparedToCol" + str(args.c) + "InFuture" + str( args.n) + "Qty" return ["TimeStamp",lNameOfTarget,args.c,"WALTPWithWtAsLTQForFuture"+args.n+"LTQQty","CurrentLTP","CurrentLTQ","CurrentMsgCode",\ "WeightedLTPSum" , "totalLTPQtys"]
def extractAttributeFromDataMatrix(args): queueOfCellValueInFutureNRows = deque() totalOfFutureNRows = 0.0 if args.n is None: print "-n has not been specified" os._exit(-1) numberOfFutureTrades = int(args.n) lPipSize = int(args.tickSize) lMargin1 = ( 1 * lPipSize ) lMargin2 = lMargin1 + ( 2 * lPipSize ) """ lets get the total of futureNrows""" futureTradesCount = 0 currentRowIndex = 0 lLastTradeIndex = 0 lNoMoreTradesFound = 0 while(futureTradesCount < numberOfFutureTrades): if dataFile.matrix[currentRowIndex][colNumberOfData.MsgCode].upper() == "T": cellValue = float(dataFile.matrix[currentRowIndex][colNumberOfData.LTP]) queueOfCellValueInFutureNRows.append(cellValue) futureTradesCount = futureTradesCount + 1 lLastTradeIndex = currentRowIndex currentRowIndex = currentRowIndex + 1 currentRowCount = 0 for dataRow in dataFile.matrix: attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp) if (dataFile.matrix[currentRowCount][colNumberOfData.MsgCode].upper() == "T") : queueOfCellValueInFutureNRows.popleft() if (lNoMoreTradesFound == 0): startIndexToFindNextTrade = lLastTradeIndex + 1 while( startIndexToFindNextTrade < len(dataFile.matrix)): if dataFile.matrix[startIndexToFindNextTrade][colNumberOfData.MsgCode].upper() == "T": futureCellValue = float(dataFile.matrix[startIndexToFindNextTrade][colNumberOfData.LTP]) queueOfCellValueInFutureNRows.append(futureCellValue) lLastTradeIndex = startIndexToFindNextTrade break startIndexToFindNextTrade = startIndexToFindNextTrade + 1 if startIndexToFindNextTrade == len(dataFile.matrix) : lNoMoreTradesFound = 1 currentBidP0 = float(dataFile.matrix[currentRowCount][colNumberOfData.BidP0]) currentAskP0 = float(dataFile.matrix[currentRowCount][colNumberOfData.AskP0]) currentLTP = float(dataFile.matrix[currentRowCount][colNumberOfData.LTP]) currentMsgCode = dataFile.matrix[currentRowCount][colNumberOfData.MsgCode] lClassOfTargetVariable = [] valueInCurrentRow = 0 for LTP in queueOfCellValueInFutureNRows: lClassifiedInOneClass = 0 if (LTP - (currentBidP0 + lPipSize)) > lMargin1 : if (LTP - (currentBidP0 + lPipSize)) > lMargin2: lClassOfTargetVariable.append(2) lClassifiedInOneClass = 1 else: lClassOfTargetVariable.append(1) lClassifiedInOneClass = 1 if ((currentAskP0 - lPipSize) - LTP ) > lMargin1 : if lClassifiedInOneClass == 1 : lClassOfTargetVariable[-1] = 0 elif ((currentAskP0 - lPipSize) - LTP ) > lMargin2 : lClassOfTargetVariable.append(-2) lClassifiedInOneClass = 1 else: lClassOfTargetVariable.append(-1) lClassifiedInOneClass = 1 if lClassifiedInOneClass == 0: lClassOfTargetVariable.append(0) if lClassOfTargetVariable[-1]==2 or lClassOfTargetVariable[-1]==-2 : break if len(queueOfCellValueInFutureNRows) == 0 : attribute.aList[currentRowCount][1] = 0 attribute.aList[currentRowCount][2] = currentBidP0 attribute.aList[currentRowCount][3] = ";".join([ str(currentAskP0) , str(currentLTP) , str(currentMsgCode) , "0" ,"0", "0" , str(LTP)]) else: lNumberTimesMarketIndicatedBuy = 0 lNumberTimesMarketIndicatedSell = 0 lNumberTimesMarketIndicatedSame = 0 if (lClassOfTargetVariable[-1] == 2) or (lClassOfTargetVariable[-1] == -2): attribute.aList[currentRowCount][1] = lClassOfTargetVariable[-1] else: lNumberTimesMarketIndicatedBuy = lClassOfTargetVariable.count(1) lNumberTimesMarketIndicatedSell = lClassOfTargetVariable.count(-1) lNumberTimesMarketIndicatedSame = lClassOfTargetVariable.count(0) if ( lNumberTimesMarketIndicatedBuy > lNumberTimesMarketIndicatedSell ) and (lNumberTimesMarketIndicatedBuy > lNumberTimesMarketIndicatedSame): attribute.aList[currentRowCount][1] = 1 elif ( lNumberTimesMarketIndicatedSell > lNumberTimesMarketIndicatedBuy ) and (lNumberTimesMarketIndicatedSell > lNumberTimesMarketIndicatedSame): attribute.aList[currentRowCount][1] = -1 else: attribute.aList[currentRowCount][1] = 0 attribute.aList[currentRowCount][2] = currentBidP0 attribute.aList[currentRowCount][3] = ";".join([ str(currentAskP0) ,str(currentLTP) , str(currentMsgCode) , str(lNumberTimesMarketIndicatedBuy) , str(lNumberTimesMarketIndicatedSell) , str(lNumberTimesMarketIndicatedSame) , str(queueOfCellValueInFutureNRows[-1]) ]) currentRowCount = currentRowCount + 1 if(currentRowCount % 1000 == 0): print "Processed row number " + str(currentRowCount) lNameOfTarget = "tClassOfTargetVariableInFuture"+ str(args.n) + "TradeTicks" return ["TimeStamp",lNameOfTarget,"CurrentBid","CurrentAsk","CurrentLTP","CurrentMsgCode",\ "NumberTimesMarketIndicatedBuy" , "NumberTimesMarketIndicatedSell" , "NumberTimesMarketIndicatedSame",\ "ValueOfTargetVariableFromCurrentToNextNthTickTaken"]
def extractAttributeFromDataMatrix(args): try: args.n except: print "Since -n has not been specified I cannot proceed" os._exit(-1) try: args.c ''' if args.n not in args.c: print "Something wrong in design file " print "moving average of last N secs we will taking as qty , so N should be there in sysnthetic fetaure also" ''' except: print "Since -c has not been specified I cannot proceed" os._exit(-1) wt = float(args.n) if(args.cType == "synthetic"): colNumberOfAttribute = colNumberOfData.SysFeature else: colNumberOfAttribute = eval("colNumberOfData."+ args.c ) colNumberOfTimeStamp = colNumberOfData.TimeStamp colNumberOfExchangeStamp = colNumberOfData.ExchangeTS if "Bid" in args.c: side = "Bid" elif "Ask" in args.c: side = "Ask" list_of_price_array = [ eval('colNumberOfData.'+side+'P0') , eval('colNumberOfData.'+side+'P1') ,\ eval('colNumberOfData.'+side+'P2') , eval('colNumberOfData.'+side+'P3') , eval('colNumberOfData.'+side+'P4') ] list_of_qty_array = [ eval('colNumberOfData.'+side+'Q0') , eval('colNumberOfData.'+side+'Q1') ,\ eval('colNumberOfData.'+side+'Q2') , eval('colNumberOfData.'+side+'Q3') , eval('colNumberOfData.'+side+'Q4') ] currentRowCount = 0 levelOfDataAvailable = 4 print "data File length"+ str(len(dataFile.matrix)) for dataRow in dataFile.matrix: qSum = 0 totalPrice = 0 totalPriceAtThisLevel = 0 i = 0 try: qtyForCalculatingWeightedAverage = wt * float(dataRow[colNumberOfAttribute]) except: print "Error" print dataRow print dataRow[colNumberOfAttribute] os._exit(1) while(i <= levelOfDataAvailable and qSum < qtyForCalculatingWeightedAverage): priceAtThisLevel = float(dataFile.matrix[currentRowCount][list_of_price_array[i]]) qtyAtThisLevel = float(dataFile.matrix[currentRowCount][list_of_qty_array[i]]) qSum += qtyAtThisLevel if(qSum > qtyForCalculatingWeightedAverage): qtyToUseAtThisLevel = qtyAtThisLevel - (qSum - qtyForCalculatingWeightedAverage) totalPriceAtThisLevel = qtyToUseAtThisLevel * priceAtThisLevel else: totalPriceAtThisLevel = qtyAtThisLevel * priceAtThisLevel totalPrice += totalPriceAtThisLevel i = i + 1 if(qSum < qtyForCalculatingWeightedAverage): # This implies that the current row does not have enough qty to fill our requirement. qtyToUseAtThisLevel = qtyForCalculatingWeightedAverage - qSum totalPriceAtThisLevel = qtyToUseAtThisLevel * priceAtThisLevel totalPrice += totalPriceAtThisLevel attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfTimeStamp) # print currentRowCount,qtyForCalculatingWeightedAverage attribute.aList[currentRowCount][1] = float(totalPrice)/qtyForCalculatingWeightedAverage attribute.aList[currentRowCount][2] = qtyForCalculatingWeightedAverage attribute.aList[currentRowCount][3] = totalPrice currentRowCount += 1 if (currentRowCount%10000==0): print "Processed row number " + str(currentRowCount) lNameOfFeaturePrinted = "fWAPriceOfCol" + side + "InLast" + str(args.n) + "Qty" return ["TimeStamp",lNameOfFeaturePrinted,"QtyUsedForCalWtAvg","TotalPriceCalculatedAtCurrRow"]
def extractAttributeFromDataMatrix(args): queueOfCellValueInFutureNLTQs = deque() totalOfFutureNRows = 0.0 if args.n is None: print "-n has not been specified" os._exit(-1) totalOfFutureLTQQty = int(args.n) lPipSize = int(args.tickSize) lMargin1 = (1 * lPipSize) lMargin2 = lMargin1 + (2 * lPipSize) """ lets get the total of futureNrows""" futureLTQSum = 0 currentRowIndex = 0 lLastTradeIndex = 0 lNoMoreTradesFound = 0 currentLTPValue = 0.0 currentLTQValue = 0 WeightedLTPSum = 0.0 totalLTPQty = totalOfFutureLTQQty while (futureLTQSum < totalOfFutureLTQQty): if dataFile.matrix[currentRowIndex][ colNumberOfData.MsgCode].upper() == "T": currentLTPValue = float( dataFile.matrix[currentRowIndex][colNumberOfData.LTP]) currentLTQValue = int( dataFile.matrix[currentRowIndex][colNumberOfData.NewQ]) queueOfCellValueInFutureNLTQs.append([ (currentLTPValue * currentLTQValue), currentLTQValue, currentLTPValue, 0 ]) futureLTQSum = futureLTQSum + currentLTQValue WeightedLTPSum = WeightedLTPSum + queueOfCellValueInFutureNLTQs[ -1][0] lLastTradeIndex = currentRowIndex currentRowIndex = currentRowIndex + 1 if futureLTQSum > totalOfFutureLTQQty: currentLTQValueToBeUsed = currentLTQValue - (futureLTQSum - totalOfFutureLTQQty) currentTickPriceToBeAdded = currentLTPValue * currentLTQValueToBeUsed LTQValueNotAdded = (futureLTQSum - totalOfFutureLTQQty) WeightedLTPSum = WeightedLTPSum - queueOfCellValueInFutureNLTQs[-1][ 0] + currentTickPriceToBeAdded queueOfCellValueInFutureNLTQs[-1] = [ currentTickPriceToBeAdded, currentLTQValueToBeUsed, currentLTPValue, LTQValueNotAdded ] currentRowCount = 0 for dataRow in dataFile.matrix: attribute.aList[currentRowCount][0] = common.getTimeStamp( dataFile.matrix[currentRowCount], colNumberOfData.TimeStamp) if (dataFile.matrix[currentRowCount][colNumberOfData.MsgCode].upper() == "T"): currentWtLTQToBeSubtracted = queueOfCellValueInFutureNLTQs.popleft( ) WeightedLTPSum = WeightedLTPSum - currentWtLTQToBeSubtracted[0] LTQSubtracted = currentWtLTQToBeSubtracted[1] if len(queueOfCellValueInFutureNLTQs) != 0: LTQWhichCanBeAddedWithoutNewTradeMessage = queueOfCellValueInFutureNLTQs[ -1][3] LTQAddedWithoutNewTradeMessage = min( LTQSubtracted, LTQWhichCanBeAddedWithoutNewTradeMessage) currentTickPriceToBeAdded = ( LTQAddedWithoutNewTradeMessage * queueOfCellValueInFutureNLTQs[-1][2] ) + queueOfCellValueInFutureNLTQs[-1][0] LTQValueNotAddedInLastElementOfQueue = queueOfCellValueInFutureNLTQs[ -1][3] - LTQAddedWithoutNewTradeMessage WeightedLTPSum = WeightedLTPSum - queueOfCellValueInFutureNLTQs[ -1][0] + currentTickPriceToBeAdded queueOfCellValueInFutureNLTQs[-1][ 0] = currentTickPriceToBeAdded queueOfCellValueInFutureNLTQs[-1][ 1] = queueOfCellValueInFutureNLTQs[-1][ 1] + LTQAddedWithoutNewTradeMessage queueOfCellValueInFutureNLTQs[-1][ 3] = LTQValueNotAddedInLastElementOfQueue else: LTQAddedWithoutNewTradeMessage = 0 print "Last element in queue = ", currentWtLTQToBeSubtracted if LTQSubtracted > LTQAddedWithoutNewTradeMessage: LTQToBeAddedFromNewTrade = LTQSubtracted - LTQAddedWithoutNewTradeMessage if (lNoMoreTradesFound == 0): startIndexToFindNextTrade = lLastTradeIndex + 1 while (startIndexToFindNextTrade < len(dataFile.matrix)): if dataFile.matrix[startIndexToFindNextTrade][ colNumberOfData.MsgCode].upper() == "T": lLastTradeIndex = startIndexToFindNextTrade futureLTPValue = float( dataFile.matrix[startIndexToFindNextTrade][ colNumberOfData.LTP]) futureLTQValue = int( dataFile.matrix[startIndexToFindNextTrade][ colNumberOfData.NewQ]) queueOfCellValueInFutureNLTQs.append([ (futureLTPValue * futureLTQValue), futureLTQValue, futureLTPValue, 0 ]) LTQToBeAddedFromNewTrade = LTQToBeAddedFromNewTrade - futureLTQValue WeightedLTPSum = WeightedLTPSum + queueOfCellValueInFutureNLTQs[ -1][0] if LTQToBeAddedFromNewTrade <= 0: if LTQToBeAddedFromNewTrade < 0: LTQValueToBeUsed = futureLTQValue + LTQToBeAddedFromNewTrade currentTickPriceToBeAdded = LTQValueToBeUsed * futureLTPValue LTQValueNotAdded = -1 * LTQToBeAddedFromNewTrade WeightedLTPSum = WeightedLTPSum - queueOfCellValueInFutureNLTQs[ -1][0] + currentTickPriceToBeAdded queueOfCellValueInFutureNLTQs[-1] = [ currentTickPriceToBeAdded, LTQValueToBeUsed, futureLTPValue, LTQValueNotAdded ] break startIndexToFindNextTrade = startIndexToFindNextTrade + 1 if startIndexToFindNextTrade == len(dataFile.matrix): if LTQToBeAddedFromNewTrade > 0: totalLTPQty = totalLTPQty - LTQToBeAddedFromNewTrade lNoMoreTradesFound = 1 else: totalLTPQty = totalLTPQty - LTQToBeAddedFromNewTrade currentBidP0 = float( dataFile.matrix[currentRowCount][colNumberOfData.BidP0]) currentAskP0 = float( dataFile.matrix[currentRowCount][colNumberOfData.AskP0]) currentLTP = float( dataFile.matrix[currentRowCount][colNumberOfData.LTP]) currentMsgCode = dataFile.matrix[currentRowCount][ colNumberOfData.MsgCode] currentLTQ = int( dataFile.matrix[currentRowCount][colNumberOfData.NewQ]) lClassOfTargetVariable = 0 valueInCurrentRow = 0 lClassifiedInOneClass = 0 if totalLTPQty != 0: WALTPOfFutureNQty = WeightedLTPSum / totalLTPQty if (WALTPOfFutureNQty - (currentBidP0 + lPipSize)) > lMargin1: if (WALTPOfFutureNQty - (currentBidP0 + lPipSize)) > lMargin2: lClassOfTargetVariable = 2 lClassifiedInOneClass = 1 else: lClassOfTargetVariable = 1 lClassifiedInOneClass = 1 if ((currentAskP0 - lPipSize) - WALTPOfFutureNQty) > lMargin1: if lClassifiedInOneClass == 1: lClassOfTargetVariable = 0 elif ((currentAskP0 - lPipSize) - WALTPOfFutureNQty) > lMargin2: lClassOfTargetVariable = -2 lClassifiedInOneClass = 1 else: lClassOfTargetVariable = -1 lClassifiedInOneClass = 1 attribute.aList[currentRowCount][1] = lClassOfTargetVariable attribute.aList[currentRowCount][2] = currentBidP0 attribute.aList[currentRowCount][3] = ";".join([ str(currentAskP0) , str(WALTPOfFutureNQty) , str(currentLTP) , str(currentLTQ) ,\ str(currentMsgCode) , str(WeightedLTPSum) , str(totalLTPQty) ]) currentRowCount = currentRowCount + 1 if (currentRowCount % 1000 == 0): print "Processed row number " + str(currentRowCount) lNameOfTarget = "tClassOfTargetVariableInFuture" + str( args.n) + "TradeTicks" return ["TimeStamp",lNameOfTarget,"CurrentBid","CurrentAsk","ValueOfTargetVariableFromCurrentToNextNthTickTaken","CurrentLTP","CurrentLTQ","CurrentMsgCode"\ "WeightedLTPSum" , "totalLTPQtys"]
def extractAttributeFromDataMatrix(args): colNumberOfTimeStamp = colNumberOfData.TimeStamp currentRowCount = 0 lPipSize = int(args.tickSize) colOfCurrentBid = colNumberOfData.BestBidP colOfCurrentAsk = colNumberOfData.BestAskP currentAsk = 0 previousAsk = 0 currentBid = 0 previousBid = 0 bidSideCases = 0 askSideCases = 0 bidSideFlag = 0 askSideFlag = 0 lAskSideAskP = 0 lAskSideBidP = 0 lBidSideAskP = 0 lBidSideBidP = 0 for dataRow in dataFile.matrix: attribute.aList[currentRowCount][0] = common.getTimeStamp( dataFile.matrix[currentRowCount], colNumberOfTimeStamp) currentBid = float(dataFile.matrix[currentRowCount][colOfCurrentBid]) currentAsk = float(dataFile.matrix[currentRowCount][colOfCurrentAsk]) if (lAskSideAskP != currentAsk and askSideFlag == 1) or (lAskSideBidP != currentBid and askSideFlag == 1): askSideFlag = 0 if (lBidSideAskP != currentAsk and askSideFlag == 1) or (lBidSideBidP != currentBid and bidSideFlag == 1): bidSideFlag = 0 if (currentAsk - currentBid) == lPipSize: if (previousAsk - currentAsk) > lPipSize or askSideFlag == 1: askSideCases += 1 askSideFlag = 1 lAskSideAskP = currentAsk lAskSideBidP = currentBid if (currentBid - previousBid) > lPipSize or bidSideFlag == 1: bidSideCases += 1 bidSideFlag = 1 lBidSideAskP = currentAsk lBidSideBidP = currentBid attribute.aList[currentRowCount][1] = askSideFlag attribute.aList[currentRowCount][2] = bidSideFlag attribute.aList[currentRowCount][3] = ";".join( map(str, [currentBid, currentAsk, previousBid, previousAsk])) currentRowCount = currentRowCount + 1 # if(currentRowCount % 1000 == 0): # print "Processed row number " + str(currentRowCount) if previousBid == 0 or previousBid != currentBid: previousBid = currentBid if previousAsk == 0 or previousAsk != currentAsk: previousAsk = currentAsk print "Number of bid side cases = ", bidSideCases, "Percentage of Bid side cases = ", ( float(bidSideCases) / currentRowCount) * 100 print "Number of ask side cases = ", askSideCases, "Percentage of Ask side cases = ", ( float(askSideCases) / currentRowCount) * 100 lNameOfFeaturePrinted = "BidSideCases" return [ "TimeStamp", lNameOfFeaturePrinted, "AskSideCases", "CurrentBid", "CurrentAsk", "PreviousBid", "PreviousAsk" ]
def extractAttributeFromDataMatrix(args): currentRowCount = 0 try: args.c except: print "Since -c has not been specified I cannot proceed" os._exit() colNumberOfPrice0 = eval("colNumberOfData." + args.c + "P0") colNumberOfQty0 = eval("colNumberOfData." + args.c + "Q0") colNumberOfPrice1 = colNumberOfPrice0 + 2 colNumberOfQty1 = colNumberOfQty0 + 2 colNumberOfPrice2 = colNumberOfPrice0 + 4 colNumberOfQty2 = colNumberOfQty0 + 4 colNumberOfPrice3 = colNumberOfPrice0 + 6 colNumberOfQty3 = colNumberOfQty0 + 6 colNumberOfPrice4 = colNumberOfPrice0 + 8 colNumberOfQty4 = colNumberOfQty0 + 8 colNumberOfTimeStamp = colNumberOfData.TimeStamp #++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Reading prices +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ for dataRow in dataFile.matrix: l_obj = parameters_for_calculating_historical_weighted_bid_or_ask( dataRow[colNumberOfTimeStamp]) if args.c == "Ask": l_obj.AskQ0 = int(dataRow[colNumberOfQty0]) l_obj.AskP0 = float(dataRow[colNumberOfPrice0]) l_obj.AskQ1 = int(dataRow[colNumberOfQty1]) l_obj.AskP1 = float(dataRow[colNumberOfPrice1]) l_obj.AskQ2 = int(dataRow[colNumberOfQty2]) l_obj.AskP2 = float(dataRow[colNumberOfPrice2]) l_obj.AskQ3 = int(dataRow[colNumberOfQty3]) l_obj.AskP3 = float(dataRow[colNumberOfPrice3]) l_obj.AskQ4 = int(dataRow[colNumberOfQty4]) l_obj.AskP4 = float(dataRow[colNumberOfPrice4]) l_obj.AskP_list = [ l_obj.AskP0, l_obj.AskP1, l_obj.AskP2, l_obj.AskP3, l_obj.AskP4 ] l_obj.AskQ_list = [ l_obj.AskQ0, l_obj.AskQ1, l_obj.AskQ2, l_obj.AskQ3, l_obj.AskQ4 ] l_obj.sum_qty_for_avg_cal_Ask = ((g_weight_list[0] * l_obj.AskQ0) + (g_weight_list[1] * l_obj.AskQ1) + (g_weight_list[2] * l_obj.AskQ2) + (g_weight_list[3] * l_obj.AskQ3) + (g_weight_list[4] * l_obj.AskQ4)) else: l_obj.BidQ0 = int(dataRow[colNumberOfQty0]) l_obj.BidP0 = float(dataRow[colNumberOfPrice0]) l_obj.BidQ1 = int(dataRow[colNumberOfQty1]) l_obj.BidP1 = float(dataRow[colNumberOfPrice1]) l_obj.BidQ2 = int(dataRow[colNumberOfQty2]) l_obj.BidP2 = float(dataRow[colNumberOfPrice2]) l_obj.BidQ3 = int(dataRow[colNumberOfQty3]) l_obj.BidP3 = float(dataRow[colNumberOfPrice3]) l_obj.BidQ4 = int(dataRow[colNumberOfQty4]) l_obj.BidP4 = float(dataRow[colNumberOfPrice4]) l_obj.BidP_list = [ l_obj.BidP0, l_obj.BidP1, l_obj.BidP2, l_obj.BidP3, l_obj.BidP4 ] l_obj.BidQ_list = [ l_obj.BidQ0, l_obj.BidQ1, l_obj.BidQ2, l_obj.BidQ3, l_obj.BidQ4 ] l_obj.sum_qty_for_avg_cal_Bid = ((g_weight_list[0] * l_obj.BidQ0) + (g_weight_list[1] * l_obj.BidQ1) + (g_weight_list[2] * l_obj.BidQ2) + (g_weight_list[3] * l_obj.BidQ3) + (g_weight_list[4] * l_obj.BidQ4)) g_filtered_object_list.append(l_obj) #++++++++++++++++++++++++++++++++++++++++++++++++++++++ Calculating ratio ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ if args.c == "Ask": calculate_historical_weighted_Ask() else: calculate_historical_weighted_Bid() #+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Storing results ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ for dataRow in dataFile.matrix: # In the next 2 rows we do not do -1 since this feature if for the current row. attribute.aList[currentRowCount][0] = common.getTimeStamp( dataFile.matrix[currentRowCount], colNumberOfTimeStamp, args.cType) if args.c == "Ask": attribute.aList[currentRowCount][1] = g_filtered_object_list[ currentRowCount].sum_qty_for_avg_cal_Ask / float( g_filtered_object_list[currentRowCount].window_Ask) else: attribute.aList[currentRowCount][1] = g_filtered_object_list[ currentRowCount].sum_qty_for_avg_cal_Bid / float( g_filtered_object_list[currentRowCount].window_Bid) currentRowCount = currentRowCount + 1 if (currentRowCount % 10000 == 0): print "Processed row number " + str(currentRowCount) lNameOfFeaturePrinted = "fOBAvgHistPriceOfCol" + args.c + "InRows" return ["TimeStamp", lNameOfFeaturePrinted, "Zero1", "Zero2"]
def extractAttributeFromDataMatrix(args): try: args.n except: print "Since -n has not been specified I cannot proceed" os._exit(-1) try: args.c except: print "Since -c has not been specified I cannot proceed" os._exit(-1) try: args.tickSize except: print "args.tickSize is required" os._exit(-1) N = int(args.n) tickSize = int(args.tickSize) list_of_price_array = map(eval, ['colNumberOfData.'+args.c+'P'+str(level) for level in xrange(N+1)]) list_of_qty_array = map(eval, ['colNumberOfData.'+args.c+'Q'+str(level) for level in xrange(N+1)]) currentRowCount = 0 qtyForCalculatingWeightedAverage = sum(list_of_qty_array) levelOfDataAvailable = N for dataRow in dataFile.matrix: qSum = 0 totalPrice = 0 totalPriceAtThisLevel = 0 i = 0 while(i <= levelOfDataAvailable and qSum < qtyForCalculatingWeightedAverage): priceAtThisLevel = float(dataFile.matrix[currentRowCount][list_of_price_array[i]]) qtyAtThisLevel = float(dataFile.matrix[currentRowCount][list_of_qty_array[i]]) if i==1: if (priceAtThisLevel-float(dataFile.matrix[currentRowCount][list_of_price_array[0]]) ) != abs(float(tickSize)): if "AskP" in args.c: priceAtThisLevel = float(dataFile.matrix[currentRowCount][list_of_price_array[0]]) + float(tickSize) else: priceAtThisLevel = float(dataFile.matrix[currentRowCount][list_of_price_array[0]]) - float(tickSize) qtyAtThisLevel = 0 qSum += qtyAtThisLevel if(qSum > qtyForCalculatingWeightedAverage): qtyToUseAtThisLevel = qtyAtThisLevel - (qSum - qtyForCalculatingWeightedAverage) totalPriceAtThisLevel = qtyToUseAtThisLevel * priceAtThisLevel else: totalPriceAtThisLevel = qtyAtThisLevel * priceAtThisLevel totalPrice += totalPriceAtThisLevel i = i + 1 if(qSum < qtyForCalculatingWeightedAverage): # This implies that the current row does not have enough qty to fill our requirement. qtyToUseAtThisLevel = qtyForCalculatingWeightedAverage - qSum totalPriceAtThisLevel = qtyToUseAtThisLevel * priceAtThisLevel totalPrice += totalPriceAtThisLevel attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp) attribute.aList[currentRowCount][1] = float(totalPrice)/qtyForCalculatingWeightedAverage currentRowCount += 1 if (currentRowCount%10000==0): print "Processed row number " + str(currentRowCount) lNameOfFeaturePrinted = "fWAPriceOfCol" + args.c + "In01PosInLast" + str(args.n) + "Qty" return ["TimeStamp",lNameOfFeaturePrinted,"Zero1","Zero2"]
def extractAttributeFromDataMatrix(args): try: args.n except: print "Since -n has not been specified I cannot proceed" os._exit(-1) try: args.c ''' if args.n not in args.c: print "Something wrong in design file " print "moving average of last N secs we will taking as qty , so N should be there in sysnthetic fetaure also" ''' except: print "Since -c has not been specified I cannot proceed" os._exit(-1) wt = float(args.n) if(args.cType == "synthetic"): colNumberOfAttribute = colNumberOfData.SysFeature else: colNumberOfAttribute = eval("colNumberOfData."+ args.c ) colNumberOfTimeStamp = colNumberOfData.TimeStamp colNumberOfExchangeStamp = colNumberOfData.ExchangeTS if "Bid" in args.c: side = "Bid" elif "Ask" in args.c: side = "Ask" list_of_price_array = [ eval('colNumberOfData.'+side+'P1') , eval('colNumberOfData.'+side+'P2') ,\ eval('colNumberOfData.'+side+'P3') , eval('colNumberOfData.'+side+'P4') , eval('colNumberOfData.'+side+'P5') ] list_of_qty_array = [ eval('colNumberOfData.'+side+'Q1') , eval('colNumberOfData.'+side+'Q2') ,\ eval('colNumberOfData.'+side+'Q3') , eval('colNumberOfData.'+side+'Q4') , eval('colNumberOfData.'+side+'Q5') ] currentRowCount = 0 levelOfDataAvailable = 4 for dataRow in dataFile.matrix: qSum = 0 totalPrice = 0 totalPriceAtThisLevel = 0 i = 0 try: qtyForCalculatingWeightedAverage = wt * float(dataRow[colNumberOfAttribute]) except: print dataRow print dataRow[colNumberOfAttribute] os._exit(1) while(i <= levelOfDataAvailable and qSum < qtyForCalculatingWeightedAverage): priceAtThisLevel = float(dataFile.matrix[currentRowCount][list_of_price_array[i]]) qtyAtThisLevel = float(dataFile.matrix[currentRowCount][list_of_qty_array[i]]) qSum += qtyAtThisLevel if(qSum > qtyForCalculatingWeightedAverage): qtyToUseAtThisLevel = qtyAtThisLevel - (qSum - qtyForCalculatingWeightedAverage) totalPriceAtThisLevel = qtyToUseAtThisLevel * priceAtThisLevel else: totalPriceAtThisLevel = qtyAtThisLevel * priceAtThisLevel totalPrice += totalPriceAtThisLevel i = i + 1 if(qSum < qtyForCalculatingWeightedAverage): # This implies that the current row does not have enough qty to fill our requirement. qtyToUseAtThisLevel = qtyForCalculatingWeightedAverage - qSum totalPriceAtThisLevel = qtyToUseAtThisLevel * priceAtThisLevel totalPrice += totalPriceAtThisLevel attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfTimeStamp) attribute.aList[currentRowCount][1] = float(totalPrice)/qtyForCalculatingWeightedAverage attribute.aList[currentRowCount][2] = qtyForCalculatingWeightedAverage attribute.aList[currentRowCount][3] = totalPrice currentRowCount += 1 if (currentRowCount%10000==0): print "Processed row number " + str(currentRowCount) lNameOfFeaturePrinted = "fWAPriceOfCol" + side + "InLast" + str(args.n) + "Qty" return ["TimeStamp",lNameOfFeaturePrinted,"QtyUsedForCalWtAvg","TotalPriceCalculatedAtCurrRow"]
def extractAttributeFromDataMatrix(args): queueOfCellValueInFutureNLTQs = deque() totalOfFutureNRows = 0.0 if args.n is None: print "-n has not been specified" os._exit(-1) totalOfFutureLTQQty = int(args.n) lPipSize = int(args.tickSize) lMargin1 = ( 1 * lPipSize ) lMargin2 = lMargin1 + ( 2 * lPipSize ) """ lets get the total of futureNrows""" futureLTQSum = 0 currentRowIndex = 0 lLastTradeIndex = 0 lNoMoreTradesFound = 0 currentLTPValue = 0.0 currentLTQValue = 0 WeightedLTPSum = 0.0 totalLTPQty = totalOfFutureLTQQty while(futureLTQSum < totalOfFutureLTQQty): if dataFile.matrix[currentRowIndex][colNumberOfData.MsgCode].upper() == "T": currentLTPValue = float(dataFile.matrix[currentRowIndex][colNumberOfData.LTP]) currentLTQValue = int(dataFile.matrix[currentRowIndex][colNumberOfData.NewQ]) queueOfCellValueInFutureNLTQs.append( [ (currentLTPValue * currentLTQValue) , currentLTQValue , currentLTPValue , 0 ] ) futureLTQSum = futureLTQSum + currentLTQValue WeightedLTPSum = WeightedLTPSum + queueOfCellValueInFutureNLTQs[-1][0] lLastTradeIndex = currentRowIndex currentRowIndex = currentRowIndex + 1 if futureLTQSum > totalOfFutureLTQQty : currentLTQValueToBeUsed = currentLTQValue - ( futureLTQSum - totalOfFutureLTQQty ) currentTickPriceToBeAdded = currentLTPValue * currentLTQValueToBeUsed LTQValueNotAdded = ( futureLTQSum - totalOfFutureLTQQty ) WeightedLTPSum = WeightedLTPSum - queueOfCellValueInFutureNLTQs[-1][0] + currentTickPriceToBeAdded queueOfCellValueInFutureNLTQs[-1] = [ currentTickPriceToBeAdded , currentLTQValueToBeUsed , currentLTPValue , LTQValueNotAdded ] currentRowCount = 0 for dataRow in dataFile.matrix: attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp) if (dataFile.matrix[currentRowCount][colNumberOfData.MsgCode].upper() == "T") : currentWtLTQToBeSubtracted = queueOfCellValueInFutureNLTQs.popleft() WeightedLTPSum = WeightedLTPSum - currentWtLTQToBeSubtracted[0] LTQSubtracted = currentWtLTQToBeSubtracted[1] if len(queueOfCellValueInFutureNLTQs) != 0: LTQWhichCanBeAddedWithoutNewTradeMessage = queueOfCellValueInFutureNLTQs[-1][3] LTQAddedWithoutNewTradeMessage = min( LTQSubtracted , LTQWhichCanBeAddedWithoutNewTradeMessage) currentTickPriceToBeAdded = ( LTQAddedWithoutNewTradeMessage * queueOfCellValueInFutureNLTQs[-1][2] ) + queueOfCellValueInFutureNLTQs[-1][0] LTQValueNotAddedInLastElementOfQueue = queueOfCellValueInFutureNLTQs[-1][3] - LTQAddedWithoutNewTradeMessage WeightedLTPSum = WeightedLTPSum - queueOfCellValueInFutureNLTQs[-1][0] + currentTickPriceToBeAdded queueOfCellValueInFutureNLTQs[-1][0] = currentTickPriceToBeAdded queueOfCellValueInFutureNLTQs[-1][1] = queueOfCellValueInFutureNLTQs[-1][1] + LTQAddedWithoutNewTradeMessage queueOfCellValueInFutureNLTQs[-1][3] = LTQValueNotAddedInLastElementOfQueue else: LTQAddedWithoutNewTradeMessage = 0 print "Last element in queue = " , currentWtLTQToBeSubtracted if LTQSubtracted > LTQAddedWithoutNewTradeMessage: LTQToBeAddedFromNewTrade = LTQSubtracted - LTQAddedWithoutNewTradeMessage if (lNoMoreTradesFound == 0): startIndexToFindNextTrade = lLastTradeIndex + 1 while( startIndexToFindNextTrade < len(dataFile.matrix)): if dataFile.matrix[startIndexToFindNextTrade][colNumberOfData.MsgCode].upper() == "T": lLastTradeIndex = startIndexToFindNextTrade futureLTPValue = float(dataFile.matrix[startIndexToFindNextTrade][colNumberOfData.LTP]) futureLTQValue = int(dataFile.matrix[startIndexToFindNextTrade][colNumberOfData.NewQ]) queueOfCellValueInFutureNLTQs.append( [ (futureLTPValue * futureLTQValue) , futureLTQValue , futureLTPValue , 0 ] ) LTQToBeAddedFromNewTrade = LTQToBeAddedFromNewTrade - futureLTQValue WeightedLTPSum = WeightedLTPSum + queueOfCellValueInFutureNLTQs[-1][0] if LTQToBeAddedFromNewTrade <= 0 : if LTQToBeAddedFromNewTrade < 0 : LTQValueToBeUsed = futureLTQValue + LTQToBeAddedFromNewTrade currentTickPriceToBeAdded = LTQValueToBeUsed * futureLTPValue LTQValueNotAdded = -1 * LTQToBeAddedFromNewTrade WeightedLTPSum = WeightedLTPSum - queueOfCellValueInFutureNLTQs[-1][0] + currentTickPriceToBeAdded queueOfCellValueInFutureNLTQs[-1] = [ currentTickPriceToBeAdded , LTQValueToBeUsed , futureLTPValue , LTQValueNotAdded ] break startIndexToFindNextTrade = startIndexToFindNextTrade + 1 if startIndexToFindNextTrade == len(dataFile.matrix): if LTQToBeAddedFromNewTrade > 0 : totalLTPQty = totalLTPQty - LTQToBeAddedFromNewTrade lNoMoreTradesFound = 1 else: totalLTPQty = totalLTPQty - LTQToBeAddedFromNewTrade currentBidP0 = float(dataFile.matrix[currentRowCount][colNumberOfData.BidP0]) currentAskP0 = float(dataFile.matrix[currentRowCount][colNumberOfData.AskP0]) currentLTP = float(dataFile.matrix[currentRowCount][colNumberOfData.LTP]) currentMsgCode = dataFile.matrix[currentRowCount][colNumberOfData.MsgCode] currentLTQ = int(dataFile.matrix[currentRowCount][colNumberOfData.NewQ]) lClassOfTargetVariable = 0 valueInCurrentRow = 0 lClassifiedInOneClass = 0 if totalLTPQty != 0: WALTPOfFutureNQty = WeightedLTPSum / totalLTPQty if (WALTPOfFutureNQty - (currentBidP0 + lPipSize)) > lMargin1 : if (WALTPOfFutureNQty - (currentBidP0 + lPipSize)) > lMargin2: lClassOfTargetVariable = 2 lClassifiedInOneClass = 1 else: lClassOfTargetVariable = 1 lClassifiedInOneClass = 1 if ((currentAskP0 - lPipSize) - WALTPOfFutureNQty ) > lMargin1 : if lClassifiedInOneClass == 1 : lClassOfTargetVariable = 0 elif ((currentAskP0 - lPipSize) - WALTPOfFutureNQty ) > lMargin2 : lClassOfTargetVariable = -2 lClassifiedInOneClass = 1 else: lClassOfTargetVariable = -1 lClassifiedInOneClass = 1 attribute.aList[currentRowCount][1] = lClassOfTargetVariable attribute.aList[currentRowCount][2] = currentBidP0 attribute.aList[currentRowCount][3] = ";".join([ str(currentAskP0) , str(WALTPOfFutureNQty) , str(currentLTP) , str(currentLTQ) ,\ str(currentMsgCode) , str(WeightedLTPSum) , str(totalLTPQty) ]) currentRowCount = currentRowCount + 1 if(currentRowCount % 1000 == 0): print "Processed row number " + str(currentRowCount) lNameOfTarget = "tClassOfTargetVariableInFuture"+ str(args.n) + "TradeTicks" return ["TimeStamp",lNameOfTarget,"CurrentBid","CurrentAsk","ValueOfTargetVariableFromCurrentToNextNthTickTaken","CurrentLTP","CurrentLTQ","CurrentMsgCode"\ "WeightedLTPSum" , "totalLTPQtys"]