def extractAttributeFromDataMatrix(args): if args.n == None: N = 5 else: N = int(args.n) try: args.c except: print "Since -c has not been specified I cannot proceed" os._exit() if (args.cType == "synthetic"): colNumberOfAttribute = colNumberOfData.SysFeature else: colNumberOfAttribute = eval("colNumberOfData." + args.c) colNumberOfTimeStamp = colNumberOfData.TimeStamp currentRowNumber = 0 for dataRow in dataFile.matrix: cellValue = float(dataRow[colNumberOfAttribute]) attribute.aList[currentRowNumber][ 0] = common.convertTimeStampFromStringToDecimal( dataRow[colNumberOfTimeStamp]) attribute.aList[currentRowNumber][1] = sqrt( cellValue) # in 1st iteration currentRowNumber = 0 currentRowNumber += 1 if currentRowNumber % 10000 == 0: print "Processed row number " + str(currentRowNumber) lNameOfFeaturePrinted = "fSquareRootOfCol" + args.c + "InCurrentRow" return ["TimeStamp", lNameOfFeaturePrinted, "Zero1", "Zero2"]
def extractAttributeFromDataMatrix(args): if args.n == None: N = 5 else: N = int(args.n) try: args.c except: print "Since -c has not been specified I cannot proceed" os._exit() if(args.cType == "synthetic"): colNumberOfAttribute = colNumberOfData.SysFeature else: colNumberOfAttribute = eval("colNumberOfData."+ args.c ) colNumberOfTimeStamp = colNumberOfData.TimeStamp currentRowNumber = 0 for dataRow in dataFile.matrix: cellValue = float(dataRow[colNumberOfAttribute]) attribute.aList[currentRowNumber][0] = common.convertTimeStampFromStringToDecimal(dataRow[colNumberOfTimeStamp]) attribute.aList[currentRowNumber][1] = sqrt(cellValue) # in 1st iteration currentRowNumber = 0 currentRowNumber += 1 if currentRowNumber%10000 == 0: print "Processed row number " + str(currentRowNumber) lNameOfFeaturePrinted = "fSquareRootOfCol" + args.c + "InCurrentRow" return [ "TimeStamp", lNameOfFeaturePrinted , "Zero1" , "Zero2"]
def extractAttributeFromDataMatrix(args): if args.n == None: N = 5 else: N = int(args.n) try: args.c except: print "Since -c has not been specified I cannot proceed" os._exit() if(args.cType == "synthetic"): colNumberOfAttribute = colNumberOfData.SysFeature else: colNumberOfAttribute = eval("colNumberOfData."+ args.c ) colNumberOfTimeStamp = colNumberOfData.TimeStamp colNumberOfExchangeStamp = colNumberOfData.ExchangeTS currentRowNumberForWhichFeatureValueIsBeingCalculated = 0 for dataRow in dataFile.matrix: lSmartFeatureValue = float(dataRow[colNumberOfAttribute]) lAskP0 = float(colNumberOfData.AskP0) lBidP0 = float(colNumberOfData.BidP0) lMidPrice = (lAskP0+lBidP0) / 2 if lSmartFeatureValue > lMidPrice: lSmartFeatureValueTransform = -lSmartFeatureValue / lAskP0 else: lSmartFeatureValueTransform = lBidP0 / lSmartFeatureValue attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][0] = common.convertTimeStampFromStringToDecimal(dataRow[colNumberOfTimeStamp]) attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][1] = lSmartFeatureValueTransform attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][2] = lAskP0 attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][3] = str(lBidP0) + ";" + str(lSmartFeatureValue) currentRowNumberForWhichFeatureValueIsBeingCalculated += 1 if (currentRowNumberForWhichFeatureValueIsBeingCalculated%10000==0): print "Processed row number " + str(currentRowNumberForWhichFeatureValueIsBeingCalculated) lNameOfFeaturePrinted = "fSmartPriceTransformOfCol" + args.c + "InCurrentRow" return [ "TimeStamp", lNameOfFeaturePrinted , "AskP0" , "BidP0" , "lPreviussmartPrice" ]
def extractAttributeFromDataMatrix(args): if args.n == None: N = 5 else: N = int(args.n) try: args.c except: print "Since -c has not been specified I cannot proceed" os._exit() if(args.cType == "synthetic"): colNumberOfAttribute = colNumberOfData.SysFeature else: colNumberOfAttribute = eval("colNumberOfData."+ args.c ) colNumberOfTimeStamp = colNumberOfData.TimeStamp colNumberOfExchangeStamp = colNumberOfData.ExchangeTS currentRowNumberForWhichFeatureValueIsBeingCalculated = 0 for dataRow in dataFile.matrix: lSmartFeatureValue = float(dataRow[colNumberOfAttribute]) lAskP0 = float(colNumberOfData.AskP0) lBidP0 = float(colNumberOfData.BidP0) lMidPrice = (lAskP0+lBidP0) / 2 if lSmartFeatureValue > lMidPrice: lSmartFeatureValueTransform = -lSmartFeatureValue / lAskP0 else: lSmartFeatureValueTransform = lBidP0 / lSmartFeatureValue attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][0] = common.convertTimeStampFromStringToDecimal(dataRow[colNumberOfTimeStamp]) attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][1] = lSmartFeatureValueTransform attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][2] = lAskP0 attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][3] = str(lBidP0) + ";" + str(lSmartFeatureValue) currentRowNumberForWhichFeatureValueIsBeingCalculated += 1 if (currentRowNumberForWhichFeatureValueIsBeingCalculated%10000==0): print "Processed row number " + str(currentRowNumberForWhichFeatureValueIsBeingCalculated) lNameOfFeaturePrinted = "fSmartPriceTransformOfCol" + args.c + "InCurrentRow" return [ "TimeStamp", lNameOfFeaturePrinted , "AskP0" , "BidP0" , "lPreviussmartPrice" ]
def extractAttributeFromDataMatrix(args): if args.n == None: N = 5 else: N = int(args.n) try: #Value of argument c can be taken as BidP0 and AskP0 args.c except: print "Since -c has not been specified I cannot proceed" os._exit() if args.m == None: M = 1 else: M = int(args.m.split(".")[0]) M1 = int(args.m.split(".")[1]) print "Values of N and N1 are ", M, M1 PIP_SIZE = 5 colNumberOfAttribute = eval("colNumberOfData."+ args.c ) colNumberOfOppositeAttribute = eval("colNumberOfData.BestBidP") if "ask" in args.c else eval("colNumberOfData.BestAskP") colNumberOfTimeStamp = colNumberOfData.TimeStamp if "ask" in args.c.lower(): colAttributeRowNo = [colNumberOfData.BestAskP1, colNumberOfData.BestAskP2]#, colNumberOfData.BestAskP2, colNumberOfData.BestAskP3, colNumberOfData.BestAskP4] colAttributeRowNoBand = [colNumberOfData.AskP1, colNumberOfData.AskP2] else: colAttributeRowNo = [colNumberOfData.BestBidP1, colNumberOfData.BestBidP2]#, colNumberOfData.BestBidP2, colNumberOfData.BestBidP3, colNumberOfData.BestBidP4] colAttributeRowNoBand = [colNumberOfData.BidP1, colNumberOfData.BidP2] numberOfRowsInLastNSecs = 0 queueOfValuesInLastNQty = [] lenQ = 0 totalOfQty = 0.0 currentRowNumberForWhichFeatureValueIsBeingCalculated = 0 lengthOfDataMatrix = len(dataFile.matrix) while (currentRowNumberForWhichFeatureValueIsBeingCalculated < lengthOfDataMatrix): totalOfRowsInLastNQty = 0.0 totalOfQtyInLastNQty = 0.0 cellValueTotal = 0 colAttributeRow = map(float, [dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][i] for i in colAttributeRowNo]) colAttributeRowBand = map(float, [dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][i] for i in colAttributeRowNoBand]) timeOfCurrentRow = common.convertTimeStampFromStringToFloat(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfTimeStamp],args.cType) oppositeAttribute = float(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfOppositeAttribute]) if currentRowNumberForWhichFeatureValueIsBeingCalculated != 0: previousRowPrice = float(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated - 1][colNumberOfAttribute]) else: previousRowPrice = 0 currentPrice = float(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfAttribute]) currentLTP = float(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfData.LTP]) currentMsgCode = dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfData.MsgCode] newQty = int(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfData.NewQ]) newPrice = int(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfData.NewP]) oldQty = int(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfData.OldQ]) oldPrice = int(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfData.OldP]) if ( currentMsgCode == 'M' and ( newPrice in colAttributeRow ) and ((newPrice > oldPrice) if "bid" in args.c.lower() else (newPrice < oldPrice))): cellValueTotal = newPrice * newQty totalOfQty += newQty cellValueQty = newQty queueOfValuesInLastNQty.append([cellValueTotal, cellValueQty, colAttributeRowBand[0]]) lenQ += 1 elif ( currentMsgCode == 'N' and ( newPrice in colAttributeRow )): cellValueTotal = newPrice * newQty totalOfQty += newQty cellValueQty = newQty queueOfValuesInLastNQty.append([cellValueTotal, cellValueQty, colAttributeRowBand[0]]) lenQ += 1 if totalOfQty > N: while totalOfQty > N: excess = totalOfQty - N if queueOfValuesInLastNQty[0][1] < excess: totalOfQty -= queueOfValuesInLastNQty[0][1] queueOfValuesInLastNQty = queueOfValuesInLastNQty[1:] lenQ -= 1 else: totalOfQty -= excess price = float(queueOfValuesInLastNQty[0][0]) / queueOfValuesInLastNQty[0][1] queueOfValuesInLastNQty[0][0] = price * (queueOfValuesInLastNQty[0][1] - excess) queueOfValuesInLastNQty[0][1] -= excess cell = lenQ - 1 if cell >= 0: while ((queueOfValuesInLastNQty[cell][2] > colAttributeRowBand[0] - M * PIP_SIZE and colAttributeRowBand[0] > queueOfValuesInLastNQty[cell][2] - M1 * PIP_SIZE) and ("bid" in args.c.lower())) or ((queueOfValuesInLastNQty[cell][2] < colAttributeRowBand[0] + M * PIP_SIZE and colAttributeRowBand[0] < queueOfValuesInLastNQty[cell][2] + M1 * PIP_SIZE) and ("ask" in args.c.lower())): totalOfRowsInLastNQty += queueOfValuesInLastNQty[cell][0] totalOfQtyInLastNQty += queueOfValuesInLastNQty[cell][1] cell -= 1 if cell < 0: break exPrice = colAttributeRowBand[0] - M * PIP_SIZE if "bid" in args.c.lower() else colAttributeRowBand[0] + M * PIP_SIZE exQty = N - totalOfQtyInLastNQty wtdAvg = float(totalOfRowsInLastNQty + exQty * exPrice) / N attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][0] = common.convertTimeStampFromStringToDecimal(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfTimeStamp],args.cType) attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][1] = wtdAvg # in 1st iteration currentRowNumberForWhichFeatureValueIsBeingCalculated = 0 attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][2] = str(currentPrice) + ";" + str(colAttributeRowBand[0]) + ";" + str(cellValueTotal) + ";" + str(currentMsgCode) + ";" + str(totalOfQtyInLastNQty) + ";" + str(totalOfRowsInLastNQty) currentRowNumberForWhichFeatureValueIsBeingCalculated += 1 continue # Since we are going back 1 row from current we cannot get data from current row print "Processed row number " + str(currentRowNumberForWhichFeatureValueIsBeingCalculated) lNameOfFeaturePrinted = "fMarketMNTSumOfCol" + args.c + "InLast" + str(args.n) + "Qty" + "WithDiff" + str(args.m) + "Pip" return [ "TimeStamp", lNameOfFeaturePrinted , args.c , "LTP" , "TradedQtyWhichIs"+ args.c , "MsgCode", "totalQty", "totalCellValue"]
def extractAttributeFromDataMatrix(args): if args.n == None: N = 5 else: N = int(args.n) try: #Value of argument c can be taken as BidP0 and AskP0 args.c except: print "Since -c has not been specified I cannot proceed" os._exit() if args.m == None: M = 1 else: M = int(args.m.split(".")[0]) M1 = int(args.m.split(".")[1]) print "Values of N and N1 are ", M, M1 PIP_SIZE = 5 colNumberOfAttribute = eval("colNumberOfData." + args.c) colNumberOfOppositeAttribute = eval( "colNumberOfData.BestBidP") if "ask" in args.c else eval( "colNumberOfData.BestAskP") colNumberOfTimeStamp = colNumberOfData.TimeStamp if "ask" in args.c.lower(): colAttributeRowNo = [ colNumberOfData.BestAskP1, colNumberOfData.BestAskP2 ] #, colNumberOfData.BestAskP2, colNumberOfData.BestAskP3, colNumberOfData.BestAskP4] colAttributeRowNoBand = [colNumberOfData.AskP1, colNumberOfData.AskP2] else: colAttributeRowNo = [ colNumberOfData.BestBidP1, colNumberOfData.BestBidP2 ] #, colNumberOfData.BestBidP2, colNumberOfData.BestBidP3, colNumberOfData.BestBidP4] colAttributeRowNoBand = [colNumberOfData.BidP1, colNumberOfData.BidP2] numberOfRowsInLastNSecs = 0 queueOfValuesInLastNQty = [] lenQ = 0 totalOfQty = 0.0 currentRowNumberForWhichFeatureValueIsBeingCalculated = 0 lengthOfDataMatrix = len(dataFile.matrix) while (currentRowNumberForWhichFeatureValueIsBeingCalculated < lengthOfDataMatrix): totalOfRowsInLastNQty = 0.0 totalOfQtyInLastNQty = 0.0 cellValueTotal = 0 colAttributeRow = map(float, [ dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][i] for i in colAttributeRowNo ]) colAttributeRowBand = map(float, [ dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][i] for i in colAttributeRowNoBand ]) timeOfCurrentRow = common.convertTimeStampFromStringToFloat( dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated] [colNumberOfTimeStamp], args.cType) oppositeAttribute = float( dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated] [colNumberOfOppositeAttribute]) if currentRowNumberForWhichFeatureValueIsBeingCalculated != 0: previousRowPrice = float(dataFile.matrix[ currentRowNumberForWhichFeatureValueIsBeingCalculated - 1][colNumberOfAttribute]) else: previousRowPrice = 0 currentPrice = float( dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated] [colNumberOfAttribute]) currentLTP = float( dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][ colNumberOfData.LTP]) currentMsgCode = dataFile.matrix[ currentRowNumberForWhichFeatureValueIsBeingCalculated][ colNumberOfData.MsgCode] newQty = int( dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][ colNumberOfData.NewQ]) newPrice = int( dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][ colNumberOfData.NewP]) oldQty = int( dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][ colNumberOfData.OldQ]) oldPrice = int( dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][ colNumberOfData.OldP]) if (currentMsgCode == 'M' and (newPrice in colAttributeRow) and ((newPrice > oldPrice) if "bid" in args.c.lower() else (newPrice < oldPrice))): cellValueTotal = newPrice * newQty totalOfQty += newQty cellValueQty = newQty queueOfValuesInLastNQty.append( [cellValueTotal, cellValueQty, colAttributeRowBand[0]]) lenQ += 1 elif (currentMsgCode == 'N' and (newPrice in colAttributeRow)): cellValueTotal = newPrice * newQty totalOfQty += newQty cellValueQty = newQty queueOfValuesInLastNQty.append( [cellValueTotal, cellValueQty, colAttributeRowBand[0]]) lenQ += 1 if totalOfQty > N: while totalOfQty > N: excess = totalOfQty - N if queueOfValuesInLastNQty[0][1] < excess: totalOfQty -= queueOfValuesInLastNQty[0][1] queueOfValuesInLastNQty = queueOfValuesInLastNQty[1:] lenQ -= 1 else: totalOfQty -= excess price = float(queueOfValuesInLastNQty[0] [0]) / queueOfValuesInLastNQty[0][1] queueOfValuesInLastNQty[0][0] = price * ( queueOfValuesInLastNQty[0][1] - excess) queueOfValuesInLastNQty[0][1] -= excess cell = lenQ - 1 if cell >= 0: while ((queueOfValuesInLastNQty[cell][2] > colAttributeRowBand[0] - M * PIP_SIZE and colAttributeRowBand[0] > queueOfValuesInLastNQty[cell][2] - M1 * PIP_SIZE) and ("bid" in args.c.lower())) or ( (queueOfValuesInLastNQty[cell][2] < colAttributeRowBand[0] + M * PIP_SIZE and colAttributeRowBand[0] < queueOfValuesInLastNQty[cell][2] + M1 * PIP_SIZE) and ("ask" in args.c.lower())): totalOfRowsInLastNQty += queueOfValuesInLastNQty[cell][0] totalOfQtyInLastNQty += queueOfValuesInLastNQty[cell][1] cell -= 1 if cell < 0: break exPrice = colAttributeRowBand[ 0] - M * PIP_SIZE if "bid" in args.c.lower( ) else colAttributeRowBand[0] + M * PIP_SIZE exQty = N - totalOfQtyInLastNQty wtdAvg = float(totalOfRowsInLastNQty + exQty * exPrice) / N attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][ 0] = common.convertTimeStampFromStringToDecimal( dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated] [colNumberOfTimeStamp], args.cType) attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][ 1] = wtdAvg # in 1st iteration currentRowNumberForWhichFeatureValueIsBeingCalculated = 0 attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][ 2] = str(currentPrice) + ";" + str( colAttributeRowBand[0]) + ";" + str( cellValueTotal) + ";" + str(currentMsgCode) + ";" + str( totalOfQtyInLastNQty) + ";" + str( totalOfRowsInLastNQty) currentRowNumberForWhichFeatureValueIsBeingCalculated += 1 continue # Since we are going back 1 row from current we cannot get data from current row print "Processed row number " + str( currentRowNumberForWhichFeatureValueIsBeingCalculated) lNameOfFeaturePrinted = "fMarketMNTSumOfCol" + args.c + "InLast" + str( args.n) + "Qty" + "WithDiff" + str(args.m) + "Pip" return [ "TimeStamp", lNameOfFeaturePrinted, args.c, "LTP", "TradedQtyWhichIs" + args.c, "MsgCode", "totalQty", "totalCellValue" ]
def extractAttributeFromDataMatrix(args): if args.n == None: N = 5 else: N = float(args.n) try: args.c except: print "Since -c has not been specified I cannot proceed" os._exit() if (args.cType == "synthetic"): colNumberOfAttribute = colNumberOfData.SysFeature else: colNumberOfAttribute = eval("colNumberOfData." + args.c) colNumberOfTimeStamp = colNumberOfData.TimeStamp colNumberOfExchangeStamp = colNumberOfData.ExchangeTS numberOfRowsInLastNSecs = 0 queueOfValuesInLastNSecs = deque() totalOfRowsInLastNSecs = 0.0 timeOfOldestRow = common.convertTimeStampFromStringToFloat( dataFile.matrix[0][colNumberOfExchangeStamp], "synthetic") currentRowNumberForWhichFeatureValueIsBeingCalculated = 0 lengthOfDataMatrix = len(dataFile.matrix) print "lengthOfDataMatrix", lengthOfDataMatrix while (currentRowNumberForWhichFeatureValueIsBeingCalculated < lengthOfDataMatrix): timeOfCurrentRow = common.convertTimeStampFromStringToFloat( dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated] [colNumberOfExchangeStamp], "synthetic") timeElapsed = timeOfCurrentRow - timeOfOldestRow if (timeElapsed < N): cellValue = float( dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated] [colNumberOfAttribute]) totalOfRowsInLastNSecs += cellValue attribute.aList[ currentRowNumberForWhichFeatureValueIsBeingCalculated][ 0] = common.convertTimeStampFromStringToDecimal( dataFile.matrix[ currentRowNumberForWhichFeatureValueIsBeingCalculated] [colNumberOfTimeStamp]) attribute.aList[ currentRowNumberForWhichFeatureValueIsBeingCalculated][ 1] = totalOfRowsInLastNSecs / ( numberOfRowsInLastNSecs + 1 ) # in 1st iteration currentRowNumberForWhichFeatureValueIsBeingCalculated = 0 attribute.aList[ currentRowNumberForWhichFeatureValueIsBeingCalculated][ 2] = str(totalOfRowsInLastNSecs) attribute.aList[ currentRowNumberForWhichFeatureValueIsBeingCalculated][ 3] = str(numberOfRowsInLastNSecs) + ";" + str( timeElapsed) + ";" + str(timeOfCurrentRow) queueOfValuesInLastNSecs.append([cellValue, timeOfCurrentRow]) numberOfRowsInLastNSecs += 1 # Every append gets a +1 currentRowNumberForWhichFeatureValueIsBeingCalculated += 1 continue # Since we are going back 1 row from current we cannot get data from current row else: # We need to reset the timeOfOldestRow since timeElapsed has exceeded N seconds while (timeElapsed >= N): if (len(queueOfValuesInLastNSecs) == 0): timeOfOldestRow = timeOfCurrentRow timeElapsed = 0 if (numberOfRowsInLastNSecs != 0): print "Sanity check: This condition is not possible logically. numberOfRowsInLastNSecs should been 0. There has been an unknown error" sys.exit(-1) if (totalOfRowsInLastNSecs != 0): print "Sanity check: This condition is not possible logically. totalOfRowsInLastNSecs should have been 0. There has been an unknown error" sys.exit(-1) else: oldestElementInQueue = queueOfValuesInLastNSecs.popleft() colValueInOldestElementInQueue = oldestElementInQueue[0] totalOfRowsInLastNSecs -= colValueInOldestElementInQueue if len(queueOfValuesInLastNSecs) == 0: timeElapsed = 0 timeOfOldestRow = timeOfCurrentRow else: timeOfOldestRow = queueOfValuesInLastNSecs[0][1] numberOfRowsInLastNSecs -= 1 # every pop from the queue gets a -1 timeElapsed = timeOfCurrentRow - timeOfOldestRow if (len(queueOfValuesInLastNSecs) != numberOfRowsInLastNSecs): print "Sanity check: This condition is not possible logically. There has been an unknown error" sys.exit(-1) print "Processed row number " + str( currentRowNumberForWhichFeatureValueIsBeingCalculated) lNameOfFeaturePrinted = "fMovingAverageOfCol" + args.c + "InLast" + str( args.n) + "Secs" return [ "TimeStamp", lNameOfFeaturePrinted, "TotalOfRowsInLastNSecs", "NumberOfRowsInLastNSecs", "TimeElapsed", "ExTimeStamp" ]
def extractAttributeFromDataMatrix(args): N = 5 if args.n == None: N = 5 else: N = int(args.n) try: args.c except: print "Since -c has not been specified I cannot proceed" os._exit() try: args.o except: print "Since -o has not been specified I cannot proceed" os._exit() colNumberOfTimeStamp = colNumberOfData.TimeStamp colNumberOfExchangeStamp = colNumberOfData.ExchangeTS numberOfRowsInLastNSecs = 0 l_first_time_elapsed = False queueOfValuesInLastNSecs = deque() timeOfOldestRow = common.convertTimeStampFromStringToFloat(dataFile.matrix[0][colNumberOfExchangeStamp],"synthetic") currentRowNumberForWhichFeatureValueIsBeingCalculated = 0 lengthOfDataMatrix = len(dataFile.matrix) totalTradedQty = 0 lPreviousTickObject = None priceIndex = eval('colNumberOfData.'+args.c+'P0') qtyIndex = [ eval('colNumberOfData.'+args.c+'Q0') , eval('colNumberOfData.'+args.c+'Q1') , eval('colNumberOfData.'+args.c+'Q2') , eval('colNumberOfData.'+args.c+'Q3') , eval('colNumberOfData.'+args.c+'Q4') ] if args.c == 'Ask': orderTypeToBeTracked = 'S' else: orderTypeToBeTracked = 'B' while (currentRowNumberForWhichFeatureValueIsBeingCalculated < lengthOfDataMatrix): lCurrentTickObject = ticks_values_to_be_stored() lCurrentDataRow = dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated] timeOfCurrentRow = common.convertTimeStampFromStringToFloat(lCurrentDataRow[colNumberOfExchangeStamp],"synthetic") timeElapsed = timeOfCurrentRow - timeOfOldestRow if (timeElapsed < N): lCurrentTickObject.MsgCode = lCurrentDataRow[colNumberOfData.MsgCode] lCurrentTickObject.OrderType = lCurrentDataRow[colNumberOfData.OrderType] lCurrentTickObject.NewP = float(lCurrentDataRow[colNumberOfData.NewP]) lCurrentTickObject.NewQ = int(lCurrentDataRow[colNumberOfData.NewQ]) lCurrentTickObject.Price = float( lCurrentDataRow[priceIndex] ) lCurrentTickObject.QtyList = [ int(lCurrentDataRow[qtyIndex[0]]) , int(lCurrentDataRow[qtyIndex[1]]), int(lCurrentDataRow[qtyIndex[2]]),int(lCurrentDataRow[qtyIndex[3]]),int(lCurrentDataRow[qtyIndex[4]]) ] totalTradedQty = updateCurrentTickAdditionToQueue(lCurrentTickObject, lPreviousTickObject , totalTradedQty , timeElapsed , l_first_time_elapsed , N , orderTypeToBeTracked , args.o , numberOfRowsInLastNSecs) attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][0] = common.convertTimeStampFromStringToDecimal(lCurrentDataRow[colNumberOfTimeStamp]) attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][1] = str(lCurrentTickObject.IntensityValue) attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][2] = ";".join( map( str , [ lCurrentDataRow[colNumberOfExchangeStamp] , timeElapsed , totalTradedQty , lCurrentTickObject.MsgCode ,\ lCurrentTickObject.OrderType , lCurrentTickObject.NewP , lCurrentTickObject.NewQ ] ) ) attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][3] = str(lCurrentTickObject.Price) queueOfValuesInLastNSecs.append([lCurrentTickObject,timeOfCurrentRow]) lPreviousTickObject = lCurrentTickObject numberOfRowsInLastNSecs += 1 # Every append gets a +1 currentRowNumberForWhichFeatureValueIsBeingCalculated += 1 continue # Since we are going back 1 row from current we cannot get data from current row else: l_first_time_elapsed = True # We need to reset the timeOfOldestRow since timeElapsed has exceeded N seconds while(timeElapsed >= N): if(len(queueOfValuesInLastNSecs) == 0): timeOfOldestRow = timeOfCurrentRow timeElapsed = 0 if(numberOfRowsInLastNSecs != 0): print "Sanity check: This condition is not possible logically. numberOfRowsInLastNSecs should been 0. There has been an unknown error" sys.exit(-1) else: oldestElementInQueue = queueOfValuesInLastNSecs.popleft() colValueInOldestElementInQueue = oldestElementInQueue[0] totalTradedQty = updateTickDeletionFromQueue(colValueInOldestElementInQueue , totalTradedQty ) if len(queueOfValuesInLastNSecs) == 0: timeElapsed = 0 timeOfOldestRow = timeOfCurrentRow else: timeOfOldestRow = queueOfValuesInLastNSecs[0][1] numberOfRowsInLastNSecs -= 1 # every pop from the queue gets a -1 timeElapsed = timeOfCurrentRow - timeOfOldestRow if(len(queueOfValuesInLastNSecs) != numberOfRowsInLastNSecs): print "Sanity check: This condition is not possible logically. There has been an unknown error" sys.exit(-1) print "Processed row number " + str(currentRowNumberForWhichFeatureValueIsBeingCalculated) lNameOfFeaturePrinted = "fIntensityOfCol" + str(args.c) + "InOrder" + str(args.o) + "InLast" + str(args.n) + "Secs" return [ "TimeStamp", lNameOfFeaturePrinted , "ExTimeStamp","TimeElapsed" ,"totalQty", "MsgCode" , "Ordertype" ,"NewP","NewQ" ,"Price" ]
def extractAttributeFromDataMatrix(args): global currentRowPriceList , currentRowQtyList , currentRowLTP , currentRowTTQ , currentRowMsgCode , currentRowOrderType , currentRowNewP , currentRowNewQ , currentRowOldP , currentRowOldQ global prevRowPriceList , prevRowQtyList , prevRowLTP , prevRowTTQ , prevRowMsgCode , prevRowOrderType , prevRowNewP , prevRowNewQ , prevRowOldP , prevRowOldQ global new_theta , cancel_theta , mod_theta , trade_theta if args.n == None: N = 5 else: N = int(args.n) try: args.c except: print "Since -c has not been specified I cannot proceed" os._exit() colNumberOfPrice = eval("colNumberOfData."+ args.c + "P0" ) colNumberOfQty = eval("colNumberOfData."+ args.c + "Q0") colNumberOfTimeStamp = colNumberOfData.TimeStamp lFirstTimeWindowPrepared = False tickSize = int(args.tickSize) numberOfRowsInLastNSecs = 0 queueOfValuesInLastNSecs = deque() dict_sum_for_price = {} timeOfOldestRow = common.convertTimeStampFromStringToFloat(dataFile.matrix[0][colNumberOfTimeStamp],args.cType) currentRowNumberForWhichFeatureValueIsBeingCalculated = 0 lengthOfDataMatrix = len(dataFile.matrix) while (currentRowNumberForWhichFeatureValueIsBeingCalculated < lengthOfDataMatrix): timeOfCurrentRow = common.convertTimeStampFromStringToFloat(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfTimeStamp],args.cType) timeElapsed = timeOfCurrentRow - timeOfOldestRow dataFileCurrentRow = dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated] currentRowPriceList = [ float(dataFileCurrentRow[colNumberOfPrice]) , float(dataFileCurrentRow[colNumberOfPrice+2]) , float(dataFileCurrentRow[colNumberOfPrice + 4]) , float(dataFileCurrentRow[colNumberOfPrice + 6]) , float(dataFileCurrentRow[colNumberOfPrice + 8]) ] currentRowQtyList = [int(dataFileCurrentRow[colNumberOfQty]),int(dataFileCurrentRow[colNumberOfQty + 2]),int(dataFileCurrentRow[colNumberOfQty + 4]),int(dataFileCurrentRow[colNumberOfQty + 6]),int(dataFileCurrentRow[colNumberOfQty + 8])] currentRowLTP = float(dataFileCurrentRow[colNumberOfData.LTP]) currentRowTTQ = float(dataFileCurrentRow[colNumberOfData.TTQ]) currentRowMsgCode = dataFileCurrentRow[colNumberOfData.MsgCode] currentRowOrderType = dataFileCurrentRow[colNumberOfData.OrderType] currentRowNewP = float(dataFileCurrentRow[colNumberOfData.NewP]) currentRowNewQ = int(dataFileCurrentRow[colNumberOfData.NewQ]) if currentRowMsgCode == 'M': currentRowOldP = float(dataFileCurrentRow[colNumberOfData.OldP]) currentRowOldQ = int(dataFileCurrentRow[colNumberOfData.OldQ]) if (timeElapsed < N): theta_calculation_and_calculate_corresponding_qty(args.c,tickSize,queueOfValuesInLastNSecs,dict_sum_for_price,lFirstTimeWindowPrepared) priceDictionary = create_price_list_dictionary(args.c,tickSize) cellValue = (g_weight_list[0]*dict_sum_for_price[currentRowPriceList[0]]) + (g_weight_list[1]*dict_sum_for_price[currentRowPriceList[1]]) + (g_weight_list[2]*dict_sum_for_price[currentRowPriceList[2]]) + \ (g_weight_list[3]*dict_sum_for_price[currentRowPriceList[3]]) + (g_weight_list[4]*dict_sum_for_price[currentRowPriceList[4]]) attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][0] = common.convertTimeStampFromStringToDecimal(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfTimeStamp],args.cType) attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][1] = cellValue/(numberOfRowsInLastNSecs+1) # in 1st iteration currentRowNumberForWhichFeatureValueIsBeingCalculated = 0 attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][2] = str(cellValue) + ";" + str(numberOfRowsInLastNSecs) + ";" + str(timeElapsed) + ";" + str(dict_sum_for_price[currentRowPriceList[0]]+currentRowQtyList[0]) +\ ";" + str(dict_sum_for_price[currentRowPriceList[1]]+currentRowQtyList[1]) + ";" + str(dict_sum_for_price[currentRowPriceList[2]]+currentRowQtyList[2]) +\ ";" + str(dict_sum_for_price[currentRowPriceList[3]]+currentRowQtyList[3]) +\ ";" + str(dict_sum_for_price[currentRowPriceList[4]]+currentRowQtyList[4]) + ";" + str(new_theta) + ";" + str(cancel_theta) +\ ";" + str(trade_theta) #new_theta , cancel_theta , mod_theta , trade_theta queueOfValuesInLastNSecs.append([cellValue,timeOfCurrentRow,priceDictionary]) numberOfRowsInLastNSecs += 1 # Every append gets a +1 currentRowNumberForWhichFeatureValueIsBeingCalculated += 1 prevRowPriceList = currentRowPriceList prevRowQtyList = currentRowQtyList prevRowLTP = currentRowLTP prevRowTTQ = currentRowTTQ prevRowMsgCode = currentRowMsgCode prevRowOrderType = currentRowOrderType prevRowNewP = currentRowNewP prevRowNewQ = currentRowNewQ if prevRowOrderType == 'M': prevRowOldP = currentRowOldP prevRowOldQ = currentRowOldQ continue # Since we are going back 1 row from current we cannot get data from current row else: # We need to reset the timeOfOldestRow since timeElapsed has exceeded N seconds while(timeElapsed >= N): lFirstTimeWindowPrepared = True if(len(queueOfValuesInLastNSecs) == 0): timeOfOldestRow = timeOfCurrentRow timeElapsed = 0 if(numberOfRowsInLastNSecs != 0): print "Sanity check: This condition is not possible logically. numberOfRowsInLastNSecs should been 0. There has been an unknown error" sys.exit(-1) else: oldestElementInQueue = queueOfValuesInLastNSecs.popleft() colDictInOldestElementInQueue = oldestElementInQueue[2] if len(queueOfValuesInLastNSecs) == 0: timeElapsed = 0 timeOfOldestRow = timeOfCurrentRow else: timeOfOldestRow = queueOfValuesInLastNSecs[0][1] numberOfRowsInLastNSecs -= 1 # every pop from the queue gets a -1 timeElapsed = timeOfCurrentRow - timeOfOldestRow oldestPricesList = colDictInOldestElementInQueue.keys() for lPrice in oldestPricesList: try: dict_sum_for_price[lPrice] -= colDictInOldestElementInQueue[lPrice] except: pass if(len(queueOfValuesInLastNSecs) != numberOfRowsInLastNSecs): print "Sanity check: This condition is not possible logically. There has been an unknown error" sys.exit(-1) print "Processed row number " + str(currentRowNumberForWhichFeatureValueIsBeingCalculated) lNameOfFeaturePrinted = "fOBWaveHistQtyOf" + args.c + "InLast" + str(args.n) + "Secs" return [ "TimeStamp", lNameOfFeaturePrinted , "TotalOfRowsInLastNSecs" , "NumberOfRowsInLastNSecs" , "TimeElapsed","HP0","HP1","HP2","HP3","HP4","NewTheta","CancelTheta","TradeTheta"]
def extractAttributeFromDataMatrix(args): if args.n == None: N = 5 else: N = int(args.n) try: args.c except: print "Since -c has not been specified I cannot proceed" os._exit() colNumberOfTimeStamp = colNumberOfData.TimeStamp colNumberOfExchangeStamp = colNumberOfData.ExchangeTS l_first_time_elapsed = False numberOfRowsInLastNSecs = 0 queueOfValuesInLastNSecs = deque() totalOfRowsInLastNSecs = 0.0 timeOfOldestRow = common.convertTimeStampFromStringToFloat( dataFile.matrix[0][colNumberOfExchangeStamp], "synthetic") currentRowNumberForWhichFeatureValueIsBeingCalculated = 0 lengthOfDataMatrix = len(dataFile.matrix) while (currentRowNumberForWhichFeatureValueIsBeingCalculated < lengthOfDataMatrix): if currentRowNumberForWhichFeatureValueIsBeingCalculated != 0: lPreviousDataRow = dataFile.matrix[ currentRowNumberForWhichFeatureValueIsBeingCalculated - 1] lCurrentDataRow = dataFile.matrix[ currentRowNumberForWhichFeatureValueIsBeingCalculated] lPreviousRowData = dataFile.matrix[ currentRowNumberForWhichFeatureValueIsBeingCalculated - 1] timeOfCurrentRow = common.convertTimeStampFromStringToFloat( lCurrentDataRow[colNumberOfExchangeStamp], "synthetic") timeElapsed = timeOfCurrentRow - timeOfOldestRow if (timeElapsed < N): lMsgCode = lCurrentDataRow[colNumberOfData.MsgCode] lOrderType = lCurrentDataRow[colNumberOfData.OrderType] lNewPrice = float(lCurrentDataRow[colNumberOfData.NewP]) lNewQty = int(lCurrentDataRow[colNumberOfData.NewQ]) if lMsgCode == "M": lOldPrice = float(lCurrentDataRow[colNumberOfData.OldP]) lOldQty = int(lCurrentDataRow[colNumberOfData.OldQ]) if args.c.lower() == "ask": lColCQtyList = [ lCurrentDataRow[colNumberOfData.AskQ0], lCurrentDataRow[colNumberOfData.AskQ1], lCurrentDataRow[colNumberOfData.AskQ2], lCurrentDataRow[colNumberOfData.AskQ3], lCurrentDataRow[colNumberOfData.AskQ4] ] else: lColCQtyList = [ lCurrentDataRow[colNumberOfData.BidQ0], lCurrentDataRow[colNumberOfData.BidQ1], lCurrentDataRow[colNumberOfData.BidQ2], lCurrentDataRow[colNumberOfData.BidQ3], lCurrentDataRow[colNumberOfData.BidQ4] ] lColCPrevPriceList = [] lColCPriceList = [] if args.c.lower() == "ask": lColCPriceList = [ lCurrentDataRow[colNumberOfData.AskP0], lCurrentDataRow[colNumberOfData.AskP1], lCurrentDataRow[colNumberOfData.AskP2], lCurrentDataRow[colNumberOfData.AskP3], lCurrentDataRow[colNumberOfData.AskP4] ] if currentRowNumberForWhichFeatureValueIsBeingCalculated != 0: lColCPrevPriceList = [ lPreviousDataRow[colNumberOfData.AskP0], lPreviousDataRow[colNumberOfData.AskP1], lPreviousDataRow[colNumberOfData.AskP2], lPreviousDataRow[colNumberOfData.AskP3], lPreviousDataRow[colNumberOfData.AskP4] ] else: lColCPriceList = [ lCurrentDataRow[colNumberOfData.BidP0], lCurrentDataRow[colNumberOfData.BidP1], lCurrentDataRow[colNumberOfData.BidP2], lCurrentDataRow[colNumberOfData.BidP3], lCurrentDataRow[colNumberOfData.BidP4] ] if currentRowNumberForWhichFeatureValueIsBeingCalculated != 0: lColCPrevPriceList = [ lPreviousDataRow[colNumberOfData.BidP0], lPreviousDataRow[colNumberOfData.BidP1], lPreviousDataRow[colNumberOfData.BidP2], lPreviousDataRow[colNumberOfData.BidP3], lPreviousDataRow[colNumberOfData.BidP4] ] cellValue = 0 if (lMsgCode == "N" and args.c.lower() == "ask" and lOrderType == "S") or (lMsgCode == "N" and args.c.lower() == "bid" and lOrderType == "B"): cellValue = lNewQty elif (lMsgCode == "X" and args.c.lower() == "ask" and lOrderType == "S") or (lMsgCode == "X" and args.c.lower() == "bid" and lOrderType == "B"): cellValue = -1 * lNewQty elif (lMsgCode == "M" and args.c.lower() == "ask" and lOrderType == "S") or (lMsgCode == "M" and args.c.lower() == "bid" and lOrderType == "B"): if len(lColCPrevPriceList) == 0: cellValue = lNewQty #Case 1------------------------------------------------------------------------------------------------------------ if (lNewPrice in lColCPriceList) and (lOldPrice not in lColCPrevPriceList): cellValue = lNewQty #Case 2------------------------------------------------------------------------------------------------------------ elif (lNewPrice not in lColCPriceList) and (lOldPrice in lColCPrevPriceList): cellValue = -1 * lOldQty else: cellValue = lNewQty - lOldQty elif (len(lColCPrevPriceList) != 0) and ( lMsgCode == "T" and lNewPrice == lColCPrevPriceList[0]): cellValue = -1 * lNewQty totalOfRowsInLastNSecs = totalOfRowsInLastNSecs + cellValue lAvgValue = 0 if timeElapsed != 0: if l_first_time_elapsed == False: lAvgValue = float(totalOfRowsInLastNSecs) / ( numberOfRowsInLastNSecs + 1) else: lAvgValue = float(totalOfRowsInLastNSecs) / N else: lAvgValue = 1 attribute.aList[ currentRowNumberForWhichFeatureValueIsBeingCalculated][ 0] = common.convertTimeStampFromStringToDecimal( lCurrentDataRow[colNumberOfTimeStamp]) attribute.aList[ currentRowNumberForWhichFeatureValueIsBeingCalculated][ 1] = lAvgValue # in 1st iteration currentRowNumberForWhichFeatureValueIsBeingCalculated = 0 attribute.aList[ currentRowNumberForWhichFeatureValueIsBeingCalculated][ 2] = str(cellValue) + ";" + str(timeElapsed) queueOfValuesInLastNSecs.append([cellValue, timeOfCurrentRow]) numberOfRowsInLastNSecs += 1 # Every append gets a +1 currentRowNumberForWhichFeatureValueIsBeingCalculated += 1 continue # Since we are going back 1 row from current we cannot get data from current row else: l_first_time_elapsed = True # We need to reset the timeOfOldestRow since timeElapsed has exceeded N seconds while (timeElapsed >= N): if (len(queueOfValuesInLastNSecs) == 0): timeOfOldestRow = timeOfCurrentRow timeElapsed = 0 if (numberOfRowsInLastNSecs != 0): print "Sanity check: This condition is not possible logically. numberOfRowsInLastNSecs should been 0. There has been an unknown error" sys.exit(-1) if (totalOfRowsInLastNSecs != 0): print "Sanity check: This condition is not possible logically. totalOfRowsInLastNSecs should have been 0. There has been an unknown error" sys.exit(-1) else: oldestElementInQueue = queueOfValuesInLastNSecs.popleft() colValueInOldestElementInQueue = oldestElementInQueue[0] totalOfRowsInLastNSecs -= colValueInOldestElementInQueue if len(queueOfValuesInLastNSecs) == 0: timeElapsed = 0 timeOfOldestRow = timeOfCurrentRow else: timeOfOldestRow = queueOfValuesInLastNSecs[0][1] numberOfRowsInLastNSecs -= 1 # every pop from the queue gets a -1 timeElapsed = timeOfCurrentRow - timeOfOldestRow if (len(queueOfValuesInLastNSecs) != numberOfRowsInLastNSecs): print "Sanity check: This condition is not possible logically. There has been an unknown error" sys.exit(-1) print "Processed row number " + str( currentRowNumberForWhichFeatureValueIsBeingCalculated) lNameOfFeaturePrinted = "fMovSumMsgQtyOverCol" + args.c + "InLast" + str( args.n) + "Secs" return [ "TimeStamp", lNameOfFeaturePrinted, "QtyAddedOrSubtracted", "TimeElapsed" ]
def extractAttributeFromDataMatrix(args): if args.n == None: N = 5 else: N = int(args.n) try: #Value of argument c can be taken as BidP0 and AskP0 args.c except: print "Since -c has not been specified I cannot proceed" os._exit() colNumberOfAttribute = eval("colNumberOfData."+ args.c ) colNumberOfOppositeAttribute = eval("colNumberOfData.BestBidP") if "ask" in args.c else eval("colNumberOfData.BestAskP") colNumberOfTimeStamp = colNumberOfData.TimeStamp if "ask" in args.c.lower(): colAttributeRowNo = [colNumberOfData.BestAskP, colNumberOfData.BestAskP1, colNumberOfData.BestAskP2, colNumberOfData.BestAskP3, colNumberOfData.BestAskP4] else: colAttributeRowNo = [colNumberOfData.BestBidP, colNumberOfData.BestBidP1, colNumberOfData.BestBidP2, colNumberOfData.BestBidP3, colNumberOfData.BestBidP4] numberOfRowsInLastNSecs = 0 queueOfValuesInLastNSecs = deque() totalOfRowsInLastNSecsX = 0.0 totalOfRowsInLastNSecsN = 0.0 totalOfQtyInLastNSecsX = 0.0 totalOfQtyInLastNSecsN = 0.0 timeOfOldestRow = common.convertTimeStampFromStringToFloat(dataFile.matrix[0][colNumberOfTimeStamp],args.cType) currentRowNumberForWhichFeatureValueIsBeingCalculated = 0 lengthOfDataMatrix = len(dataFile.matrix) while (currentRowNumberForWhichFeatureValueIsBeingCalculated < lengthOfDataMatrix): colAttributeRow = map(float, [dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][i] for i in colAttributeRowNo]) timeOfCurrentRow = common.convertTimeStampFromStringToFloat(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfTimeStamp],args.cType) timeElapsed = timeOfCurrentRow - timeOfOldestRow if (timeElapsed < N): if currentRowNumberForWhichFeatureValueIsBeingCalculated != 0: previousRowPrice = float(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated - 1][colNumberOfAttribute]) else: previousRowPrice = 0 currentPrice = float(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfAttribute]) currentLTP = float(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfData.LTP]) currentMsgCode = dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfData.MsgCode] newQty = int(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfData.NewQ]) newPrice = int(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfData.NewP]) oldQty = int(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfData.OldQ]) oldPrice = int(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfData.OldP]) if ( currentMsgCode =='T' and (( colNumberOfOppositeAttribute <= currentLTP ) if "bid" in args.c.lower() else (colNumberOfOppositeAttribute >= currentLTP))): totalOfRowsInLastNSecsX += newQty * currentLTP cellValueTotalX = newQty * currentLTP totalOfQtyInLastNSecsX += newQty cellValueQtyX = newQty cellValueTotalN = 0 cellValueQtyN = 0 elif currentMsgCode == 'M': totalOfRowsInLastNSecsN += (newPrice * newQty) if ( newPrice in colAttributeRow ) else 0 cellValueTotalN = newPrice * newQty if ( newPrice in colAttributeRow ) else 0 totalOfQtyInLastNSecsN += newQty if ( newPrice in colAttributeRow ) else 0 cellValueQtyN = newQty if ( newPrice in colAttributeRow ) else 0 totalOfRowsInLastNSecsX += oldPrice * oldQty if ( oldPrice in colAttributeRow ) else 0 cellValueTotalX = oldPrice * oldQty if ( oldPrice in colAttributeRow ) else 0 totalOfQtyInLastNSecsX += oldQty if ( oldPrice in colAttributeRow ) else 0 cellValueQtyX = oldQty if ( oldPrice in colAttributeRow ) else 0 elif ( currentMsgCode == 'N' and ( newPrice in colAttributeRow )): totalOfRowsInLastNSecsN += newPrice * newQty cellValueTotalN = newPrice * newQty totalOfQtyInLastNSecsN += newQty cellValueQtyN = newQty cellValueTotalX = 0 cellValueQtyX = 0 elif ( currentMsgCode == 'X' and ( newPrice in colAttributeRow )): totalOfRowsInLastNSecsX += newPrice * newQty cellValueTotalX = newPrice * newQty totalOfQtyInLastNSecsX += newQty cellValueQtyX = newQty cellValueTotalN = 0 cellValueQtyN = 0 else: cellValueTotalN = 0 cellValueQtyN = 0 cellValueTotalX = 0 cellValueQtyX = 0 maxQty = max(totalOfQtyInLastNSecsN, totalOfQtyInLastNSecsX) newSide = (float(totalOfRowsInLastNSecsN + (maxQty - totalOfQtyInLastNSecsN) * colAttributeRow[4]) / maxQty) if maxQty > 0 else currentPrice cancelSide = (float(totalOfRowsInLastNSecsX + (maxQty - totalOfQtyInLastNSecsX) * colAttributeRow[4]) / maxQty) if maxQty > 0 else currentPrice attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][0] = common.convertTimeStampFromStringToDecimal(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfTimeStamp],args.cType) attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][1] = (float(newSide) / cancelSide) # in 1st iteration currentRowNumberForWhichFeatureValueIsBeingCalculated = 0 attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][2] = str(currentPrice) + ";" + str(cellValueQtyN) + ";" + str(cellValueQtyX) + ";" + str(cellValueTotalN) + ";" + str(cellValueTotalX) + ";" + str(currentMsgCode) + ";" + str(timeElapsed) queueOfValuesInLastNSecs.append([cellValueTotalN,timeOfCurrentRow,cellValueQtyN,cellValueTotalX,cellValueQtyX]) numberOfRowsInLastNSecs += 1 # Every append gets a +1 currentRowNumberForWhichFeatureValueIsBeingCalculated += 1 continue # Since we are going back 1 row from current we cannot get data from current row else: # We need to reset the timeOfOldestRow since timeElapsed has exceeded N seconds while(timeElapsed >= N): if(len(queueOfValuesInLastNSecs) == 0): timeOfOldestRow = timeOfCurrentRow timeElapsed = 0 if(numberOfRowsInLastNSecs != 0): print "Sanity check: This condition is not possible logically. numberOfRowsInLastNSecs should been 0. There has been an unknown error" sys.exit(-1) if(totalOfRowsInLastNSecs != 0): print "Sanity check: This condition is not possible logically. totalOfRowsInLastNSecs should have been 0. There has been an unknown error" sys.exit(-1) else: oldestElementInQueue = queueOfValuesInLastNSecs.popleft() colValueInOldestElementInQueueN = oldestElementInQueue[0] colQtyInOldestElementInQueueN = oldestElementInQueue[2] colValueInOldestElementInQueueX = oldestElementInQueue[3] colQtyInOldestElementInQueueX = oldestElementInQueue[4] totalOfRowsInLastNSecsN -= colValueInOldestElementInQueueN totalOfQtyInLastNSecsN -= colQtyInOldestElementInQueueN totalOfRowsInLastNSecsX -= colValueInOldestElementInQueueX totalOfQtyInLastNSecsX -= colQtyInOldestElementInQueueX if len(queueOfValuesInLastNSecs) == 0: timeElapsed = 0 timeOfOldestRow = timeOfCurrentRow else: timeOfOldestRow = queueOfValuesInLastNSecs[0][1] numberOfRowsInLastNSecs -= 1 # every pop from the queue gets a -1 timeElapsed = timeOfCurrentRow - timeOfOldestRow if(len(queueOfValuesInLastNSecs) != numberOfRowsInLastNSecs): print "Sanity check: This condition is not possible logically. There has been an unknown error" sys.exit(-1) print "Processed row number " + str(currentRowNumberForWhichFeatureValueIsBeingCalculated) lNameOfFeaturePrinted = "fMarketTradedQtySumOfCol" + args.c + "InLast" + str(args.n) + "Secs" return [ "TimeStamp", lNameOfFeaturePrinted , args.c , "QtyForN" , "QtyForX", "AmountN" , "AmountX", "MsgCode" , "TimeElapsed"]
def extractAttributeFromDataMatrix(args): if args.n == None: N = 5 else: N = int(args.n) try: args.c except: print "Since -c has not been specified I cannot proceed" os._exit() if(args.cType == "synthetic"): colNumberOfAttribute = colNumberOfData.SysFeature else: colNumberOfAttribute = eval("colNumberOfData."+ args.c ) colNumberOfTimeStamp = colNumberOfData.TimeStamp colNumberOfExchangeStamp = colNumberOfData.ExchangeTS numberOfRowsInLastNSecs = 0 queueOfValuesInLastNSecs = deque() totalOfRowsInLastNSecs = 0.0 totalOfSquareOfRowsInLastNSecs = 0.0 timeOfOldestRow = common.convertTimeStampFromStringToFloat(dataFile.matrix[0][colNumberOfExchangeStamp],"synthetic") currentRowNumberForWhichFeatureValueIsBeingCalculated = 0 lengthOfDataMatrix = len(dataFile.matrix) while (currentRowNumberForWhichFeatureValueIsBeingCalculated < lengthOfDataMatrix): timeOfCurrentRow = common.convertTimeStampFromStringToFloat(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfExchangeStamp],"synthetic") timeElapsed = timeOfCurrentRow - timeOfOldestRow if (timeElapsed < N): cellValue = float(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfAttribute]) totalOfRowsInLastNSecs += cellValue totalOfSquareOfRowsInLastNSecs += ( cellValue * cellValue ) meanOfRows = totalOfRowsInLastNSecs/(numberOfRowsInLastNSecs+1) meanOfSquareOfRows = totalOfSquareOfRowsInLastNSecs/( numberOfRowsInLastNSecs+1 ) variance = meanOfSquareOfRows - ( meanOfRows * meanOfRows ) if variance < 0: variance = 0 totalOfSquareOfRowsInLastNSecs = ( meanOfRows * meanOfRows ) * ( numberOfRowsInLastNSecs+1 ) attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][0] = common.convertTimeStampFromStringToDecimal(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfTimeStamp]) attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][1] = variance # in 1st iteration currentRowNumberForWhichFeatureValueIsBeingCalculated = 0 attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][2] = str(timeElapsed) attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][3] = str(totalOfRowsInLastNSecs)+ ";" + str(totalOfSquareOfRowsInLastNSecs) + ";" + str(numberOfRowsInLastNSecs+1) queueOfValuesInLastNSecs.append([cellValue,timeOfCurrentRow]) numberOfRowsInLastNSecs += 1 # Every append gets a +1 currentRowNumberForWhichFeatureValueIsBeingCalculated += 1 continue # Since we are going back 1 row from current we cannot get data from current row else: # We need to reset the timeOfOldestRow since timeElapsed has exceeded N seconds while(timeElapsed >= N): if(len(queueOfValuesInLastNSecs) == 0): timeOfOldestRow = timeOfCurrentRow timeElapsed = 0 if(numberOfRowsInLastNSecs != 0): print "Sanity check: This condition is not possible logically. numberOfRowsInLastNSecs should been 0. There has been an unknown error" sys.exit(-1) if(totalOfRowsInLastNSecs != 0): print "Sanity check: This condition is not possible logically. totalOfRowsInLastNSecs should have been 0. There has been an unknown error" sys.exit(-1) else: oldestElementInQueue = queueOfValuesInLastNSecs.popleft() colValueInOldestElementInQueue = oldestElementInQueue[0] if len(queueOfValuesInLastNSecs) == 0: timeElapsed = 0 timeOfOldestRow = timeOfCurrentRow else: timeOfOldestRow = queueOfValuesInLastNSecs[0][1] totalOfRowsInLastNSecs -= colValueInOldestElementInQueue totalOfSquareOfRowsInLastNSecs -= ( colValueInOldestElementInQueue * colValueInOldestElementInQueue ) numberOfRowsInLastNSecs -= 1 # every pop from the queue gets a -1 timeElapsed = timeOfCurrentRow - timeOfOldestRow if(len(queueOfValuesInLastNSecs) != numberOfRowsInLastNSecs): print "Sanity check: This condition is not possible logically. There has been an unknown error" sys.exit(-1) print "Processed row number " + str(currentRowNumberForWhichFeatureValueIsBeingCalculated) lNameOfFeaturePrinted = "fVarianceOfCol" + args.c + "InLast" + str(args.n) + "Secs" return [ "TimeStamp", lNameOfFeaturePrinted , "TimeElapsed","TotalOfRowsInLastNSecs" , "TotalOfSquareOfRowsInLastNSecs" , "NumberOfRowsInLastNSecs" , ]
def extractAttributeFromDataMatrix(args): N = 5 if args.n == None: N = 5 else: N = int(args.n) try: args.c except: print "Since -c has not been specified I cannot proceed" os._exit() try: args.o except: print "Since -o has not been specified I cannot proceed" os._exit() colNumberOfTimeStamp = colNumberOfData.TimeStamp colNumberOfExchangeStamp = colNumberOfData.ExchangeTS numberOfRowsInLastNSecs = 0 l_first_time_elapsed = False queueOfValuesInLastNSecs = deque() timeOfOldestRow = common.convertTimeStampFromStringToFloat( dataFile.matrix[0][colNumberOfExchangeStamp], "synthetic") currentRowNumberForWhichFeatureValueIsBeingCalculated = 0 lengthOfDataMatrix = len(dataFile.matrix) totalTradedQty = 0 lPreviousTickObject = None priceIndex = eval('colNumberOfData.' + args.c + 'P0') qtyIndex = [ eval('colNumberOfData.' + args.c + 'Q0'), eval('colNumberOfData.' + args.c + 'Q1'), eval('colNumberOfData.' + args.c + 'Q2'), eval('colNumberOfData.' + args.c + 'Q3'), eval('colNumberOfData.' + args.c + 'Q4') ] if args.c == 'Ask': orderTypeToBeTracked = 'S' else: orderTypeToBeTracked = 'B' while (currentRowNumberForWhichFeatureValueIsBeingCalculated < lengthOfDataMatrix): lCurrentTickObject = ticks_values_to_be_stored() lCurrentDataRow = dataFile.matrix[ currentRowNumberForWhichFeatureValueIsBeingCalculated] timeOfCurrentRow = common.convertTimeStampFromStringToFloat( lCurrentDataRow[colNumberOfExchangeStamp], "synthetic") timeElapsed = timeOfCurrentRow - timeOfOldestRow if (timeElapsed < N): lCurrentTickObject.MsgCode = lCurrentDataRow[ colNumberOfData.MsgCode] lCurrentTickObject.OrderType = lCurrentDataRow[ colNumberOfData.OrderType] lCurrentTickObject.NewP = float( lCurrentDataRow[colNumberOfData.NewP]) lCurrentTickObject.NewQ = int( lCurrentDataRow[colNumberOfData.NewQ]) lCurrentTickObject.Price = float(lCurrentDataRow[priceIndex]) lCurrentTickObject.QtyList = [ int(lCurrentDataRow[qtyIndex[0]]), int(lCurrentDataRow[qtyIndex[1]]), int(lCurrentDataRow[qtyIndex[2]]), int(lCurrentDataRow[qtyIndex[3]]), int(lCurrentDataRow[qtyIndex[4]]) ] totalTradedQty = updateCurrentTickAdditionToQueue( lCurrentTickObject, lPreviousTickObject, totalTradedQty, timeElapsed, l_first_time_elapsed, N, orderTypeToBeTracked, args.o, numberOfRowsInLastNSecs) attribute.aList[ currentRowNumberForWhichFeatureValueIsBeingCalculated][ 0] = common.convertTimeStampFromStringToDecimal( lCurrentDataRow[colNumberOfTimeStamp]) attribute.aList[ currentRowNumberForWhichFeatureValueIsBeingCalculated][ 1] = str(lCurrentTickObject.IntensityValue) attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][2] = ";".join( map( str , [ lCurrentDataRow[colNumberOfExchangeStamp] , timeElapsed , totalTradedQty , lCurrentTickObject.MsgCode ,\ lCurrentTickObject.OrderType , lCurrentTickObject.NewP , lCurrentTickObject.NewQ ] ) ) attribute.aList[ currentRowNumberForWhichFeatureValueIsBeingCalculated][ 3] = str(lCurrentTickObject.Price) queueOfValuesInLastNSecs.append( [lCurrentTickObject, timeOfCurrentRow]) lPreviousTickObject = lCurrentTickObject numberOfRowsInLastNSecs += 1 # Every append gets a +1 currentRowNumberForWhichFeatureValueIsBeingCalculated += 1 continue # Since we are going back 1 row from current we cannot get data from current row else: l_first_time_elapsed = True # We need to reset the timeOfOldestRow since timeElapsed has exceeded N seconds while (timeElapsed >= N): if (len(queueOfValuesInLastNSecs) == 0): timeOfOldestRow = timeOfCurrentRow timeElapsed = 0 if (numberOfRowsInLastNSecs != 0): print "Sanity check: This condition is not possible logically. numberOfRowsInLastNSecs should been 0. There has been an unknown error" sys.exit(-1) else: oldestElementInQueue = queueOfValuesInLastNSecs.popleft() colValueInOldestElementInQueue = oldestElementInQueue[0] totalTradedQty = updateTickDeletionFromQueue( colValueInOldestElementInQueue, totalTradedQty) if len(queueOfValuesInLastNSecs) == 0: timeElapsed = 0 timeOfOldestRow = timeOfCurrentRow else: timeOfOldestRow = queueOfValuesInLastNSecs[0][1] numberOfRowsInLastNSecs -= 1 # every pop from the queue gets a -1 timeElapsed = timeOfCurrentRow - timeOfOldestRow if (len(queueOfValuesInLastNSecs) != numberOfRowsInLastNSecs): print "Sanity check: This condition is not possible logically. There has been an unknown error" sys.exit(-1) print "Processed row number " + str( currentRowNumberForWhichFeatureValueIsBeingCalculated) lNameOfFeaturePrinted = "fIntensityOfCol" + str(args.c) + "InOrder" + str( args.o) + "InLast" + str(args.n) + "Secs" return [ "TimeStamp", lNameOfFeaturePrinted, "ExTimeStamp", "TimeElapsed", "totalQty", "MsgCode", "Ordertype", "NewP", "NewQ", "Price" ]
def extractAttributeFromDataMatrix(args): if args.n == None: N = 5 else: N = int(args.n) try: #Value of argument c can be taken as BidP0 and AskP0 args.c except: print "Since -c has not been specified I cannot proceed" os._exit() try: M = int(args.m) except: print "-m wrong!!" colNumberOfAttribute = eval("colNumberOfData."+ args.c ) colNumberOfOppositeAttribute = eval("colNumberOfData.BestBidP") if "ask" in args.c else eval("colNumberOfData.BestAskP") colNumberOfTimeStamp = colNumberOfData.TimeStamp if "ask" in args.c.lower(): colAttributeRowNo = [colNumberOfData.AskP0, colNumberOfData.AskP1]#, colNumberOfData.AskP2, colNumberOfData.AskP3, colNumberOfData.AskP4] else: colAttributeRowNo = [colNumberOfData.BidP0, colNumberOfData.BidP1]#, colNumberOfData.BidP2, colNumberOfData.BidP3, colNumberOfData.BidP4] numberOfRowsInLastNSecs = 0 queueOfValuesInLastNSecsN = [] queueOfValuesInLastNSecsX = [] totalOfRowsInLastNSecsX = 0.0 totalOfRowsInLastNSecsN = 0.0 totalOfQtyInLastNSecsX = 0.0 totalOfQtyInLastNSecsN = 0.0 timeOfOldestRow = common.convertTimeStampFromStringToFloat(dataFile.matrix[0][colNumberOfTimeStamp],args.cType) currentRowNumberForWhichFeatureValueIsBeingCalculated = 0 lengthOfDataMatrix = len(dataFile.matrix) while (currentRowNumberForWhichFeatureValueIsBeingCalculated < lengthOfDataMatrix): cellValueQtyN, cellValueQtyX, cellValueTotalN, cellValueTotalX = [0,0,0,0] colAttributeRow = map(float, [dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][i] for i in colAttributeRowNo]) timeOfCurrentRow = common.convertTimeStampFromStringToFloat(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfTimeStamp],args.cType) if currentRowNumberForWhichFeatureValueIsBeingCalculated != 0: previousRowPrice = float(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated - 1][colNumberOfAttribute]) previousRowOppPrice = float(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated - 1][colNumberOfOppositeAttribute]) else: previousRowPrice = 0 previousRowOppPrice = 0 currentPrice = float(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfAttribute]) currentLTP = float(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfData.LTP]) currentMsgCode = dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfData.MsgCode] newQty = int(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfData.NewQ]) newPrice = int(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfData.NewP]) oldQty = int(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfData.OldQ]) oldPrice = int(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfData.OldP]) if ( currentMsgCode =='T' and (( previousRowOppPrice <= currentLTP ) if "bid" in args.c.lower() else (previousRowOppPrice >= currentLTP))): totalOfRowsInLastNSecsN += newQty * currentLTP cellValueTotalN = newQty * currentLTP totalOfQtyInLastNSecsN += newQty cellValueQtyN = newQty queueOfValuesInLastNSecsN.append([cellValueTotalN,colAttributeRow[-1],cellValueQtyN,currentLTP,timeOfCurrentRow]) elif currentMsgCode == 'M': if ("ask" in args.c.lower() and newPrice <= colAttributeRow[-1] and newPrice >= currentPrice) or ("bid" in args.c.lower() and newPrice >= colAttributeRow[-1] and newPrice <= currentPrice): check = True else: check = False totalOfRowsInLastNSecsN += (newPrice * newQty) if check else 0 cellValueTotalN = newPrice * newQty if check else 0 totalOfQtyInLastNSecsN += newQty if check else 0 cellValueQtyN = newQty if check else 0 if check: queueOfValuesInLastNSecsN.append([cellValueTotalN,colAttributeRow[-1],cellValueQtyN,newPrice,timeOfCurrentRow]) if ("ask" in args.c.lower() and oldPrice <= colAttributeRow[-1] and oldPrice >= previousRowPrice) or ("bid" in args.c.lower() and oldPrice >= colAttributeRow[-1] and oldPrice <= previousRowPrice): check = True else: check = False totalOfRowsInLastNSecsX += oldPrice * oldQty if check else 0 cellValueTotalX = oldPrice * oldQty if check else 0 totalOfQtyInLastNSecsX += oldQty if check else 0 cellValueQtyX = oldQty if check else 0 if check: queueOfValuesInLastNSecsX.append([cellValueTotalX,colAttributeRow[-1],cellValueQtyX,oldPrice,timeOfCurrentRow]) elif ( currentMsgCode == 'N'): if ("ask" in args.c.lower() and newPrice <= colAttributeRow[-1] and newPrice >= currentPrice) or ("bid" in args.c.lower() and newPrice >= colAttributeRow[-1] and newPrice <= currentPrice): check = True else: check = False totalOfRowsInLastNSecsN += newPrice * newQty if check else 0 cellValueTotalN = newPrice * newQty if check else 0 totalOfQtyInLastNSecsN += newQty if check else 0 cellValueQtyN = newQty if check else 0 if check: queueOfValuesInLastNSecsN.append([cellValueTotalN,colAttributeRow[-1],cellValueQtyN,newPrice,timeOfCurrentRow]) elif ( currentMsgCode == 'X'): if ("ask" in args.c.lower() and newPrice <= colAttributeRow[-1] and newPrice >= previousRowPrice) or ("bid" in args.c.lower() and newPrice >= colAttributeRow[-1] and newPrice <= previousRowPrice): check = True else: check = False totalOfRowsInLastNSecsX += newPrice * newQty if check else 0 cellValueTotalX = newPrice * newQty if check else 0 totalOfQtyInLastNSecsX += newQty if check else 0 cellValueQtyX = newQty if check else 0 if check: queueOfValuesInLastNSecsX.append([cellValueTotalX,colAttributeRow[-1],cellValueQtyX,newPrice,timeOfCurrentRow]) if(totalOfQtyInLastNSecsN) > N: while totalOfQtyInLastNSecsN > N: excess = totalOfQtyInLastNSecsN - N if queueOfValuesInLastNSecsN[0][2] <= excess: totalOfQtyInLastNSecsN -= queueOfValuesInLastNSecsN[0][2] totalOfRowsInLastNSecsN -= queueOfValuesInLastNSecsN[0][0] queueOfValuesInLastNSecsN = queueOfValuesInLastNSecsN[1:] else: totalOfQtyInLastNSecsN -= excess totalOfRowsInLastNSecsN -= excess * float(queueOfValuesInLastNSecsN[0][0]) / queueOfValuesInLastNSecsN[0][2] queueOfValuesInLastNSecsN[0][2] -= excess queueOfValuesInLastNSecsN[0][0] -= excess * queueOfValuesInLastNSecsN[0][3] if(totalOfQtyInLastNSecsX) > N: while totalOfQtyInLastNSecsX > N: excess = totalOfQtyInLastNSecsX - N if queueOfValuesInLastNSecsX[0][2] <= excess: totalOfQtyInLastNSecsX -= queueOfValuesInLastNSecsX[0][2] totalOfRowsInLastNSecsX -= queueOfValuesInLastNSecsX[0][0] queueOfValuesInLastNSecsX = queueOfValuesInLastNSecsX[1:] else: totalOfQtyInLastNSecsX -= excess totalOfRowsInLastNSecsX -= excess * float(queueOfValuesInLastNSecsX[0][0]) / queueOfValuesInLastNSecsX[0][2] queueOfValuesInLastNSecsX[0][2] -= excess queueOfValuesInLastNSecsX[0][0] -= excess * queueOfValuesInLastNSecsX[0][3] if M == 1: exPrice = colAttributeRow[-1] elif M == 2: try: exPrice = queueOfValuesInLastNSecsN[0][1] if queueOfValuesInLastNSecsN[0][-1] < queueOfValuesInLastNSecsX[0][-1] else queueOfValuesInLastNSecsX[0][1] except: print("Warning!!!!") exPrice = colAttributeRow[-1] elif M == 3: try: exPrice = totalOfRowsInLastNSecsN / float(totalOfQtyInLastNSecsN) except: print("Warning!!!!") exPrice = colAttributeRow[-1] elif M == 4: try: exPrice = totalOfRowsInLastNSecsX / float(totalOfQtyInLastNSecsX) except: print("Warning!!!!") exPrice = colAttributeRow[-1] newSide = (float(totalOfRowsInLastNSecsN + (N - totalOfQtyInLastNSecsN) * exPrice) / N) cancelSide = (float(totalOfRowsInLastNSecsX + (N - totalOfQtyInLastNSecsX) * exPrice) / N) attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][0] = common.convertTimeStampFromStringToDecimal(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfTimeStamp],args.cType) attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][1] = (float(newSide) - cancelSide) # in 1st iteration currentRowNumberForWhichFeatureValueIsBeingCalculated = 0 attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][2] = str(currentPrice) + ";" + str(cellValueQtyN) + ";" + str(cellValueQtyX) + ";" + str(cellValueTotalN) + ";" + str(cellValueTotalX) + ";" + str(currentMsgCode) numberOfRowsInLastNSecs += 1 # Every append gets a +1 # if currentRowNumberForWhichFeatureValueIsBeingCalculated == 164: # import pdb # pdb.set_trace() currentRowNumberForWhichFeatureValueIsBeingCalculated += 1 continue # Since we are going back 1 row from current we cannot get data from current row print "Processed row number " + str(currentRowNumberForWhichFeatureValueIsBeingCalculated) lNameOfFeaturePrinted = "fMarketModNewSumOfCol" + args.c + "InLast" + str(args.n) + "Secs" return [ "TimeStamp", lNameOfFeaturePrinted , args.c , "QtyForN" , "QtyForX", "AmountN" , "AmountX", "MsgCode"]
def extractAttributeFromDataMatrix(args): if args.n == None: N = 5 else: N = int(args.n) try: #Value of argument c can be taken as BidP0 and AskP0 args.c except: print "Since -c has not been specified I cannot proceed" os._exit() colNumberOfAttribute = eval("colNumberOfData." + args.c) colNumberOfOppositeAttribute = eval( "colNumberOfData.BestBidP") if "ask" in args.c else eval( "colNumberOfData.BestAskP") colNumberOfTimeStamp = colNumberOfData.TimeStamp if "ask" in args.c.lower(): colAttributeRowNo = [ colNumberOfData.BestAskP, colNumberOfData.BestAskP1 ] #, colNumberOfData.BestAskP2, colNumberOfData.BestAskP3, colNumberOfData.BestAskP4] else: colAttributeRowNo = [ colNumberOfData.BestBidP, colNumberOfData.BestBidP1 ] #, colNumberOfData.BestBidP2, colNumberOfData.BestBidP3, colNumberOfData.BestBidP4] numberOfRowsInLastNSecs = 0 queueOfValuesInLastNSecs = deque() totalOfRowsInLastNSecs = 0.0 totalOfRowsInLastNSecs = 0.0 totalOfQtyInLastNSecs = 0.0 timeOfOldestRow = common.convertTimeStampFromStringToFloat( dataFile.matrix[0][colNumberOfTimeStamp], args.cType) currentRowNumberForWhichFeatureValueIsBeingCalculated = 0 lengthOfDataMatrix = len(dataFile.matrix) while (currentRowNumberForWhichFeatureValueIsBeingCalculated < lengthOfDataMatrix): colAttributeRow = map(float, [ dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][i] for i in colAttributeRowNo ]) timeOfCurrentRow = common.convertTimeStampFromStringToFloat( dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated] [colNumberOfTimeStamp], args.cType) timeElapsed = timeOfCurrentRow - timeOfOldestRow oppositeAttribute = float( dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated] [colNumberOfOppositeAttribute]) if (timeElapsed < N): if currentRowNumberForWhichFeatureValueIsBeingCalculated != 0: previousRowPrice = float(dataFile.matrix[ currentRowNumberForWhichFeatureValueIsBeingCalculated - 1][colNumberOfAttribute]) else: previousRowPrice = 0 currentPrice = float( dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated] [colNumberOfAttribute]) currentLTP = float( dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][ colNumberOfData.LTP]) currentMsgCode = dataFile.matrix[ currentRowNumberForWhichFeatureValueIsBeingCalculated][ colNumberOfData.MsgCode] newQty = int( dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][ colNumberOfData.NewQ]) newPrice = int( dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][ colNumberOfData.NewP]) oldQty = int( dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][ colNumberOfData.OldQ]) oldPrice = int( dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][ colNumberOfData.OldP]) if (currentMsgCode == 'T' and ((oppositeAttribute <= currentLTP) if "bid" in args.c.lower() else (oppositeAttribute >= currentLTP))): totalOfRowsInLastNSecs += newQty * currentLTP cellValueTotal = newQty * currentLTP totalOfQtyInLastNSecs += newQty cellValueQty = newQty if (currentMsgCode == 'M' and (newPrice in colAttributeRow) and ((newPrice > oldPrice) if "bid" in args.c.lower() else (newPrice < oldPrice))): totalOfRowsInLastNSecs += newPrice * newQty cellValueTotal = newPrice * newQty totalOfQtyInLastNSecs += newQty cellValueQty = newQty elif (currentMsgCode == 'N' and (newPrice in colAttributeRow)): totalOfRowsInLastNSecs += newPrice * newQty cellValueTotal = newPrice * newQty totalOfQtyInLastNSecs += newQty cellValueQty = newQty else: cellValueTotal = 0 cellValueQty = 0 attribute.aList[ currentRowNumberForWhichFeatureValueIsBeingCalculated][ 0] = common.convertTimeStampFromStringToDecimal( dataFile.matrix[ currentRowNumberForWhichFeatureValueIsBeingCalculated] [colNumberOfTimeStamp], args.cType) attribute.aList[ currentRowNumberForWhichFeatureValueIsBeingCalculated][1] = ( float(totalOfRowsInLastNSecs) / totalOfQtyInLastNSecs ) if totalOfQtyInLastNSecs > 0 else currentPrice # in 1st iteration currentRowNumberForWhichFeatureValueIsBeingCalculated = 0 attribute.aList[ currentRowNumberForWhichFeatureValueIsBeingCalculated][ 2] = str(currentPrice) + ";" + str(currentLTP) + ";" + str( cellValueTotal) + ";" + str( currentMsgCode) + ";" + str(timeElapsed) queueOfValuesInLastNSecs.append( [cellValueTotal, timeOfCurrentRow, cellValueQty]) numberOfRowsInLastNSecs += 1 # Every append gets a +1 currentRowNumberForWhichFeatureValueIsBeingCalculated += 1 continue # Since we are going back 1 row from current we cannot get data from current row else: # We need to reset the timeOfOldestRow since timeElapsed has exceeded N seconds while (timeElapsed >= N): if (len(queueOfValuesInLastNSecs) == 0): timeOfOldestRow = timeOfCurrentRow timeElapsed = 0 if (numberOfRowsInLastNSecs != 0): print "Sanity check: This condition is not possible logically. numberOfRowsInLastNSecs should been 0. There has been an unknown error" sys.exit(-1) if (totalOfRowsInLastNSecs != 0): print "Sanity check: This condition is not possible logically. totalOfRowsInLastNSecs should have been 0. There has been an unknown error" sys.exit(-1) else: oldestElementInQueue = queueOfValuesInLastNSecs.popleft() colValueInOldestElementInQueue = oldestElementInQueue[0] colQtyInOldestElementInQueue = oldestElementInQueue[2] totalOfRowsInLastNSecs -= colValueInOldestElementInQueue totalOfQtyInLastNSecs -= colQtyInOldestElementInQueue if len(queueOfValuesInLastNSecs) == 0: timeElapsed = 0 timeOfOldestRow = timeOfCurrentRow else: timeOfOldestRow = queueOfValuesInLastNSecs[0][1] numberOfRowsInLastNSecs -= 1 # every pop from the queue gets a -1 timeElapsed = timeOfCurrentRow - timeOfOldestRow if (len(queueOfValuesInLastNSecs) != numberOfRowsInLastNSecs): print "Sanity check: This condition is not possible logically. There has been an unknown error" sys.exit(-1) print "Processed row number " + str( currentRowNumberForWhichFeatureValueIsBeingCalculated) lNameOfFeaturePrinted = "fMarketTradedQtySumOfCol" + args.c + "InLast" + str( args.n) + "Secs" return [ "TimeStamp", lNameOfFeaturePrinted, args.c, "LTP", "TradedQtyWhichIs=" + args.c, "MsgCode", "TimeElapsed" ]
def extractAttributeFromDataMatrix(args): if args.n == None: N = 5 else: N = int(args.n) try: #Value of argument c can be taken as BidP0 and AskP0 args.c except: print "Since -c has not been specified I cannot proceed" os._exit() colNumberOfAttribute = eval("colNumberOfData."+ args.c ) colNumberOfTimeStamp = colNumberOfData.TimeStamp numberOfRowsInLastNSecs = 0 queueOfValuesInLastNSecs = deque() totalOfRowsInLastNSecs = 0.0 timeOfOldestRow = common.convertTimeStampFromStringToFloat(dataFile.matrix[0][colNumberOfTimeStamp],args.cType) currentRowNumberForWhichFeatureValueIsBeingCalculated = 0 lengthOfDataMatrix = len(dataFile.matrix) while (currentRowNumberForWhichFeatureValueIsBeingCalculated < lengthOfDataMatrix): timeOfCurrentRow = common.convertTimeStampFromStringToFloat(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfTimeStamp],args.cType) timeElapsed = timeOfCurrentRow - timeOfOldestRow if (timeElapsed < N): if currentRowNumberForWhichFeatureValueIsBeingCalculated != 0: previousRowPrice = float(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated - 1][colNumberOfAttribute]) else: previousRowPrice = 0 currentPrice = float(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfAttribute]) currentLTP = float(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfData.LTP]) currentMsgCode = dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfData.MsgCode] currentLTQ = int(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfData.NewQ]) if ( currentMsgCode =='T' and ( previousRowPrice == currentLTP ) ): totalOfRowsInLastNSecs += currentLTQ cellValue = currentLTQ else: cellValue = 0 attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][0] = common.convertTimeStampFromStringToDecimal(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfTimeStamp],args.cType) attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][1] = totalOfRowsInLastNSecs # in 1st iteration currentRowNumberForWhichFeatureValueIsBeingCalculated = 0 attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][2] = str(currentPrice) + ";" + str(currentLTP) + ";" + str(cellValue) + ";" + str(currentMsgCode) + ";" + str(timeElapsed) queueOfValuesInLastNSecs.append([cellValue,timeOfCurrentRow]) numberOfRowsInLastNSecs += 1 # Every append gets a +1 currentRowNumberForWhichFeatureValueIsBeingCalculated += 1 continue # Since we are going back 1 row from current we cannot get data from current row else: # We need to reset the timeOfOldestRow since timeElapsed has exceeded N seconds while(timeElapsed >= N): if(len(queueOfValuesInLastNSecs) == 0): timeOfOldestRow = timeOfCurrentRow timeElapsed = 0 if(numberOfRowsInLastNSecs != 0): print "Sanity check: This condition is not possible logically. numberOfRowsInLastNSecs should been 0. There has been an unknown error" sys.exit(-1) if(totalOfRowsInLastNSecs != 0): print "Sanity check: This condition is not possible logically. totalOfRowsInLastNSecs should have been 0. There has been an unknown error" sys.exit(-1) else: oldestElementInQueue = queueOfValuesInLastNSecs.popleft() colValueInOldestElementInQueue = oldestElementInQueue[0] totalOfRowsInLastNSecs -= colValueInOldestElementInQueue if len(queueOfValuesInLastNSecs) == 0: timeElapsed = 0 timeOfOldestRow = timeOfCurrentRow else: timeOfOldestRow = queueOfValuesInLastNSecs[0][1] numberOfRowsInLastNSecs -= 1 # every pop from the queue gets a -1 timeElapsed = timeOfCurrentRow - timeOfOldestRow if(len(queueOfValuesInLastNSecs) != numberOfRowsInLastNSecs): print "Sanity check: This condition is not possible logically. There has been an unknown error" sys.exit(-1) print "Processed row number " + str(currentRowNumberForWhichFeatureValueIsBeingCalculated) lNameOfFeaturePrinted = "fMarketTradedQtySumOfCol" + args.c + "InLast" + str(args.n) + "Secs" return [ "TimeStamp", lNameOfFeaturePrinted , args.c , "LTP" , "TradedQtyWhichIs="+ args.c , "MsgCode" , "TimeElapsed"]
def extractAttributeFromDataMatrix(args): if args.n == None: N = 5 else: N = float(args.n) try: args.c except: print "Since -c has not been specified I cannot proceed" os._exit() colNumberOfLTP = colNumberOfData.NiftyLTP colNumberOfTTQ = colNumberOfData.NiftyTTQ colNumberOfTimeStamp = colNumberOfData.TimeStamp colNumberOfExchangeStamp = colNumberOfData.ExchangeTS numberOfRowsInLastNSecs = 0 queueOfValuesInLastNSecs = deque() totalOfRowsInLastNSecs = 0.0 timeOfOldestRow = common.convertTimeStampFromStringToFloat(dataFile.matrix[0][colNumberOfExchangeStamp],"synthetic") currentRowNumberForWhichFeatureValueIsBeingCalculated = 0 l_sum_LtQ = 0 prevTTQ = 0 lengthOfDataMatrix = len(dataFile.matrix) l_prev_feature_val = 0 print "lengthOfDataMatrix",lengthOfDataMatrix while (currentRowNumberForWhichFeatureValueIsBeingCalculated < lengthOfDataMatrix): timeOfCurrentRow = common.convertTimeStampFromStringToFloat(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfExchangeStamp],"synthetic") timeElapsed = timeOfCurrentRow - timeOfOldestRow TtqOfCurrentRow = int(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfTTQ]) LtpCurrentRow = float(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfLTP]) if (timeElapsed < N): l_diff_TTQ = TtqOfCurrentRow-prevTTQ cellValue = LtpCurrentRow * l_diff_TTQ totalOfRowsInLastNSecs += cellValue l_sum_LtQ += l_diff_TTQ attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][0] = common.convertTimeStampFromStringToDecimal(dataFile.matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][colNumberOfTimeStamp]) if l_sum_LtQ != 0: attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][1] = totalOfRowsInLastNSecs/(l_sum_LtQ) # in 1st iteration currentRowNumberForWhichFeatureValueIsBeingCalculated = 0 l_prev_feature_val = totalOfRowsInLastNSecs/(l_sum_LtQ) else: attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][1] = l_prev_feature_val attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][2] = str(totalOfRowsInLastNSecs) attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][3] = str(l_sum_LtQ) + ";" + str(TtqOfCurrentRow) + ";"+str(LtpCurrentRow) + ";" + str(timeElapsed) + ";" + str(timeOfCurrentRow) queueOfValuesInLastNSecs.append([l_diff_TTQ,cellValue,timeOfCurrentRow]) prevTTQ = TtqOfCurrentRow numberOfRowsInLastNSecs +=1 currentRowNumberForWhichFeatureValueIsBeingCalculated += 1 continue # Since we are going back 1 row from current we cannot get data from current row else: # We need to reset the timeOfOldestRow since timeElapsed has exceeded N seconds while(timeElapsed >= N): if(len(queueOfValuesInLastNSecs) == 0): timeOfOldestRow = timeOfCurrentRow timeElapsed = 0 if(numberOfRowsInLastNSecs != 0): print "Sanity check: This condition is not possible logically. numberOfRowsInLastNSecs should been 0. There has been an unknown error" sys.exit(-1) if(totalOfRowsInLastNSecs != 0): print "Sanity check: This condition is not possible logically. totalOfRowsInLastNSecs should have been 0. There has been an unknown error" sys.exit(-1) else: oldestElementInQueue = queueOfValuesInLastNSecs.popleft() LtqValueInOldRow = oldestElementInQueue[0] colValueInOldestElementInQueue = oldestElementInQueue[1] totalOfRowsInLastNSecs -= colValueInOldestElementInQueue l_sum_LtQ -= LtqValueInOldRow if len(queueOfValuesInLastNSecs) == 0: timeElapsed = 0 timeOfOldestRow = timeOfCurrentRow else: timeOfOldestRow = queueOfValuesInLastNSecs[0][2] numberOfRowsInLastNSecs -= 1 # every pop from the queue gets a -1 timeElapsed = timeOfCurrentRow - timeOfOldestRow if(len(queueOfValuesInLastNSecs) != numberOfRowsInLastNSecs): print "Sanity check: This condition is not possible logically. There has been an unknown error" sys.exit(-1) print "Processed row number " + str(currentRowNumberForWhichFeatureValueIsBeingCalculated) lNameOfFeaturePrinted = "fWALTPInLast" + str(args.n) + "Secs" return [ "TimeStamp", lNameOfFeaturePrinted , "totalValueOfRowsInLastNSecs" , "LtqSum" ,"TTQ", "LTP", "TimeElapsed" ,"ExTimeStamp"]
def extractAttributeFromDataMatrix(args): if args.n == None: N = 5 else: N = int(args.n) try: #Value of argument c can be taken as BidP0 and AskP0 args.c except: print "Since -c has not been specified I cannot proceed" os._exit() try: M = int(args.m) except: print "-m wrong!!" colNumberOfAttribute = eval("colNumberOfData." + args.c) colNumberOfOppositeAttribute = eval( "colNumberOfData.BestBidP") if "ask" in args.c else eval( "colNumberOfData.BestAskP") colNumberOfTimeStamp = colNumberOfData.TimeStamp if "ask" in args.c.lower(): colAttributeRowNo = [ colNumberOfData.AskP0, colNumberOfData.AskP1 ] #, colNumberOfData.AskP2, colNumberOfData.AskP3, colNumberOfData.AskP4] else: colAttributeRowNo = [ colNumberOfData.BidP0, colNumberOfData.BidP1 ] #, colNumberOfData.BidP2, colNumberOfData.BidP3, colNumberOfData.BidP4] numberOfRowsInLastNSecs = 0 queueOfValuesInLastNSecsN = [] queueOfValuesInLastNSecsX = [] totalOfRowsInLastNSecsX = 0.0 totalOfRowsInLastNSecsN = 0.0 totalOfQtyInLastNSecsX = 0.0 totalOfQtyInLastNSecsN = 0.0 timeOfOldestRow = common.convertTimeStampFromStringToFloat( dataFile.matrix[0][colNumberOfTimeStamp], args.cType) currentRowNumberForWhichFeatureValueIsBeingCalculated = 0 lengthOfDataMatrix = len(dataFile.matrix) while (currentRowNumberForWhichFeatureValueIsBeingCalculated < lengthOfDataMatrix): cellValueQtyN, cellValueQtyX, cellValueTotalN, cellValueTotalX = [ 0, 0, 0, 0 ] colAttributeRow = map(float, [ dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][i] for i in colAttributeRowNo ]) timeOfCurrentRow = common.convertTimeStampFromStringToFloat( dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated] [colNumberOfTimeStamp], args.cType) if currentRowNumberForWhichFeatureValueIsBeingCalculated != 0: previousRowPrice = float(dataFile.matrix[ currentRowNumberForWhichFeatureValueIsBeingCalculated - 1][colNumberOfAttribute]) previousRowOppPrice = float(dataFile.matrix[ currentRowNumberForWhichFeatureValueIsBeingCalculated - 1][colNumberOfOppositeAttribute]) else: previousRowPrice = 0 previousRowOppPrice = 0 currentPrice = float( dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated] [colNumberOfAttribute]) currentLTP = float( dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][ colNumberOfData.LTP]) currentMsgCode = dataFile.matrix[ currentRowNumberForWhichFeatureValueIsBeingCalculated][ colNumberOfData.MsgCode] newQty = int( dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][ colNumberOfData.NewQ]) newPrice = int( dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][ colNumberOfData.NewP]) oldQty = int( dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][ colNumberOfData.OldQ]) oldPrice = int( dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated][ colNumberOfData.OldP]) if (currentMsgCode == 'T' and ( (previousRowOppPrice <= currentLTP) if "bid" in args.c.lower() else (previousRowOppPrice >= currentLTP))): totalOfRowsInLastNSecsN += newQty * currentLTP cellValueTotalN = newQty * currentLTP totalOfQtyInLastNSecsN += newQty cellValueQtyN = newQty queueOfValuesInLastNSecsN.append([ cellValueTotalN, colAttributeRow[-1], cellValueQtyN, currentLTP, timeOfCurrentRow ]) elif currentMsgCode == 'M': if ("ask" in args.c.lower() and newPrice <= colAttributeRow[-1] and newPrice >= currentPrice) or ( "bid" in args.c.lower() and newPrice >= colAttributeRow[-1] and newPrice <= currentPrice): check = True else: check = False totalOfRowsInLastNSecsN += (newPrice * newQty) if check else 0 cellValueTotalN = newPrice * newQty if check else 0 totalOfQtyInLastNSecsN += newQty if check else 0 cellValueQtyN = newQty if check else 0 if check: queueOfValuesInLastNSecsN.append([ cellValueTotalN, colAttributeRow[-1], cellValueQtyN, newPrice, timeOfCurrentRow ]) if ("ask" in args.c.lower() and oldPrice <= colAttributeRow[-1] and oldPrice >= previousRowPrice) or ( "bid" in args.c.lower() and oldPrice >= colAttributeRow[-1] and oldPrice <= previousRowPrice): check = True else: check = False totalOfRowsInLastNSecsX += oldPrice * oldQty if check else 0 cellValueTotalX = oldPrice * oldQty if check else 0 totalOfQtyInLastNSecsX += oldQty if check else 0 cellValueQtyX = oldQty if check else 0 if check: queueOfValuesInLastNSecsX.append([ cellValueTotalX, colAttributeRow[-1], cellValueQtyX, oldPrice, timeOfCurrentRow ]) elif (currentMsgCode == 'N'): if ("ask" in args.c.lower() and newPrice <= colAttributeRow[-1] and newPrice >= currentPrice) or ( "bid" in args.c.lower() and newPrice >= colAttributeRow[-1] and newPrice <= currentPrice): check = True else: check = False totalOfRowsInLastNSecsN += newPrice * newQty if check else 0 cellValueTotalN = newPrice * newQty if check else 0 totalOfQtyInLastNSecsN += newQty if check else 0 cellValueQtyN = newQty if check else 0 if check: queueOfValuesInLastNSecsN.append([ cellValueTotalN, colAttributeRow[-1], cellValueQtyN, newPrice, timeOfCurrentRow ]) elif (currentMsgCode == 'X'): if ("ask" in args.c.lower() and newPrice <= colAttributeRow[-1] and newPrice >= previousRowPrice) or ( "bid" in args.c.lower() and newPrice >= colAttributeRow[-1] and newPrice <= previousRowPrice): check = True else: check = False totalOfRowsInLastNSecsX += newPrice * newQty if check else 0 cellValueTotalX = newPrice * newQty if check else 0 totalOfQtyInLastNSecsX += newQty if check else 0 cellValueQtyX = newQty if check else 0 if check: queueOfValuesInLastNSecsX.append([ cellValueTotalX, colAttributeRow[-1], cellValueQtyX, newPrice, timeOfCurrentRow ]) if (totalOfQtyInLastNSecsN) > N: while totalOfQtyInLastNSecsN > N: excess = totalOfQtyInLastNSecsN - N if queueOfValuesInLastNSecsN[0][2] <= excess: totalOfQtyInLastNSecsN -= queueOfValuesInLastNSecsN[0][2] totalOfRowsInLastNSecsN -= queueOfValuesInLastNSecsN[0][0] queueOfValuesInLastNSecsN = queueOfValuesInLastNSecsN[1:] else: totalOfQtyInLastNSecsN -= excess totalOfRowsInLastNSecsN -= excess * float( queueOfValuesInLastNSecsN[0] [0]) / queueOfValuesInLastNSecsN[0][2] queueOfValuesInLastNSecsN[0][2] -= excess queueOfValuesInLastNSecsN[0][ 0] -= excess * queueOfValuesInLastNSecsN[0][3] if (totalOfQtyInLastNSecsX) > N: while totalOfQtyInLastNSecsX > N: excess = totalOfQtyInLastNSecsX - N if queueOfValuesInLastNSecsX[0][2] <= excess: totalOfQtyInLastNSecsX -= queueOfValuesInLastNSecsX[0][2] totalOfRowsInLastNSecsX -= queueOfValuesInLastNSecsX[0][0] queueOfValuesInLastNSecsX = queueOfValuesInLastNSecsX[1:] else: totalOfQtyInLastNSecsX -= excess totalOfRowsInLastNSecsX -= excess * float( queueOfValuesInLastNSecsX[0] [0]) / queueOfValuesInLastNSecsX[0][2] queueOfValuesInLastNSecsX[0][2] -= excess queueOfValuesInLastNSecsX[0][ 0] -= excess * queueOfValuesInLastNSecsX[0][3] if M == 1: exPrice = colAttributeRow[-1] elif M == 2: try: exPrice = queueOfValuesInLastNSecsN[0][ 1] if queueOfValuesInLastNSecsN[0][ -1] < queueOfValuesInLastNSecsX[0][ -1] else queueOfValuesInLastNSecsX[0][1] except: print("Warning!!!!") exPrice = colAttributeRow[-1] elif M == 3: try: exPrice = totalOfRowsInLastNSecsN / float( totalOfQtyInLastNSecsN) except: print("Warning!!!!") exPrice = colAttributeRow[-1] elif M == 4: try: exPrice = totalOfRowsInLastNSecsX / float( totalOfQtyInLastNSecsX) except: print("Warning!!!!") exPrice = colAttributeRow[-1] newSide = (float(totalOfRowsInLastNSecsN + (N - totalOfQtyInLastNSecsN) * exPrice) / N) cancelSide = (float(totalOfRowsInLastNSecsX + (N - totalOfQtyInLastNSecsX) * exPrice) / N) attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][ 0] = common.convertTimeStampFromStringToDecimal( dataFile. matrix[currentRowNumberForWhichFeatureValueIsBeingCalculated] [colNumberOfTimeStamp], args.cType) attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][ 1] = ( float(newSide) - cancelSide ) # in 1st iteration currentRowNumberForWhichFeatureValueIsBeingCalculated = 0 attribute.aList[currentRowNumberForWhichFeatureValueIsBeingCalculated][ 2] = str(currentPrice) + ";" + str(cellValueQtyN) + ";" + str( cellValueQtyX) + ";" + str(cellValueTotalN) + ";" + str( cellValueTotalX) + ";" + str(currentMsgCode) numberOfRowsInLastNSecs += 1 # Every append gets a +1 # if currentRowNumberForWhichFeatureValueIsBeingCalculated == 164: # import pdb # pdb.set_trace() currentRowNumberForWhichFeatureValueIsBeingCalculated += 1 continue # Since we are going back 1 row from current we cannot get data from current row print "Processed row number " + str( currentRowNumberForWhichFeatureValueIsBeingCalculated) lNameOfFeaturePrinted = "fMarketModNewSumOfCol" + args.c + "InLast" + str( args.n) + "Secs" return [ "TimeStamp", lNameOfFeaturePrinted, args.c, "QtyForN", "QtyForX", "AmountN", "AmountX", "MsgCode" ]