def extractAttributeFromDataMatrix(args):
if args.n is None:
print "N has not been specified"
os._exit(-1)
try:
args.c
colNumberWeAreWorkingWith = eval('colNumberOfData.'+args.c)
except:
print "C has not been specified"
os._exit(-1)
numberOfSecondsInFuture = int(args.n)
currentRowCount = 0
totalNumberOfDataRows = len(dataFile.matrix)
futureRowCount = 1 # Since we have a queue we do not need to initialize everytime inside the for loop
totalOfRowsInFutureNSeconds = 0.0
queue = deque()
for dataRow in dataFile.matrix:
timeOfCurrentRow = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp)
# lets remove all values from queue which have time <= timeOfCurrentRow
if(len(queue) > 0): # When currentRowCount is 0 then there is nothing in the queue
oldestElementinQueue = queue.popleft()
while(oldestElementinQueue[0] < timeOfCurrentRow):
oldestElementinQueue = queue.popleft()
totalOfRowsInFutureNSeconds -= oldestElementinQueue[1]
attribute.aList[currentRowCount][0] = timeOfCurrentRow
while True:
if(currentRowCount + futureRowCount < totalNumberOfDataRows): # If the totalNumberOfDataRows is 5 we can access index values 0 to 4.
timeOfFutureRow = common.getTimeStamp(dataFile.matrix[currentRowCount + futureRowCount])
if(timeOfFutureRow - timeOfCurrentRow < numberOfSecondsInFuture):
cellValue = float(dataFile.matrix[currentRowCount + futureRowCount][colNumberWeAreWorkingWith])
totalOfRowsInFutureNSeconds += cellValue
futureRowCount = futureRowCount + 1
# add the data to queue for future needs
queue.append([timeOfFutureRow,cellValue])
else:
break
else:
break
averageOfFutureRows = totalOfRowsInFutureNSeconds / futureRowCount
currentCellValue = float(dataFile.matrix[currentRowCount][colNumberWeAreWorkingWith])
if( averageOfFutureRows > currentCellValue):
attribute.aList[currentRowCount][1] = 1
else:
attribute.aList[currentRowCount][1] = 0
attribute.aList[currentRowCount][2] = totalOfRowsInFutureNSeconds
attribute.aList[currentRowCount][3] = float(dataFile.matrix[currentRowCount][colNumberOfData.LTP])
currentRowCount = currentRowCount + 1
if(currentRowCount % 1 == 0):
print "Processed row number " + str(currentRowCount)
lNameOfTarget = "tAverageOfCol" + args.c + "InFuture" + str(args.n) + "SecsIsGreaterThanCol" + args.c + "InCurrentRow"
return ["TimeStamp",lNameOfTarget,"TotalOfFutureNSecs","CurrentRowLTPValue"]
def extractAttributeFromDataMatrix(args):
queueOfCellValueInLastNRows = deque()
totalOfLastNRows = 0.0
if args.n == None:
N = 5
else:
N = int(args.n)
try:
args.c
except:
print "Since -c has not been specified I cannot proceed"
os._exit()
currentRowCount = 0
if(args.cType == "synthetic"):
colNumberOfAttribute = colNumberOfData.SysFeature
else:
colNumberOfAttribute = eval("colNumberOfData."+ args.c )
colNumberOfTimeStamp = colNumberOfData.TimeStamp
for dataRow in dataFile.matrix:
cellValue = float(dataFile.matrix[currentRowCount][colNumberOfAttribute])
queueOfCellValueInLastNRows.append(cellValue)
totalOfLastNRows += cellValue
if (currentRowCount < N):
attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfTimeStamp)
attribute.aList[currentRowCount][1] = totalOfLastNRows/(currentRowCount+1) # in 1st iteration currentRowCount = 0
currentRowCount = currentRowCount + 1
continue # Since we are going back 1 row from current we cannot get data from current row
totalOfLastNRows -= queueOfCellValueInLastNRows.popleft()
# In the next 2 rows we do not do -1 since this feature if for the current row.
attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfTimeStamp)
attribute.aList[currentRowCount][1] = totalOfLastNRows / N
currentRowCount = currentRowCount + 1
if (currentRowCount%10000==0):
print "Processed row number " + str(currentRowCount)
lNameOfFeaturePrinted = "fMovingAverageOfCol" + args.c + "InLast" + str(args.n) + "Rows"
return ["TimeStamp",lNameOfFeaturePrinted,"Zero1","Zero2"]
def extractAttributeFromDataMatrix(args):
queueOfCellValueInFutureNRows = deque()
totalOfFutureNRows = 0.0
if args.n is None:
print "-n has not been specified"
os._exit(-1)
numberOfFutureRows = int(args.n)
try:
args.c
except:
print "-c has not been specified"
os._exit(-1)
""" lets get the total of futureNrows"""
futureRowCount = 0
while (futureRowCount < numberOfFutureRows):
codeString = 'float(dataFile.matrix[futureRowCount][colNumberOfData.' + args.c + '])'
cellValue = eval(codeString)
queueOfCellValueInFutureNRows.append(cellValue)
totalOfFutureNRows += cellValue
futureRowCount = futureRowCount + 1
currentRowCount = 0
for dataRow in dataFile.matrix:
attribute.aList[currentRowCount][0] = common.getTimeStamp(
dataFile.matrix[currentRowCount], colNumberOfData.TimeStamp)
futureCellValue = 0
if (currentRowCount + numberOfFutureRows < len(dataFile.matrix)):
codeString = 'float(dataFile.matrix[currentRowCount+numberOfFutureRows][colNumberOfData.' + args.c + '])'
futureCellValue = eval(codeString)
queueOfCellValueInFutureNRows.append(futureCellValue)
divisor = numberOfFutureRows
else:
divisor = len(dataFile.matrix) - currentRowCount
totalOfFutureNRows += futureCellValue
totalOfFutureNRows -= queueOfCellValueInFutureNRows.popleft()
averageOfFutureRows = totalOfFutureNRows / float(divisor)
codeString = 'float(dataFile.matrix[currentRowCount][colNumberOfData.' + args.c + '])'
valueInCurrentRow = eval(codeString)
if (averageOfFutureRows > valueInCurrentRow):
attribute.aList[currentRowCount][1] = 1
else:
attribute.aList[currentRowCount][1] = 0
attribute.aList[currentRowCount][2] = totalOfFutureNRows
attribute.aList[currentRowCount][3] = valueInCurrentRow
currentRowCount = currentRowCount + 1
if (currentRowCount % 1000 == 0):
print "Processed row number " + str(currentRowCount)
lNameOfTarget = "tAverageOfCol" + args.c + "InFuture" + str(
args.n) + "RowsIsGreaterThanCol" + args.c + "InCurrentRow"
return ["TimeStamp", lNameOfTarget, "TotalOfFutureRows", "CurrentRowValue"]
def extractAttributeFromDataMatrix(args):
"""
In the next 100 order book entries is there a case where the bidP0 is > current askP0
"""
currentRowCount = 0
while currentRowCount < len(dataFile.matrix) - 101:
futureRowCount = 1
attribute.aList[currentRowCount][0] = common.getTimeStamp(
dataFile.matrix[currentRowCount], colNumberOfData.TimeStamp)
while (futureRowCount < 100):
if (float(
dataFile.matrix[currentRowCount + futureRowCount]
[colNumberOfData.BidP0]) < float(
dataFile.matrix[currentRowCount][colNumberOfData.AskP0])):
attribute.aList[currentRowCount][1] = 1
futureRowCount = futureRowCount + 1
currentRowCount = currentRowCount + 1
if (currentRowCount % 1000 == 0):
print "Processed row number " + str(currentRowCount)
lNameOfTarget = "tBidLesserThanAskNext" + str(args.n)
return ["TimeStamp", lNameOfTarget, "Zero1", "Zero2"]
def extractAttributeFromDataMatrix(args):
try:
args.n
except:
print "Since -n has not been specified I cannot proceed"
os._exit(-1)
try:
args.c
except:
print "Since -c has not been specified I cannot proceed"
os._exit(-1)
list_of_price_array = [ eval('colNumberOfData.'+args.c+'P0') , eval('colNumberOfData.'+args.c+'P1') ,\
eval('colNumberOfData.'+args.c+'P2') , eval('colNumberOfData.'+args.c+'P3') , eval('colNumberOfData.'+args.c+'P4') ]
list_of_qty_array = [ eval('colNumberOfData.'+args.c+'Q0') , eval('colNumberOfData.'+args.c+'Q1') ,\
eval('colNumberOfData.'+args.c+'Q2') , eval('colNumberOfData.'+args.c+'Q3') , eval('colNumberOfData.'+args.c+'Q4') ]
currentRowCount = 0
qtyForCalculatingWeightedAverage = float(args.n)
levelOfDataAvailable = 4
for dataRow in dataFile.matrix:
qSum = 0
totalPrice = 0
totalPriceAtThisLevel = 0
i = 0
while (i <= levelOfDataAvailable
and qSum < qtyForCalculatingWeightedAverage):
priceAtThisLevel = float(
dataFile.matrix[currentRowCount][list_of_price_array[i]])
qtyAtThisLevel = float(
dataFile.matrix[currentRowCount][list_of_qty_array[i]])
qSum += qtyAtThisLevel
if (qSum > qtyForCalculatingWeightedAverage):
qtyToUseAtThisLevel = qtyAtThisLevel - (
qSum - qtyForCalculatingWeightedAverage)
totalPriceAtThisLevel = qtyToUseAtThisLevel * priceAtThisLevel
else:
totalPriceAtThisLevel = qtyAtThisLevel * priceAtThisLevel
totalPrice += totalPriceAtThisLevel
i = i + 1
if (
qSum < qtyForCalculatingWeightedAverage
): # This implies that the current row does not have enough qty to fill our requirement.
qtyToUseAtThisLevel = qtyForCalculatingWeightedAverage - qSum
totalPriceAtThisLevel = qtyToUseAtThisLevel * priceAtThisLevel
totalPrice += totalPriceAtThisLevel
attribute.aList[currentRowCount][0] = common.getTimeStamp(
dataFile.matrix[currentRowCount], colNumberOfData.TimeStamp)
attribute.aList[currentRowCount][1] = float(
totalPrice) / qtyForCalculatingWeightedAverage
currentRowCount += 1
if (currentRowCount % 10000 == 0):
print "Processed row number " + str(currentRowCount)
lNameOfFeaturePrinted = "fWAPriceOfCol" + args.c + "InLast" + str(
args.n) + "Qty"
return ["TimeStamp", lNameOfFeaturePrinted, "Zero1", "Zero2"]
def extractAttributeFromDataMatrix(args):
try:
args.c
except:
print "Since -c has not been specified I cannot proceed"
os._exit()
if (args.cType == "synthetic"):
colNumberOfAttribute = colNumberOfData.SysFeature
else:
colNumberOfAttribute = eval("colNumberOfData." + args.c)
colNumberOfTimeStamp = colNumberOfData.TimeStamp
currentRowCount = 0
for dataRow in dataFile.matrix:
attribute.aList[currentRowCount][0] = common.getTimeStamp(
dataFile.matrix[currentRowCount], colNumberOfTimeStamp)
attribute.aList[currentRowCount][1] = float(
dataFile.matrix[currentRowCount][colNumberOfAttribute])
currentRowCount = currentRowCount + 1
if (currentRowCount % 1000 == 0):
print "Processed row number " + str(currentRowCount)
lNameOfFeaturePrinted = "fCol" + args.c + "InCurrentRow"
return ["TimeStamp", lNameOfFeaturePrinted, "Zero1", "Zero2"]
def extractAttributeFromDataMatrix(args):
if args.n == None:
N = 5
else:
N = int(args.n)
try:
args.c
except:
print "Since -c has not been specified I cannot proceed"
os._exit()
currentRowCount = 0
codeString = 'float(dataFile.matrix[currentRowCount][colNumberOfData.' + args.c + '])'
eParam = math.pow((.5), (1.0 / N))
for dataRow in dataFile.matrix:
cellValue = eval(codeString)
if currentRowCount == 0:
attribute.aList[currentRowCount][0] = common.getTimeStamp(
dataFile.matrix[currentRowCount], colNumberOfData.TimeStamp)
attribute.aList[currentRowCount][1] = cellValue
currentRowCount = currentRowCount + 1
continue
attribute.aList[currentRowCount][0] = common.getTimeStamp(
dataFile.matrix[currentRowCount], colNumberOfData.TimeStamp)
firstPart = float(attribute.aList[currentRowCount - 1][1]) * eParam
secondPart = (1 - eParam) * cellValue
attribute.aList[currentRowCount][1] = firstPart + secondPart
currentRowCount = currentRowCount + 1
if (currentRowCount % 10000 == 0):
print "Processed row number " + str(currentRowCount)
lNameOfFeaturePrinted = "fEMovingAverageOfCol" + args.c + "InLast" + str(
args.n) + "Secs"
return ["TimeStamp", lNameOfFeaturePrinted, "Zero1", "Zero2"]
def extractAttributeFromDataMatrix(args):
if args.n == None:
N = 5
else:
N = int(args.n)
try:
args.c
except:
print "Since -c has not been specified I cannot proceed"
os._exit()
currentRowCount = 0
codeString = 'float(dataFile.matrix[currentRowCount][colNumberOfData.'+ args.c + '])'
eParam = math.pow((.5),(1.0 / N))
for dataRow in dataFile.matrix:
cellValue = eval(codeString)
if currentRowCount == 0:
attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp)
attribute.aList[currentRowCount][1] = cellValue
currentRowCount = currentRowCount + 1
continue
attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp)
firstPart = float(attribute.aList[currentRowCount - 1][1]) * eParam
secondPart = (1- eParam) * cellValue
attribute.aList[currentRowCount][1] = firstPart + secondPart
currentRowCount = currentRowCount + 1
if (currentRowCount%10000==0):
print "Processed row number " + str(currentRowCount)
lNameOfFeaturePrinted = "fEMovingAverageOfCol" + args.c + "InLast" + str(args.n) + "Secs"
return ["TimeStamp",lNameOfFeaturePrinted,"Zero1","Zero2"]
def extractAttributeFromDataMatrix(args):
if args.n is None:
print "-n has not been specified"
os._exit(-1)
totalOfLastLTQQty = int(args.n)
colNumberOfLTP = colNumberOfData.LTP
colNumberOfTTQ = colNumberOfData.TTQ
prev_ttq = 0
ltq_ltp_list = []
sum_ltq = 0
prod_sum = 0
currentRowCount = 0
for dataRow in dataFile.matrix:
ttq, ltp = (int(dataRow[colNumberOfTTQ]),
float(dataRow[colNumberOfLTP]))
attribute.aList[currentRowCount][0] = common.getTimeStamp(
dataFile.matrix[currentRowCount], colNumberOfData.TimeStamp)
if ttq > prev_ttq:
ltq = ttq - prev_ttq
ltq_ltp_list.append([ltq, ltp])
prod_sum += ltq * ltp
sum_ltq += ltq
if sum_ltq > totalOfLastLTQQty:
index = 0
while sum_ltq <> totalOfLastLTQQty:
if ltq_ltp_list[index][0] > sum_ltq - totalOfLastLTQQty:
ltq_ltp_list[index][0] -= sum_ltq - totalOfLastLTQQty
prod_sum -= (sum_ltq - totalOfLastLTQQty
) * ltq_ltp_list[index][1]
sum_ltq = totalOfLastLTQQty
else:
sum_ltq -= ltq_ltp_list[index][0]
prod_sum -= (ltq_ltp_list[index][0] *
ltq_ltp_list[index][1])
ltq_ltp_list = ltq_ltp_list[1:]
wtd_ltp = prod_sum / (max(sum_ltq, 1))
prev_ttq = ttq
attribute.aList[currentRowCount][1] = max(wtd_ltp, 1)
currentRowCount = currentRowCount + 1
if (currentRowCount % 1000 == 0):
print "Processed row number " + str(currentRowCount)
lNameOfFeature = "fWALTPInLast" + str(args.n) + "Qty"
return ["TimeStamp", lNameOfFeature, "0", "1"]
def extractAttributeFromDataMatrix(args):
currentRowCount = 0
lAPL = [
colNumberOfData.AskP0, colNumberOfData.AskP1, colNumberOfData.AskP2,
colNumberOfData.AskP3, colNumberOfData.AskP4
]
lAQL = [
colNumberOfData.AskQ0, colNumberOfData.AskQ1, colNumberOfData.AskQ2,
colNumberOfData.AskQ3, colNumberOfData.AskQ4
]
lBPL = [
colNumberOfData.BidP0, colNumberOfData.BidP1, colNumberOfData.BidP2,
colNumberOfData.BidP3, colNumberOfData.BidP4
]
lBQL = [
colNumberOfData.BidQ0, colNumberOfData.BidQ1, colNumberOfData.BidQ2,
colNumberOfData.BidQ3, colNumberOfData.BidQ4
]
if args.n == None:
print "Since -n has not been specified I cannot proceed"
os._exit()
else:
N = int(args.n)
for dataRow in dataFile.matrix:
askp = []
askq = []
bidp = []
bidq = []
lPriceMulQtySum = 0
lQtySum = 0
for i in range(0, N):
askp.append(float(dataRow[lAPL[i]]))
askq.append(float(dataRow[lAQL[i]]))
bidp.append(float(dataRow[lBPL[i]]))
bidq.append(float(dataRow[lBQL[i]]))
for price, qty in zip(askp[:N] + bidp[:N], askq[:N] + bidq[:N]):
lPriceMulQtySum += float(price) * float(qty)
lQtySum += float(qty)
qAverage = lPriceMulQtySum / lQtySum
attribute.aList[currentRowCount][0] = common.getTimeStamp(
dataFile.matrix[currentRowCount], colNumberOfData.TimeStamp)
attribute.aList[currentRowCount][1] = qAverage
currentRowCount = currentRowCount + 1
if (currentRowCount % 10000 == 0):
print "Processed row number " + str(currentRowCount)
lNameOfFeaturePrinted = "fInverseWAInLast" + str(N) + "Levels"
return ["TimeStamp", lNameOfFeaturePrinted, "Zero1", "Zero2"]
def extractAttributeFromDataMatrix(args):
queueOfCellValueInLastNRows = deque()
totalOfLastNRows = 0.0
if args.n == None:
N = 5
else:
N = int(args.n)
try:
args.c
except:
print "Since -c has not been specified I cannot proceed"
os._exit()
currentRowCount = 0
if(args.cType == "synthetic"):
colNumberOfAttribute = colNumberOfData.SysFeature
else:
colNumberOfAttribute = eval("colNumberOfData."+ args.c )
colNumberOfTimeStamp = colNumberOfData.TimeStamp
for dataRow in dataFile.matrix:
currentTickValue = float(dataFile.matrix[currentRowCount][colNumberOfAttribute])
attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfTimeStamp,args.cType)
if currentTickValue >= UpperCutoff:
attribute.aList[currentRowCount][1] = UpperValue
elif currentTickValue <= LowerCutOff:
attribute.aList[currentRowCount][1] = LowerValue
else:
if currentRowCount == 0:
attribute.aList[currentRowCount][1] = MidValue
elif attribute.aList[currentRowCount-1][1] == LowerValue :
if currentTickValue <= MidCutOff:
attribute.aList[currentRowCount][1] = LowerValue
else:
attribute.aList[currentRowCount][1] = MidValue
elif attribute.aList[currentRowCount-1][1] == UpperValue:
if currentTickValue >= MidCutOff:
attribute.aList[currentRowCount][1] = UpperValue
else:
attribute.aList[currentRowCount][1] = MidValue
else:
attribute.aList[currentRowCount][1] = MidValue
currentRowCount = currentRowCount + 1
if (currentRowCount%10000==0):
print "Processed row number " + str(currentRowCount)
lNameOfFeaturePrinted = "fMovingAverageOfCol" + args.c + "InLast" + str(args.n) + "Rows"
return ["TimeStamp",lNameOfFeaturePrinted,"Zero1","Zero2"]
def extractAttributeFromDataMatrix(args):
try:
args.n
except:
print "Since -n has not been specified I cannot proceed"
os._exit(-1)
try:
args.c
except:
print "Since -c has not been specified I cannot proceed"
os._exit(-1)
list_of_price_array = [ eval('colNumberOfData.'+args.c+'P0') , eval('colNumberOfData.'+args.c+'P1') ,\
eval('colNumberOfData.'+args.c+'P2') , eval('colNumberOfData.'+args.c+'P3') , eval('colNumberOfData.'+args.c+'P4') ]
list_of_qty_array = [ eval('colNumberOfData.'+args.c+'Q0') , eval('colNumberOfData.'+args.c+'Q1') ,\
eval('colNumberOfData.'+args.c+'Q2') , eval('colNumberOfData.'+args.c+'Q3') , eval('colNumberOfData.'+args.c+'Q4') ]
currentRowCount = 0
qtyForCalculatingWeightedAverage = float(args.n)
levelOfDataAvailable = 4
for dataRow in dataFile.matrix:
qSum = 0
totalPrice = 0
totalPriceAtThisLevel = 0
i = 0
while(i <= levelOfDataAvailable and qSum < qtyForCalculatingWeightedAverage):
priceAtThisLevel = float(dataFile.matrix[currentRowCount][list_of_price_array[i]])
qtyAtThisLevel = float(dataFile.matrix[currentRowCount][list_of_qty_array[i]])
qSum += qtyAtThisLevel
if(qSum > qtyForCalculatingWeightedAverage):
qtyToUseAtThisLevel = qtyAtThisLevel - (qSum - qtyForCalculatingWeightedAverage)
totalPriceAtThisLevel = qtyToUseAtThisLevel * priceAtThisLevel
else:
totalPriceAtThisLevel = qtyAtThisLevel * priceAtThisLevel
totalPrice += totalPriceAtThisLevel
i = i + 1
if(qSum < qtyForCalculatingWeightedAverage): # This implies that the current row does not have enough qty to fill our requirement.
qtyToUseAtThisLevel = qtyForCalculatingWeightedAverage - qSum
totalPriceAtThisLevel = qtyToUseAtThisLevel * priceAtThisLevel
totalPrice += totalPriceAtThisLevel
attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp)
attribute.aList[currentRowCount][1] = float(totalPrice)/qtyForCalculatingWeightedAverage
currentRowCount += 1
if (currentRowCount%10000==0):
print "Processed row number " + str(currentRowCount)
lNameOfFeaturePrinted = "fWAPriceOfCol" + args.c + "InLast" + str(args.n) + "Qty"
return ["TimeStamp",lNameOfFeaturePrinted,"Zero1","Zero2"]
def extractAttributeFromDataMatrix(args):
currentRowCount = 0
list_of_array = [ eval('colNumberOfData.'+args.c+'0') , eval('colNumberOfData.'+args.c+'4') ]
for dataRow in dataFile.matrix:
qSum = float(dataRow[list_of_array[0]]) + float(dataRow[list_of_array[1]])
qAverage = qSum / 2
attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp)
attribute.aList[currentRowCount][1] = qAverage
currentRowCount = currentRowCount + 1
if (currentRowCount%10000==0):
print "Processed row number " + str(currentRowCount)
lNameOfFeaturePrinted = "fCol" + args.c + "InCurrentRowAvg"
return ["TimeStamp",lNameOfFeaturePrinted,"Zero1","Zero2"]
def extractAttributeFromDataMatrix(args):
currentRowCount = 0
list_of_array = [ eval('colNumberOfData.'+args.c+'1') , eval('colNumberOfData.'+args.c+'2') ,eval('colNumberOfData.'+args.c+'3') , eval('colNumberOfData.'+args.c+'4') , eval('colNumberOfData.'+args.c+'5') ]
for dataRow in dataFile.matrix:
qSum = float(dataRow[list_of_array[0]]) + float(dataRow[list_of_array[1]]) + \
float(dataRow[list_of_array[2]]) + float(dataRow[list_of_array[3]]) + float(dataRow[list_of_array[4]])
attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp)
attribute.aList[currentRowCount][1] = qSum
currentRowCount = currentRowCount + 1
if (currentRowCount%10000==0):
print "Processed row number " + str(currentRowCount)
lNameOfFeaturePrinted = "fCol" + args.c + "InCurrentRowSum"
return ["TimeStamp",lNameOfFeaturePrinted,"Zero1","Zero2"]
def extractAttributeFromDataMatrix(args):
if args.n is None:
print "-n has not been specified"
os._exit(-1)
totalOfLastLTQQty = int(args.n)
colNumberOfLTP = colNumberOfData.LTP
colNumberOfTTQ = colNumberOfData.TTQ
prev_ttq = 0
ltq_ltp_list = []
sum_ltq = 0
prod_sum = 0
currentRowCount = 0
for dataRow in dataFile.matrix:
ttq, ltp = (int(dataRow[colNumberOfTTQ]), float(dataRow[colNumberOfLTP]))
attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp)
if ttq > prev_ttq:
ltq = ttq - prev_ttq
ltq_ltp_list.append([ltq, ltp])
prod_sum += ltq * ltp
sum_ltq += ltq
if sum_ltq > totalOfLastLTQQty:
index = 0
while sum_ltq <> totalOfLastLTQQty:
if ltq_ltp_list[index][0] > sum_ltq - totalOfLastLTQQty:
ltq_ltp_list[index][0] -= sum_ltq - totalOfLastLTQQty
prod_sum -= (sum_ltq - totalOfLastLTQQty) * ltq_ltp_list[index][1]
sum_ltq = totalOfLastLTQQty
else:
sum_ltq -= ltq_ltp_list[index][0]
prod_sum -= (ltq_ltp_list[index][0] * ltq_ltp_list[index][1])
ltq_ltp_list = ltq_ltp_list[1:]
wtd_ltp = prod_sum / (max(sum_ltq,1))
prev_ttq = ttq
attribute.aList[currentRowCount][1] = max(wtd_ltp, 1)
currentRowCount = currentRowCount + 1
if(currentRowCount % 1000 == 0):
print "Processed row number " + str(currentRowCount)
lNameOfFeature = "fWALTPInLast" + str(args.n) + "Qty"
return ["TimeStamp",lNameOfFeature,"0", "1"]
def extractAttributeFromDataMatrix(args):
"""
In the next 100 order book entries is there a case where the bidP0 is > current askP0
"""
currentRowCount = 0
while currentRowCount < len(dataFile.matrix)-101:
futureRowCount = 1
attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp)
while(futureRowCount < 100):
if( float(dataFile.matrix[currentRowCount+futureRowCount][colNumberOfData.BidP0]) >= float(dataFile.matrix[currentRowCount][colNumberOfData.AskP0]) ):
attribute.aList[currentRowCount][1] = 1
futureRowCount = futureRowCount + 1
currentRowCount = currentRowCount + 1
if(currentRowCount % 1000 == 0):
print "Processed row number " + str(currentRowCount)
lNameOfTarget = "tBidEqualsOrGreaterThanAskInNext" + str(args.n)
return ["TimeStamp",lNameOfTarget,"Zero1","Zero2"]
def extractAttributeFromDataMatrix(args):
currentRowCount = 0
list_of_price_array = [ eval('colNumberOfData.'+args.c+'P0') , eval('colNumberOfData.'+args.c+'P1') ,\
eval('colNumberOfData.'+args.c+'P2') , eval('colNumberOfData.'+args.c+'P3') , eval('colNumberOfData.'+args.c+'P4') ]
list_of_qty_array = [ eval('colNumberOfData.'+args.c+'Q0') , eval('colNumberOfData.'+args.c+'Q1') ,\
eval('colNumberOfData.'+args.c+'Q2') , eval('colNumberOfData.'+args.c+'Q3') , eval('colNumberOfData.'+args.c+'Q4') ]
for dataRow in dataFile.matrix:
qPriceSum = 0.0
qQtySum = 0
for i in range(5):
qPriceSum += int(dataRow[list_of_qty_array[i]]) * float(dataRow[list_of_price_array[i]])
qQtySum += int(dataRow[list_of_qty_array[i]])
qAverage = qPriceSum / qQtySum
attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp)
attribute.aList[currentRowCount][1] = qAverage
currentRowCount = currentRowCount + 1
if (currentRowCount%10000==0):
print "Processed row number " + str(currentRowCount)
lNameOfFeaturePrinted = "fCol" + args.c + "InCurrentRowAvg"
return ["TimeStamp",lNameOfFeaturePrinted,"Zero1","Zero2"]
def extractAttributeFromDataMatrix(args):
currentRowCount = 0
lAPL = [ colNumberOfData.AskP0 , colNumberOfData.AskP1 , colNumberOfData.AskP2 , colNumberOfData.AskP3 , colNumberOfData.AskP4 ]
lAQL = [ colNumberOfData.AskQ0 , colNumberOfData.AskQ1 , colNumberOfData.AskQ2 , colNumberOfData.AskQ3 , colNumberOfData.AskQ4 ]
lBPL = [ colNumberOfData.BidP0 , colNumberOfData.BidP1 , colNumberOfData.BidP2 , colNumberOfData.BidP3 , colNumberOfData.BidP4 ]
lBQL = [ colNumberOfData.BidQ0 , colNumberOfData.BidQ1 , colNumberOfData.BidQ2 , colNumberOfData.BidQ3 , colNumberOfData.BidQ4 ]
if args.n == None:
print "Since -n has not been specified I cannot proceed"
os._exit()
else:
N = int(args.n)
for dataRow in dataFile.matrix:
askp = []
askq = []
bidp = []
bidq = []
lPriceMulQtySum = 0
lQtySum = 0
for i in range(0,N):
askp.append(float(dataRow[lAPL[i]]))
askq.append(float(dataRow[lAQL[i]]))
bidp.append(float(dataRow[lBPL[i]]))
bidq.append(float(dataRow[lBQL[i]]))
for price, qty in zip(askp[:N] + bidp[:N], askq[:N] + bidq[:N]):
lPriceMulQtySum += float(price) * float(qty)
lQtySum += float(qty)
qAverage = lPriceMulQtySum / lQtySum
attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp)
attribute.aList[currentRowCount][1] = qAverage
currentRowCount = currentRowCount + 1
if (currentRowCount%10000==0):
print "Processed row number " + str(currentRowCount)
lNameOfFeaturePrinted = "fInverseWAInLast" + str(N) + "Levels"
return ["TimeStamp",lNameOfFeaturePrinted,"Zero1","Zero2"]
def extractAttributeFromDataMatrix(args):
try:
args.c
except:
print "Since -c has not been specified I cannot proceed"
os._exit()
if(args.cType == "synthetic"):
colNumberOfAttribute = colNumberOfData.SysFeature
else:
colNumberOfAttribute = eval("colNumberOfData."+ args.c )
colNumberOfTimeStamp = colNumberOfData.TimeStamp
currentRowCount = 0
for dataRow in dataFile.matrix:
attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfTimeStamp)
attribute.aList[currentRowCount][1] = float(dataFile.matrix[currentRowCount][colNumberOfAttribute])
currentRowCount = currentRowCount + 1
if(currentRowCount % 1000 == 0):
print "Processed row number " + str(currentRowCount)
lNameOfFeaturePrinted = "fCol" + args.c + "InCurrentRow"
return ["TimeStamp",lNameOfFeaturePrinted,"Zero1","Zero2"]
def extractAttributeFromDataMatrix(args):
queueOfCellValueInFutureNRows = deque()
totalOfFutureNRows = 0.0
if args.n is None:
print "-n has not been specified"
os._exit(-1)
numberOfFutureRows = int(args.n)
try:
args.c
except:
print "-c has not been specified"
os._exit(-1)
""" lets get the total of futureNrows"""
futureRowCount = 0
while(futureRowCount < numberOfFutureRows):
codeString = 'float(dataFile.matrix[futureRowCount][colNumberOfData.'+args.c+'])'
cellValue = eval(codeString)
queueOfCellValueInFutureNRows.append(cellValue)
totalOfFutureNRows += cellValue
futureRowCount = futureRowCount + 1
currentRowCount = 0
for dataRow in dataFile.matrix:
attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp)
futureCellValue = 0
if(currentRowCount + numberOfFutureRows < len(dataFile.matrix)):
codeString = 'float(dataFile.matrix[currentRowCount+numberOfFutureRows][colNumberOfData.'+args.c+'])'
futureCellValue = eval(codeString)
queueOfCellValueInFutureNRows.append(futureCellValue)
divisor = numberOfFutureRows
else:
divisor = len(dataFile.matrix) - currentRowCount
totalOfFutureNRows += futureCellValue
totalOfFutureNRows -= queueOfCellValueInFutureNRows.popleft()
averageOfFutureRows = totalOfFutureNRows / float(divisor)
codeString = 'float(dataFile.matrix[currentRowCount][colNumberOfData.'+args.c+'])'
valueInCurrentRow = eval(codeString)
if( averageOfFutureRows > valueInCurrentRow):
attribute.aList[currentRowCount][1] = 1
else:
attribute.aList[currentRowCount][1] = 0
attribute.aList[currentRowCount][2] = totalOfFutureNRows
attribute.aList[currentRowCount][3] = valueInCurrentRow
currentRowCount = currentRowCount + 1
if(currentRowCount % 1000 == 0):
print "Processed row number " + str(currentRowCount)
lNameOfTarget = "tAverageOfCol" + args.c + "InFuture" + str(args.n) + "RowsIsGreaterThanCol" + args.c + "InCurrentRow"
return ["TimeStamp",lNameOfTarget,"TotalOfFutureRows","CurrentRowValue"]
def extractAttributeFromDataMatrix(args):
if args.n is None:
print "N has not been specified"
os._exit(-1)
try:
args.c
colNumberWeAreWorkingWith = eval('colNumberOfData.' + args.c)
except:
print "C has not been specified"
os._exit(-1)
numberOfSecondsInFuture = int(args.n)
currentRowCount = 0
totalNumberOfDataRows = len(dataFile.matrix)
futureRowCount = 1 # Since we have a queue we do not need to initialize everytime inside the for loop
totalOfRowsInFutureNSeconds = 0.0
queue = deque()
for dataRow in dataFile.matrix:
timeOfCurrentRow = common.getTimeStamp(
dataFile.matrix[currentRowCount], colNumberOfData.TimeStamp)
# lets remove all values from queue which have time <= timeOfCurrentRow
if (len(queue) > 0
): # When currentRowCount is 0 then there is nothing in the queue
oldestElementinQueue = queue.popleft()
while (oldestElementinQueue[0] < timeOfCurrentRow):
oldestElementinQueue = queue.popleft()
totalOfRowsInFutureNSeconds -= oldestElementinQueue[1]
attribute.aList[currentRowCount][0] = timeOfCurrentRow
while True:
if (
currentRowCount + futureRowCount < totalNumberOfDataRows
): # If the totalNumberOfDataRows is 5 we can access index values 0 to 4.
timeOfFutureRow = common.getTimeStamp(
dataFile.matrix[currentRowCount + futureRowCount])
if (timeOfFutureRow - timeOfCurrentRow <
numberOfSecondsInFuture):
cellValue = float(dataFile.matrix[currentRowCount +
futureRowCount]
[colNumberWeAreWorkingWith])
totalOfRowsInFutureNSeconds += cellValue
futureRowCount = futureRowCount + 1
# add the data to queue for future needs
queue.append([timeOfFutureRow, cellValue])
else:
break
else:
break
averageOfFutureRows = totalOfRowsInFutureNSeconds / futureRowCount
currentCellValue = float(
dataFile.matrix[currentRowCount][colNumberWeAreWorkingWith])
if (averageOfFutureRows > currentCellValue):
attribute.aList[currentRowCount][1] = 1
else:
attribute.aList[currentRowCount][1] = 0
attribute.aList[currentRowCount][2] = totalOfRowsInFutureNSeconds
attribute.aList[currentRowCount][3] = float(
dataFile.matrix[currentRowCount][colNumberOfData.LTP])
currentRowCount = currentRowCount + 1
if (currentRowCount % 1 == 0):
print "Processed row number " + str(currentRowCount)
lNameOfTarget = "tAverageOfCol" + args.c + "InFuture" + str(
args.n) + "SecsIsGreaterThanCol" + args.c + "InCurrentRow"
return [
"TimeStamp", lNameOfTarget, "TotalOfFutureNSecs", "CurrentRowLTPValue"
]
def extractAttributeFromDataMatrix(args):
queueOfCellValueInFutureNLTQs = deque()
totalOfFutureNRows = 0.0
if args.n is None:
print "-n has not been specified"
os._exit(-1)
totalOfFutureLTQQty = int(args.n)
try:
args.c
except:
print "-c has not been specified"
os._exit(-1)
try:
args.m
except:
print "-m has not been specified"
os._exit(-1)
colNumberOfAttribute = eval("colNumberOfData."+ args.c )
print "attribute column number " , colNumberOfAttribute , args.c
print args.m
lDiffPip = float(args.m)
lPipSize = int(args.tickSize)
""" lets get the total of futureNrows"""
futureLTQSum = 0
currentRowIndex = 0
lLastTradeIndex = 0
lNoMoreTradesFound = 0
currentLTPValue = 0.0
currentLTQValue = 0
WeightedLTPSum = 0.0
totalLTPQty = totalOfFutureLTQQty
while(futureLTQSum < totalOfFutureLTQQty):
if dataFile.matrix[currentRowIndex][colNumberOfData.MsgCode].upper() == "T":
currentLTPValue = float(dataFile.matrix[currentRowIndex][colNumberOfData.LTP])
currentLTQValue = int(dataFile.matrix[currentRowIndex][colNumberOfData.NewQ])
queueOfCellValueInFutureNLTQs.append( [ (currentLTPValue * currentLTQValue) , currentLTQValue , currentLTPValue , 0 ] )
futureLTQSum = futureLTQSum + currentLTQValue
WeightedLTPSum = WeightedLTPSum + queueOfCellValueInFutureNLTQs[-1][0]
lLastTradeIndex = currentRowIndex
currentRowIndex = currentRowIndex + 1
if futureLTQSum > totalOfFutureLTQQty :
currentLTQValueToBeUsed = currentLTQValue - ( futureLTQSum - totalOfFutureLTQQty )
currentTickPriceToBeAdded = currentLTPValue * currentLTQValueToBeUsed
LTQValueNotAdded = ( futureLTQSum - totalOfFutureLTQQty )
WeightedLTPSum = WeightedLTPSum - queueOfCellValueInFutureNLTQs[-1][0] + currentTickPriceToBeAdded
queueOfCellValueInFutureNLTQs[-1] = [ currentTickPriceToBeAdded , currentLTQValueToBeUsed , currentLTPValue , LTQValueNotAdded ]
currentRowCount = 0
for dataRow in dataFile.matrix:
sum = 0
currentPrice = float(dataFile.matrix[currentRowCount][colNumberOfAttribute])
currentLTP = float(dataFile.matrix[currentRowCount][colNumberOfData.LTP])
currentMsgCode = dataFile.matrix[currentRowCount][colNumberOfData.MsgCode]
currentLTQ = int(dataFile.matrix[currentRowCount][colNumberOfData.NewQ])
# try:
attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp)
# except:
# print colNumberOfData.TimeStamp,currentRowCount,currentMsgCode
# import pdb
# pdb.set_trace()
if (dataFile.matrix[currentRowCount][colNumberOfData.MsgCode].upper() == "T") :
lZerothElementOfQueue = queueOfCellValueInFutureNLTQs.popleft()
WeightedLTPSum = WeightedLTPSum - lZerothElementOfQueue[0]
LTQSubtracted = lZerothElementOfQueue[1]
if len(queueOfCellValueInFutureNLTQs) != 0:
LTQWhichCanBeAddedWithoutNewTradeMessage = queueOfCellValueInFutureNLTQs[-1][3]
LTQAddedWithoutNewTradeMessage = min( LTQSubtracted , LTQWhichCanBeAddedWithoutNewTradeMessage)
currentTickPriceToBeAdded = ( LTQAddedWithoutNewTradeMessage * queueOfCellValueInFutureNLTQs[-1][2] ) + queueOfCellValueInFutureNLTQs[-1][0]
LTQValueNotAddedInLastElementOfQueue = queueOfCellValueInFutureNLTQs[-1][3] - LTQAddedWithoutNewTradeMessage
WeightedLTPSum = WeightedLTPSum - queueOfCellValueInFutureNLTQs[-1][0] + currentTickPriceToBeAdded
queueOfCellValueInFutureNLTQs[-1][0] = currentTickPriceToBeAdded
queueOfCellValueInFutureNLTQs[-1][1] = queueOfCellValueInFutureNLTQs[-1][1] + LTQAddedWithoutNewTradeMessage
queueOfCellValueInFutureNLTQs[-1][3] = LTQValueNotAddedInLastElementOfQueue
else:
LTQAddedWithoutNewTradeMessage = 0
if LTQSubtracted > LTQAddedWithoutNewTradeMessage:
LTQToBeAddedFromNewTrade = LTQSubtracted - LTQAddedWithoutNewTradeMessage
if (lNoMoreTradesFound == 0):
startIndexToFindNextTrade = lLastTradeIndex + 1
while( startIndexToFindNextTrade < len(dataFile.matrix)):
if dataFile.matrix[startIndexToFindNextTrade][colNumberOfData.MsgCode].upper() == "T":
lLastTradeIndex = startIndexToFindNextTrade
futureLTPValue = float(dataFile.matrix[startIndexToFindNextTrade][colNumberOfData.LTP])
futureLTQValue = int(dataFile.matrix[startIndexToFindNextTrade][colNumberOfData.NewQ])
queueOfCellValueInFutureNLTQs.append( [ (futureLTPValue * futureLTQValue) , futureLTQValue , futureLTPValue , 0 ] )
LTQToBeAddedFromNewTrade = LTQToBeAddedFromNewTrade - futureLTQValue
WeightedLTPSum = WeightedLTPSum + queueOfCellValueInFutureNLTQs[-1][0]
if LTQToBeAddedFromNewTrade <= 0 :
if LTQToBeAddedFromNewTrade < 0 :
LTQValueToBeUsed = futureLTQValue + LTQToBeAddedFromNewTrade
currentTickPriceToBeAdded = LTQValueToBeUsed * futureLTPValue
LTQValueNotAdded = -1 * LTQToBeAddedFromNewTrade
WeightedLTPSum = WeightedLTPSum - queueOfCellValueInFutureNLTQs[-1][0] + currentTickPriceToBeAdded
queueOfCellValueInFutureNLTQs[-1] = [ currentTickPriceToBeAdded , LTQValueToBeUsed , futureLTPValue , LTQValueNotAdded ]
break
startIndexToFindNextTrade = startIndexToFindNextTrade + 1
if startIndexToFindNextTrade == len(dataFile.matrix):
if LTQToBeAddedFromNewTrade > 0 :
totalLTPQty = totalLTPQty - LTQToBeAddedFromNewTrade
lNoMoreTradesFound = 1
else:
totalLTPQty = totalLTPQty - LTQToBeAddedFromNewTrade
for ele in queueOfCellValueInFutureNLTQs:
sum = sum + (ele[1]*ele[2])
if (ele[1]*ele[2] != ele[0]):
print "Error in Code"
os.exit(-1)
if sum != 0 and sum != WeightedLTPSum :
print "Error in Code"
os.exit(-1)
if totalLTPQty != 0:
WALTPOfFutureNQty = WeightedLTPSum / totalLTPQty
if lPipSize== 25000:
tTranCost = 0.000015 * 2
else:
tTranCost = 0.00015 * 2
l_margin_ticks = ceil((lDiffPip*tTranCost*currentPrice)/lPipSize)
if args.c == "BidP0" or args.c == "BestBidP":
if WALTPOfFutureNQty >= (currentPrice + l_margin_ticks* lPipSize):
lClassOfTargetVariable = 1
else:
lClassOfTargetVariable = 0
else:
if WALTPOfFutureNQty <= (currentPrice - l_margin_ticks*lPipSize):
lClassOfTargetVariable = 1
else:
lClassOfTargetVariable = 0
attribute.aList[currentRowCount][1] = lClassOfTargetVariable
attribute.aList[currentRowCount][2] = currentPrice
attribute.aList[currentRowCount][3] = ";".join([ str(WALTPOfFutureNQty) , str(currentLTP) , str(currentLTQ) ,\
str(currentMsgCode) , str(WeightedLTPSum) , str(totalLTPQty)
])
currentRowCount = currentRowCount + 1
if(currentRowCount % 1000 == 0):
print "Processed row number " + str(currentRowCount)
lNameOfTarget = "tWALTPComparedToCol" + str(args.c) + "InFuture" + str(args.n) + "Qty"
return ["TimeStamp",lNameOfTarget,args.c,"WALTPWithWtAsLTQForFuture"+args.n+"LTQQty","CurrentLTP","CurrentLTQ","CurrentMsgCode",\
"WeightedLTPSum" , "totalLTPQtys"]
def extractAttributeFromDataMatrix(args):
queueOfCellValueInFutureNLTQs = deque()
totalOfFutureNRows = 0.0
if args.n is None:
print "-n has not been specified"
os._exit(-1)
totalOfFutureLTQQty = int(args.n)
try:
args.c
except:
print "-c has not been specified"
os._exit(-1)
colNumberOfAttribute = eval("colNumberOfData." + args.c)
lPipSize = int(args.tickSize)
""" lets get the total of futureNrows"""
futureLTQSum = 0
currentRowIndex = 0
lLastTradeIndex = 0
lNoMoreTradesFound = 0
currentLTPValue = 0.0
currentLTQValue = 0
WeightedLTPSum = 0.0
totalLTPQty = totalOfFutureLTQQty
while (futureLTQSum < totalOfFutureLTQQty):
try:
dataFile.matrix[currentRowIndex][
colNumberOfData.MsgCode].upper() == "T"
except:
print "Data Line = ", dataFile.matrix[currentRowIndex]
print "Current Row Index = ", currentRowIndex
if dataFile.matrix[currentRowIndex][
colNumberOfData.MsgCode].upper() == "T":
currentLTPValue = float(
dataFile.matrix[currentRowIndex][colNumberOfData.LTP])
currentLTQValue = int(
dataFile.matrix[currentRowIndex][colNumberOfData.NewQ])
queueOfCellValueInFutureNLTQs.append([
(currentLTPValue * currentLTQValue), currentLTQValue,
currentLTPValue, 0
])
futureLTQSum = futureLTQSum + currentLTQValue
WeightedLTPSum = WeightedLTPSum + queueOfCellValueInFutureNLTQs[
-1][0]
lLastTradeIndex = currentRowIndex
currentRowIndex = currentRowIndex + 1
if futureLTQSum > totalOfFutureLTQQty:
currentLTQValueToBeUsed = currentLTQValue - (futureLTQSum -
totalOfFutureLTQQty)
currentTickPriceToBeAdded = currentLTPValue * currentLTQValueToBeUsed
LTQValueNotAdded = (futureLTQSum - totalOfFutureLTQQty)
WeightedLTPSum = WeightedLTPSum - queueOfCellValueInFutureNLTQs[-1][
0] + currentTickPriceToBeAdded
queueOfCellValueInFutureNLTQs[-1] = [
currentTickPriceToBeAdded, currentLTQValueToBeUsed,
currentLTPValue, LTQValueNotAdded
]
currentRowCount = 0
for dataRow in dataFile.matrix:
sum = 0
currentPrice = float(
dataFile.matrix[currentRowCount][colNumberOfAttribute])
currentLTP = float(
dataFile.matrix[currentRowCount][colNumberOfData.LTP])
currentMsgCode = dataFile.matrix[currentRowCount][
colNumberOfData.MsgCode]
currentLTQ = int(
dataFile.matrix[currentRowCount][colNumberOfData.NewQ])
attribute.aList[currentRowCount][0] = common.getTimeStamp(
dataFile.matrix[currentRowCount], colNumberOfData.TimeStamp)
if (dataFile.matrix[currentRowCount][colNumberOfData.MsgCode].upper()
== "T"):
lZerothElementOfQueue = queueOfCellValueInFutureNLTQs.popleft()
WeightedLTPSum = WeightedLTPSum - lZerothElementOfQueue[0]
LTQSubtracted = lZerothElementOfQueue[1]
if len(queueOfCellValueInFutureNLTQs) != 0:
LTQWhichCanBeAddedWithoutNewTradeMessage = queueOfCellValueInFutureNLTQs[
-1][3]
LTQAddedWithoutNewTradeMessage = min(
LTQSubtracted, LTQWhichCanBeAddedWithoutNewTradeMessage)
currentTickPriceToBeAdded = (
LTQAddedWithoutNewTradeMessage *
queueOfCellValueInFutureNLTQs[-1][2]
) + queueOfCellValueInFutureNLTQs[-1][0]
LTQValueNotAddedInLastElementOfQueue = queueOfCellValueInFutureNLTQs[
-1][3] - LTQAddedWithoutNewTradeMessage
WeightedLTPSum = WeightedLTPSum - queueOfCellValueInFutureNLTQs[
-1][0] + currentTickPriceToBeAdded
queueOfCellValueInFutureNLTQs[-1][
0] = currentTickPriceToBeAdded
queueOfCellValueInFutureNLTQs[-1][
1] = queueOfCellValueInFutureNLTQs[-1][
1] + LTQAddedWithoutNewTradeMessage
queueOfCellValueInFutureNLTQs[-1][
3] = LTQValueNotAddedInLastElementOfQueue
else:
LTQAddedWithoutNewTradeMessage = 0
if LTQSubtracted > LTQAddedWithoutNewTradeMessage:
LTQToBeAddedFromNewTrade = LTQSubtracted - LTQAddedWithoutNewTradeMessage
if (lNoMoreTradesFound == 0):
startIndexToFindNextTrade = lLastTradeIndex + 1
while (startIndexToFindNextTrade < len(dataFile.matrix)):
if dataFile.matrix[startIndexToFindNextTrade][
colNumberOfData.MsgCode].upper() == "T":
lLastTradeIndex = startIndexToFindNextTrade
futureLTPValue = float(
dataFile.matrix[startIndexToFindNextTrade][
colNumberOfData.LTP])
futureLTQValue = int(
dataFile.matrix[startIndexToFindNextTrade][
colNumberOfData.NewQ])
queueOfCellValueInFutureNLTQs.append([
(futureLTPValue * futureLTQValue),
futureLTQValue, futureLTPValue, 0
])
LTQToBeAddedFromNewTrade = LTQToBeAddedFromNewTrade - futureLTQValue
WeightedLTPSum = WeightedLTPSum + queueOfCellValueInFutureNLTQs[
-1][0]
if LTQToBeAddedFromNewTrade <= 0:
if LTQToBeAddedFromNewTrade < 0:
LTQValueToBeUsed = futureLTQValue + LTQToBeAddedFromNewTrade
currentTickPriceToBeAdded = LTQValueToBeUsed * futureLTPValue
LTQValueNotAdded = -1 * LTQToBeAddedFromNewTrade
WeightedLTPSum = WeightedLTPSum - queueOfCellValueInFutureNLTQs[
-1][0] + currentTickPriceToBeAdded
queueOfCellValueInFutureNLTQs[-1] = [
currentTickPriceToBeAdded,
LTQValueToBeUsed, futureLTPValue,
LTQValueNotAdded
]
break
startIndexToFindNextTrade = startIndexToFindNextTrade + 1
if startIndexToFindNextTrade == len(dataFile.matrix):
if LTQToBeAddedFromNewTrade > 0:
totalLTPQty = totalLTPQty - LTQToBeAddedFromNewTrade
lNoMoreTradesFound = 1
else:
totalLTPQty = totalLTPQty - LTQToBeAddedFromNewTrade
for ele in queueOfCellValueInFutureNLTQs:
sum = sum + (ele[1] * ele[2])
if (ele[1] * ele[2] != ele[0]):
print "Error in Code"
os.exit(-1)
if sum != 0 and sum != WeightedLTPSum:
print "Error in Code"
os.exit(-1)
if totalLTPQty != 0:
WALTPOfFutureNQty = WeightedLTPSum / totalLTPQty
if args.c == "BidP0":
if WALTPOfFutureNQty >= (currentPrice + lPipSize):
lClassOfTargetVariable = 1
else:
lClassOfTargetVariable = 0
else:
if WALTPOfFutureNQty <= (currentPrice - lPipSize):
lClassOfTargetVariable = 1
else:
lClassOfTargetVariable = 0
attribute.aList[currentRowCount][1] = lClassOfTargetVariable
attribute.aList[currentRowCount][2] = currentPrice
attribute.aList[currentRowCount][3] = ";".join([ str(WALTPOfFutureNQty) , str(currentLTP) , str(currentLTQ) ,\
str(currentMsgCode) , str(WeightedLTPSum) , str(totalLTPQty)
])
currentRowCount = currentRowCount + 1
if (currentRowCount % 1000 == 0):
print "Processed row number " + str(currentRowCount)
lNameOfTarget = "tWALTPComparedToCol" + str(args.c) + "InFuture" + str(
args.n) + "Qty"
return ["TimeStamp",lNameOfTarget,args.c,"WALTPWithWtAsLTQForFuture"+args.n+"LTQQty","CurrentLTP","CurrentLTQ","CurrentMsgCode",\
"WeightedLTPSum" , "totalLTPQtys"]
def extractAttributeFromDataMatrix(args):
queueOfCellValueInFutureNRows = deque()
totalOfFutureNRows = 0.0
if args.n is None:
print "-n has not been specified"
os._exit(-1)
numberOfFutureTrades = int(args.n)
lPipSize = int(args.tickSize)
lMargin1 = ( 1 * lPipSize )
lMargin2 = lMargin1 + ( 2 * lPipSize )
""" lets get the total of futureNrows"""
futureTradesCount = 0
currentRowIndex = 0
lLastTradeIndex = 0
lNoMoreTradesFound = 0
while(futureTradesCount < numberOfFutureTrades):
if dataFile.matrix[currentRowIndex][colNumberOfData.MsgCode].upper() == "T":
cellValue = float(dataFile.matrix[currentRowIndex][colNumberOfData.LTP])
queueOfCellValueInFutureNRows.append(cellValue)
futureTradesCount = futureTradesCount + 1
lLastTradeIndex = currentRowIndex
currentRowIndex = currentRowIndex + 1
currentRowCount = 0
for dataRow in dataFile.matrix:
attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp)
if (dataFile.matrix[currentRowCount][colNumberOfData.MsgCode].upper() == "T") :
queueOfCellValueInFutureNRows.popleft()
if (lNoMoreTradesFound == 0):
startIndexToFindNextTrade = lLastTradeIndex + 1
while( startIndexToFindNextTrade < len(dataFile.matrix)):
if dataFile.matrix[startIndexToFindNextTrade][colNumberOfData.MsgCode].upper() == "T":
futureCellValue = float(dataFile.matrix[startIndexToFindNextTrade][colNumberOfData.LTP])
queueOfCellValueInFutureNRows.append(futureCellValue)
lLastTradeIndex = startIndexToFindNextTrade
break
startIndexToFindNextTrade = startIndexToFindNextTrade + 1
if startIndexToFindNextTrade == len(dataFile.matrix) :
lNoMoreTradesFound = 1
currentBidP0 = float(dataFile.matrix[currentRowCount][colNumberOfData.BidP0])
currentAskP0 = float(dataFile.matrix[currentRowCount][colNumberOfData.AskP0])
currentLTP = float(dataFile.matrix[currentRowCount][colNumberOfData.LTP])
currentMsgCode = dataFile.matrix[currentRowCount][colNumberOfData.MsgCode]
lClassOfTargetVariable = []
valueInCurrentRow = 0
for LTP in queueOfCellValueInFutureNRows:
lClassifiedInOneClass = 0
if (LTP - (currentBidP0 + lPipSize)) > lMargin1 :
if (LTP - (currentBidP0 + lPipSize)) > lMargin2:
lClassOfTargetVariable.append(2)
lClassifiedInOneClass = 1
else:
lClassOfTargetVariable.append(1)
lClassifiedInOneClass = 1
if ((currentAskP0 - lPipSize) - LTP ) > lMargin1 :
if lClassifiedInOneClass == 1 :
lClassOfTargetVariable[-1] = 0
elif ((currentAskP0 - lPipSize) - LTP ) > lMargin2 :
lClassOfTargetVariable.append(-2)
lClassifiedInOneClass = 1
else:
lClassOfTargetVariable.append(-1)
lClassifiedInOneClass = 1
if lClassifiedInOneClass == 0:
lClassOfTargetVariable.append(0)
if lClassOfTargetVariable[-1]==2 or lClassOfTargetVariable[-1]==-2 :
break
if len(queueOfCellValueInFutureNRows) == 0 :
attribute.aList[currentRowCount][1] = 0
attribute.aList[currentRowCount][2] = currentBidP0
attribute.aList[currentRowCount][3] = ";".join([ str(currentAskP0) , str(currentLTP) , str(currentMsgCode) , "0" ,"0", "0" , str(LTP)])
else:
lNumberTimesMarketIndicatedBuy = 0
lNumberTimesMarketIndicatedSell = 0
lNumberTimesMarketIndicatedSame = 0
if (lClassOfTargetVariable[-1] == 2) or (lClassOfTargetVariable[-1] == -2):
attribute.aList[currentRowCount][1] = lClassOfTargetVariable[-1]
else:
lNumberTimesMarketIndicatedBuy = lClassOfTargetVariable.count(1)
lNumberTimesMarketIndicatedSell = lClassOfTargetVariable.count(-1)
lNumberTimesMarketIndicatedSame = lClassOfTargetVariable.count(0)
if ( lNumberTimesMarketIndicatedBuy > lNumberTimesMarketIndicatedSell ) and (lNumberTimesMarketIndicatedBuy > lNumberTimesMarketIndicatedSame):
attribute.aList[currentRowCount][1] = 1
elif ( lNumberTimesMarketIndicatedSell > lNumberTimesMarketIndicatedBuy ) and (lNumberTimesMarketIndicatedSell > lNumberTimesMarketIndicatedSame):
attribute.aList[currentRowCount][1] = -1
else:
attribute.aList[currentRowCount][1] = 0
attribute.aList[currentRowCount][2] = currentBidP0
attribute.aList[currentRowCount][3] = ";".join([ str(currentAskP0) ,str(currentLTP) , str(currentMsgCode) , str(lNumberTimesMarketIndicatedBuy) , str(lNumberTimesMarketIndicatedSell) , str(lNumberTimesMarketIndicatedSame) , str(queueOfCellValueInFutureNRows[-1]) ])
currentRowCount = currentRowCount + 1
if(currentRowCount % 1000 == 0):
print "Processed row number " + str(currentRowCount)
lNameOfTarget = "tClassOfTargetVariableInFuture"+ str(args.n) + "TradeTicks"
return ["TimeStamp",lNameOfTarget,"CurrentBid","CurrentAsk","CurrentLTP","CurrentMsgCode",\
"NumberTimesMarketIndicatedBuy" , "NumberTimesMarketIndicatedSell" , "NumberTimesMarketIndicatedSame",\
"ValueOfTargetVariableFromCurrentToNextNthTickTaken"]
def extractAttributeFromDataMatrix(args):
try:
args.n
except:
print "Since -n has not been specified I cannot proceed"
os._exit(-1)
try:
args.c
'''
if args.n not in args.c:
print "Something wrong in design file "
print "moving average of last N secs we will taking as qty , so N should be there in sysnthetic fetaure also"
'''
except:
print "Since -c has not been specified I cannot proceed"
os._exit(-1)
wt = float(args.n)
if(args.cType == "synthetic"):
colNumberOfAttribute = colNumberOfData.SysFeature
else:
colNumberOfAttribute = eval("colNumberOfData."+ args.c )
colNumberOfTimeStamp = colNumberOfData.TimeStamp
colNumberOfExchangeStamp = colNumberOfData.ExchangeTS
if "Bid" in args.c:
side = "Bid"
elif "Ask" in args.c:
side = "Ask"
list_of_price_array = [ eval('colNumberOfData.'+side+'P0') , eval('colNumberOfData.'+side+'P1') ,\
eval('colNumberOfData.'+side+'P2') , eval('colNumberOfData.'+side+'P3') , eval('colNumberOfData.'+side+'P4') ]
list_of_qty_array = [ eval('colNumberOfData.'+side+'Q0') , eval('colNumberOfData.'+side+'Q1') ,\
eval('colNumberOfData.'+side+'Q2') , eval('colNumberOfData.'+side+'Q3') , eval('colNumberOfData.'+side+'Q4') ]
currentRowCount = 0
levelOfDataAvailable = 4
print "data File length"+ str(len(dataFile.matrix))
for dataRow in dataFile.matrix:
qSum = 0
totalPrice = 0
totalPriceAtThisLevel = 0
i = 0
try:
qtyForCalculatingWeightedAverage = wt * float(dataRow[colNumberOfAttribute])
except:
print "Error"
print dataRow
print dataRow[colNumberOfAttribute]
os._exit(1)
while(i <= levelOfDataAvailable and qSum < qtyForCalculatingWeightedAverage):
priceAtThisLevel = float(dataFile.matrix[currentRowCount][list_of_price_array[i]])
qtyAtThisLevel = float(dataFile.matrix[currentRowCount][list_of_qty_array[i]])
qSum += qtyAtThisLevel
if(qSum > qtyForCalculatingWeightedAverage):
qtyToUseAtThisLevel = qtyAtThisLevel - (qSum - qtyForCalculatingWeightedAverage)
totalPriceAtThisLevel = qtyToUseAtThisLevel * priceAtThisLevel
else:
totalPriceAtThisLevel = qtyAtThisLevel * priceAtThisLevel
totalPrice += totalPriceAtThisLevel
i = i + 1
if(qSum < qtyForCalculatingWeightedAverage): # This implies that the current row does not have enough qty to fill our requirement.
qtyToUseAtThisLevel = qtyForCalculatingWeightedAverage - qSum
totalPriceAtThisLevel = qtyToUseAtThisLevel * priceAtThisLevel
totalPrice += totalPriceAtThisLevel
attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfTimeStamp)
# print currentRowCount,qtyForCalculatingWeightedAverage
attribute.aList[currentRowCount][1] = float(totalPrice)/qtyForCalculatingWeightedAverage
attribute.aList[currentRowCount][2] = qtyForCalculatingWeightedAverage
attribute.aList[currentRowCount][3] = totalPrice
currentRowCount += 1
if (currentRowCount%10000==0):
print "Processed row number " + str(currentRowCount)
lNameOfFeaturePrinted = "fWAPriceOfCol" + side + "InLast" + str(args.n) + "Qty"
return ["TimeStamp",lNameOfFeaturePrinted,"QtyUsedForCalWtAvg","TotalPriceCalculatedAtCurrRow"]
def extractAttributeFromDataMatrix(args):
queueOfCellValueInFutureNLTQs = deque()
totalOfFutureNRows = 0.0
if args.n is None:
print "-n has not been specified"
os._exit(-1)
totalOfFutureLTQQty = int(args.n)
lPipSize = int(args.tickSize)
lMargin1 = (1 * lPipSize)
lMargin2 = lMargin1 + (2 * lPipSize)
""" lets get the total of futureNrows"""
futureLTQSum = 0
currentRowIndex = 0
lLastTradeIndex = 0
lNoMoreTradesFound = 0
currentLTPValue = 0.0
currentLTQValue = 0
WeightedLTPSum = 0.0
totalLTPQty = totalOfFutureLTQQty
while (futureLTQSum < totalOfFutureLTQQty):
if dataFile.matrix[currentRowIndex][
colNumberOfData.MsgCode].upper() == "T":
currentLTPValue = float(
dataFile.matrix[currentRowIndex][colNumberOfData.LTP])
currentLTQValue = int(
dataFile.matrix[currentRowIndex][colNumberOfData.NewQ])
queueOfCellValueInFutureNLTQs.append([
(currentLTPValue * currentLTQValue), currentLTQValue,
currentLTPValue, 0
])
futureLTQSum = futureLTQSum + currentLTQValue
WeightedLTPSum = WeightedLTPSum + queueOfCellValueInFutureNLTQs[
-1][0]
lLastTradeIndex = currentRowIndex
currentRowIndex = currentRowIndex + 1
if futureLTQSum > totalOfFutureLTQQty:
currentLTQValueToBeUsed = currentLTQValue - (futureLTQSum -
totalOfFutureLTQQty)
currentTickPriceToBeAdded = currentLTPValue * currentLTQValueToBeUsed
LTQValueNotAdded = (futureLTQSum - totalOfFutureLTQQty)
WeightedLTPSum = WeightedLTPSum - queueOfCellValueInFutureNLTQs[-1][
0] + currentTickPriceToBeAdded
queueOfCellValueInFutureNLTQs[-1] = [
currentTickPriceToBeAdded, currentLTQValueToBeUsed,
currentLTPValue, LTQValueNotAdded
]
currentRowCount = 0
for dataRow in dataFile.matrix:
attribute.aList[currentRowCount][0] = common.getTimeStamp(
dataFile.matrix[currentRowCount], colNumberOfData.TimeStamp)
if (dataFile.matrix[currentRowCount][colNumberOfData.MsgCode].upper()
== "T"):
currentWtLTQToBeSubtracted = queueOfCellValueInFutureNLTQs.popleft(
)
WeightedLTPSum = WeightedLTPSum - currentWtLTQToBeSubtracted[0]
LTQSubtracted = currentWtLTQToBeSubtracted[1]
if len(queueOfCellValueInFutureNLTQs) != 0:
LTQWhichCanBeAddedWithoutNewTradeMessage = queueOfCellValueInFutureNLTQs[
-1][3]
LTQAddedWithoutNewTradeMessage = min(
LTQSubtracted, LTQWhichCanBeAddedWithoutNewTradeMessage)
currentTickPriceToBeAdded = (
LTQAddedWithoutNewTradeMessage *
queueOfCellValueInFutureNLTQs[-1][2]
) + queueOfCellValueInFutureNLTQs[-1][0]
LTQValueNotAddedInLastElementOfQueue = queueOfCellValueInFutureNLTQs[
-1][3] - LTQAddedWithoutNewTradeMessage
WeightedLTPSum = WeightedLTPSum - queueOfCellValueInFutureNLTQs[
-1][0] + currentTickPriceToBeAdded
queueOfCellValueInFutureNLTQs[-1][
0] = currentTickPriceToBeAdded
queueOfCellValueInFutureNLTQs[-1][
1] = queueOfCellValueInFutureNLTQs[-1][
1] + LTQAddedWithoutNewTradeMessage
queueOfCellValueInFutureNLTQs[-1][
3] = LTQValueNotAddedInLastElementOfQueue
else:
LTQAddedWithoutNewTradeMessage = 0
print "Last element in queue = ", currentWtLTQToBeSubtracted
if LTQSubtracted > LTQAddedWithoutNewTradeMessage:
LTQToBeAddedFromNewTrade = LTQSubtracted - LTQAddedWithoutNewTradeMessage
if (lNoMoreTradesFound == 0):
startIndexToFindNextTrade = lLastTradeIndex + 1
while (startIndexToFindNextTrade < len(dataFile.matrix)):
if dataFile.matrix[startIndexToFindNextTrade][
colNumberOfData.MsgCode].upper() == "T":
lLastTradeIndex = startIndexToFindNextTrade
futureLTPValue = float(
dataFile.matrix[startIndexToFindNextTrade][
colNumberOfData.LTP])
futureLTQValue = int(
dataFile.matrix[startIndexToFindNextTrade][
colNumberOfData.NewQ])
queueOfCellValueInFutureNLTQs.append([
(futureLTPValue * futureLTQValue),
futureLTQValue, futureLTPValue, 0
])
LTQToBeAddedFromNewTrade = LTQToBeAddedFromNewTrade - futureLTQValue
WeightedLTPSum = WeightedLTPSum + queueOfCellValueInFutureNLTQs[
-1][0]
if LTQToBeAddedFromNewTrade <= 0:
if LTQToBeAddedFromNewTrade < 0:
LTQValueToBeUsed = futureLTQValue + LTQToBeAddedFromNewTrade
currentTickPriceToBeAdded = LTQValueToBeUsed * futureLTPValue
LTQValueNotAdded = -1 * LTQToBeAddedFromNewTrade
WeightedLTPSum = WeightedLTPSum - queueOfCellValueInFutureNLTQs[
-1][0] + currentTickPriceToBeAdded
queueOfCellValueInFutureNLTQs[-1] = [
currentTickPriceToBeAdded,
LTQValueToBeUsed, futureLTPValue,
LTQValueNotAdded
]
break
startIndexToFindNextTrade = startIndexToFindNextTrade + 1
if startIndexToFindNextTrade == len(dataFile.matrix):
if LTQToBeAddedFromNewTrade > 0:
totalLTPQty = totalLTPQty - LTQToBeAddedFromNewTrade
lNoMoreTradesFound = 1
else:
totalLTPQty = totalLTPQty - LTQToBeAddedFromNewTrade
currentBidP0 = float(
dataFile.matrix[currentRowCount][colNumberOfData.BidP0])
currentAskP0 = float(
dataFile.matrix[currentRowCount][colNumberOfData.AskP0])
currentLTP = float(
dataFile.matrix[currentRowCount][colNumberOfData.LTP])
currentMsgCode = dataFile.matrix[currentRowCount][
colNumberOfData.MsgCode]
currentLTQ = int(
dataFile.matrix[currentRowCount][colNumberOfData.NewQ])
lClassOfTargetVariable = 0
valueInCurrentRow = 0
lClassifiedInOneClass = 0
if totalLTPQty != 0:
WALTPOfFutureNQty = WeightedLTPSum / totalLTPQty
if (WALTPOfFutureNQty - (currentBidP0 + lPipSize)) > lMargin1:
if (WALTPOfFutureNQty - (currentBidP0 + lPipSize)) > lMargin2:
lClassOfTargetVariable = 2
lClassifiedInOneClass = 1
else:
lClassOfTargetVariable = 1
lClassifiedInOneClass = 1
if ((currentAskP0 - lPipSize) - WALTPOfFutureNQty) > lMargin1:
if lClassifiedInOneClass == 1:
lClassOfTargetVariable = 0
elif ((currentAskP0 - lPipSize) - WALTPOfFutureNQty) > lMargin2:
lClassOfTargetVariable = -2
lClassifiedInOneClass = 1
else:
lClassOfTargetVariable = -1
lClassifiedInOneClass = 1
attribute.aList[currentRowCount][1] = lClassOfTargetVariable
attribute.aList[currentRowCount][2] = currentBidP0
attribute.aList[currentRowCount][3] = ";".join([ str(currentAskP0) , str(WALTPOfFutureNQty) , str(currentLTP) , str(currentLTQ) ,\
str(currentMsgCode) , str(WeightedLTPSum) , str(totalLTPQty)
])
currentRowCount = currentRowCount + 1
if (currentRowCount % 1000 == 0):
print "Processed row number " + str(currentRowCount)
lNameOfTarget = "tClassOfTargetVariableInFuture" + str(
args.n) + "TradeTicks"
return ["TimeStamp",lNameOfTarget,"CurrentBid","CurrentAsk","ValueOfTargetVariableFromCurrentToNextNthTickTaken","CurrentLTP","CurrentLTQ","CurrentMsgCode"\
"WeightedLTPSum" , "totalLTPQtys"]
def extractAttributeFromDataMatrix(args):
colNumberOfTimeStamp = colNumberOfData.TimeStamp
currentRowCount = 0
lPipSize = int(args.tickSize)
colOfCurrentBid = colNumberOfData.BestBidP
colOfCurrentAsk = colNumberOfData.BestAskP
currentAsk = 0
previousAsk = 0
currentBid = 0
previousBid = 0
bidSideCases = 0
askSideCases = 0
bidSideFlag = 0
askSideFlag = 0
lAskSideAskP = 0
lAskSideBidP = 0
lBidSideAskP = 0
lBidSideBidP = 0
for dataRow in dataFile.matrix:
attribute.aList[currentRowCount][0] = common.getTimeStamp(
dataFile.matrix[currentRowCount], colNumberOfTimeStamp)
currentBid = float(dataFile.matrix[currentRowCount][colOfCurrentBid])
currentAsk = float(dataFile.matrix[currentRowCount][colOfCurrentAsk])
if (lAskSideAskP != currentAsk
and askSideFlag == 1) or (lAskSideBidP != currentBid
and askSideFlag == 1):
askSideFlag = 0
if (lBidSideAskP != currentAsk
and askSideFlag == 1) or (lBidSideBidP != currentBid
and bidSideFlag == 1):
bidSideFlag = 0
if (currentAsk - currentBid) == lPipSize:
if (previousAsk - currentAsk) > lPipSize or askSideFlag == 1:
askSideCases += 1
askSideFlag = 1
lAskSideAskP = currentAsk
lAskSideBidP = currentBid
if (currentBid - previousBid) > lPipSize or bidSideFlag == 1:
bidSideCases += 1
bidSideFlag = 1
lBidSideAskP = currentAsk
lBidSideBidP = currentBid
attribute.aList[currentRowCount][1] = askSideFlag
attribute.aList[currentRowCount][2] = bidSideFlag
attribute.aList[currentRowCount][3] = ";".join(
map(str, [currentBid, currentAsk, previousBid, previousAsk]))
currentRowCount = currentRowCount + 1
# if(currentRowCount % 1000 == 0):
# print "Processed row number " + str(currentRowCount)
if previousBid == 0 or previousBid != currentBid:
previousBid = currentBid
if previousAsk == 0 or previousAsk != currentAsk:
previousAsk = currentAsk
print "Number of bid side cases = ", bidSideCases, "Percentage of Bid side cases = ", (
float(bidSideCases) / currentRowCount) * 100
print "Number of ask side cases = ", askSideCases, "Percentage of Ask side cases = ", (
float(askSideCases) / currentRowCount) * 100
lNameOfFeaturePrinted = "BidSideCases"
return [
"TimeStamp", lNameOfFeaturePrinted, "AskSideCases", "CurrentBid",
"CurrentAsk", "PreviousBid", "PreviousAsk"
]
def extractAttributeFromDataMatrix(args):
currentRowCount = 0
try:
args.c
except:
print "Since -c has not been specified I cannot proceed"
os._exit()
colNumberOfPrice0 = eval("colNumberOfData." + args.c + "P0")
colNumberOfQty0 = eval("colNumberOfData." + args.c + "Q0")
colNumberOfPrice1 = colNumberOfPrice0 + 2
colNumberOfQty1 = colNumberOfQty0 + 2
colNumberOfPrice2 = colNumberOfPrice0 + 4
colNumberOfQty2 = colNumberOfQty0 + 4
colNumberOfPrice3 = colNumberOfPrice0 + 6
colNumberOfQty3 = colNumberOfQty0 + 6
colNumberOfPrice4 = colNumberOfPrice0 + 8
colNumberOfQty4 = colNumberOfQty0 + 8
colNumberOfTimeStamp = colNumberOfData.TimeStamp
#++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Reading prices +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
for dataRow in dataFile.matrix:
l_obj = parameters_for_calculating_historical_weighted_bid_or_ask(
dataRow[colNumberOfTimeStamp])
if args.c == "Ask":
l_obj.AskQ0 = int(dataRow[colNumberOfQty0])
l_obj.AskP0 = float(dataRow[colNumberOfPrice0])
l_obj.AskQ1 = int(dataRow[colNumberOfQty1])
l_obj.AskP1 = float(dataRow[colNumberOfPrice1])
l_obj.AskQ2 = int(dataRow[colNumberOfQty2])
l_obj.AskP2 = float(dataRow[colNumberOfPrice2])
l_obj.AskQ3 = int(dataRow[colNumberOfQty3])
l_obj.AskP3 = float(dataRow[colNumberOfPrice3])
l_obj.AskQ4 = int(dataRow[colNumberOfQty4])
l_obj.AskP4 = float(dataRow[colNumberOfPrice4])
l_obj.AskP_list = [
l_obj.AskP0, l_obj.AskP1, l_obj.AskP2, l_obj.AskP3, l_obj.AskP4
]
l_obj.AskQ_list = [
l_obj.AskQ0, l_obj.AskQ1, l_obj.AskQ2, l_obj.AskQ3, l_obj.AskQ4
]
l_obj.sum_qty_for_avg_cal_Ask = ((g_weight_list[0] * l_obj.AskQ0) +
(g_weight_list[1] * l_obj.AskQ1) +
(g_weight_list[2] * l_obj.AskQ2) +
(g_weight_list[3] * l_obj.AskQ3) +
(g_weight_list[4] * l_obj.AskQ4))
else:
l_obj.BidQ0 = int(dataRow[colNumberOfQty0])
l_obj.BidP0 = float(dataRow[colNumberOfPrice0])
l_obj.BidQ1 = int(dataRow[colNumberOfQty1])
l_obj.BidP1 = float(dataRow[colNumberOfPrice1])
l_obj.BidQ2 = int(dataRow[colNumberOfQty2])
l_obj.BidP2 = float(dataRow[colNumberOfPrice2])
l_obj.BidQ3 = int(dataRow[colNumberOfQty3])
l_obj.BidP3 = float(dataRow[colNumberOfPrice3])
l_obj.BidQ4 = int(dataRow[colNumberOfQty4])
l_obj.BidP4 = float(dataRow[colNumberOfPrice4])
l_obj.BidP_list = [
l_obj.BidP0, l_obj.BidP1, l_obj.BidP2, l_obj.BidP3, l_obj.BidP4
]
l_obj.BidQ_list = [
l_obj.BidQ0, l_obj.BidQ1, l_obj.BidQ2, l_obj.BidQ3, l_obj.BidQ4
]
l_obj.sum_qty_for_avg_cal_Bid = ((g_weight_list[0] * l_obj.BidQ0) +
(g_weight_list[1] * l_obj.BidQ1) +
(g_weight_list[2] * l_obj.BidQ2) +
(g_weight_list[3] * l_obj.BidQ3) +
(g_weight_list[4] * l_obj.BidQ4))
g_filtered_object_list.append(l_obj)
#++++++++++++++++++++++++++++++++++++++++++++++++++++++ Calculating ratio ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
if args.c == "Ask":
calculate_historical_weighted_Ask()
else:
calculate_historical_weighted_Bid()
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Storing results ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
for dataRow in dataFile.matrix:
# In the next 2 rows we do not do -1 since this feature if for the current row.
attribute.aList[currentRowCount][0] = common.getTimeStamp(
dataFile.matrix[currentRowCount], colNumberOfTimeStamp, args.cType)
if args.c == "Ask":
attribute.aList[currentRowCount][1] = g_filtered_object_list[
currentRowCount].sum_qty_for_avg_cal_Ask / float(
g_filtered_object_list[currentRowCount].window_Ask)
else:
attribute.aList[currentRowCount][1] = g_filtered_object_list[
currentRowCount].sum_qty_for_avg_cal_Bid / float(
g_filtered_object_list[currentRowCount].window_Bid)
currentRowCount = currentRowCount + 1
if (currentRowCount % 10000 == 0):
print "Processed row number " + str(currentRowCount)
lNameOfFeaturePrinted = "fOBAvgHistPriceOfCol" + args.c + "InRows"
return ["TimeStamp", lNameOfFeaturePrinted, "Zero1", "Zero2"]
def extractAttributeFromDataMatrix(args):
try:
args.n
except:
print "Since -n has not been specified I cannot proceed"
os._exit(-1)
try:
args.c
except:
print "Since -c has not been specified I cannot proceed"
os._exit(-1)
try:
args.tickSize
except:
print "args.tickSize is required"
os._exit(-1)
N = int(args.n)
tickSize = int(args.tickSize)
list_of_price_array = map(eval, ['colNumberOfData.'+args.c+'P'+str(level) for level in xrange(N+1)])
list_of_qty_array = map(eval, ['colNumberOfData.'+args.c+'Q'+str(level) for level in xrange(N+1)])
currentRowCount = 0
qtyForCalculatingWeightedAverage = sum(list_of_qty_array)
levelOfDataAvailable = N
for dataRow in dataFile.matrix:
qSum = 0
totalPrice = 0
totalPriceAtThisLevel = 0
i = 0
while(i <= levelOfDataAvailable and qSum < qtyForCalculatingWeightedAverage):
priceAtThisLevel = float(dataFile.matrix[currentRowCount][list_of_price_array[i]])
qtyAtThisLevel = float(dataFile.matrix[currentRowCount][list_of_qty_array[i]])
if i==1:
if (priceAtThisLevel-float(dataFile.matrix[currentRowCount][list_of_price_array[0]]) ) != abs(float(tickSize)):
if "AskP" in args.c:
priceAtThisLevel = float(dataFile.matrix[currentRowCount][list_of_price_array[0]]) + float(tickSize)
else:
priceAtThisLevel = float(dataFile.matrix[currentRowCount][list_of_price_array[0]]) - float(tickSize)
qtyAtThisLevel = 0
qSum += qtyAtThisLevel
if(qSum > qtyForCalculatingWeightedAverage):
qtyToUseAtThisLevel = qtyAtThisLevel - (qSum - qtyForCalculatingWeightedAverage)
totalPriceAtThisLevel = qtyToUseAtThisLevel * priceAtThisLevel
else:
totalPriceAtThisLevel = qtyAtThisLevel * priceAtThisLevel
totalPrice += totalPriceAtThisLevel
i = i + 1
if(qSum < qtyForCalculatingWeightedAverage): # This implies that the current row does not have enough qty to fill our requirement.
qtyToUseAtThisLevel = qtyForCalculatingWeightedAverage - qSum
totalPriceAtThisLevel = qtyToUseAtThisLevel * priceAtThisLevel
totalPrice += totalPriceAtThisLevel
attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp)
attribute.aList[currentRowCount][1] = float(totalPrice)/qtyForCalculatingWeightedAverage
currentRowCount += 1
if (currentRowCount%10000==0):
print "Processed row number " + str(currentRowCount)
lNameOfFeaturePrinted = "fWAPriceOfCol" + args.c + "In01PosInLast" + str(args.n) + "Qty"
return ["TimeStamp",lNameOfFeaturePrinted,"Zero1","Zero2"]
def extractAttributeFromDataMatrix(args):
try:
args.n
except:
print "Since -n has not been specified I cannot proceed"
os._exit(-1)
try:
args.c
'''
if args.n not in args.c:
print "Something wrong in design file "
print "moving average of last N secs we will taking as qty , so N should be there in sysnthetic fetaure also"
'''
except:
print "Since -c has not been specified I cannot proceed"
os._exit(-1)
wt = float(args.n)
if(args.cType == "synthetic"):
colNumberOfAttribute = colNumberOfData.SysFeature
else:
colNumberOfAttribute = eval("colNumberOfData."+ args.c )
colNumberOfTimeStamp = colNumberOfData.TimeStamp
colNumberOfExchangeStamp = colNumberOfData.ExchangeTS
if "Bid" in args.c:
side = "Bid"
elif "Ask" in args.c:
side = "Ask"
list_of_price_array = [ eval('colNumberOfData.'+side+'P1') , eval('colNumberOfData.'+side+'P2') ,\
eval('colNumberOfData.'+side+'P3') , eval('colNumberOfData.'+side+'P4') , eval('colNumberOfData.'+side+'P5') ]
list_of_qty_array = [ eval('colNumberOfData.'+side+'Q1') , eval('colNumberOfData.'+side+'Q2') ,\
eval('colNumberOfData.'+side+'Q3') , eval('colNumberOfData.'+side+'Q4') , eval('colNumberOfData.'+side+'Q5') ]
currentRowCount = 0
levelOfDataAvailable = 4
for dataRow in dataFile.matrix:
qSum = 0
totalPrice = 0
totalPriceAtThisLevel = 0
i = 0
try:
qtyForCalculatingWeightedAverage = wt * float(dataRow[colNumberOfAttribute])
except:
print dataRow
print dataRow[colNumberOfAttribute]
os._exit(1)
while(i <= levelOfDataAvailable and qSum < qtyForCalculatingWeightedAverage):
priceAtThisLevel = float(dataFile.matrix[currentRowCount][list_of_price_array[i]])
qtyAtThisLevel = float(dataFile.matrix[currentRowCount][list_of_qty_array[i]])
qSum += qtyAtThisLevel
if(qSum > qtyForCalculatingWeightedAverage):
qtyToUseAtThisLevel = qtyAtThisLevel - (qSum - qtyForCalculatingWeightedAverage)
totalPriceAtThisLevel = qtyToUseAtThisLevel * priceAtThisLevel
else:
totalPriceAtThisLevel = qtyAtThisLevel * priceAtThisLevel
totalPrice += totalPriceAtThisLevel
i = i + 1
if(qSum < qtyForCalculatingWeightedAverage): # This implies that the current row does not have enough qty to fill our requirement.
qtyToUseAtThisLevel = qtyForCalculatingWeightedAverage - qSum
totalPriceAtThisLevel = qtyToUseAtThisLevel * priceAtThisLevel
totalPrice += totalPriceAtThisLevel
attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfTimeStamp)
attribute.aList[currentRowCount][1] = float(totalPrice)/qtyForCalculatingWeightedAverage
attribute.aList[currentRowCount][2] = qtyForCalculatingWeightedAverage
attribute.aList[currentRowCount][3] = totalPrice
currentRowCount += 1
if (currentRowCount%10000==0):
print "Processed row number " + str(currentRowCount)
lNameOfFeaturePrinted = "fWAPriceOfCol" + side + "InLast" + str(args.n) + "Qty"
return ["TimeStamp",lNameOfFeaturePrinted,"QtyUsedForCalWtAvg","TotalPriceCalculatedAtCurrRow"]
def extractAttributeFromDataMatrix(args):
queueOfCellValueInFutureNLTQs = deque()
totalOfFutureNRows = 0.0
if args.n is None:
print "-n has not been specified"
os._exit(-1)
totalOfFutureLTQQty = int(args.n)
lPipSize = int(args.tickSize)
lMargin1 = ( 1 * lPipSize )
lMargin2 = lMargin1 + ( 2 * lPipSize )
""" lets get the total of futureNrows"""
futureLTQSum = 0
currentRowIndex = 0
lLastTradeIndex = 0
lNoMoreTradesFound = 0
currentLTPValue = 0.0
currentLTQValue = 0
WeightedLTPSum = 0.0
totalLTPQty = totalOfFutureLTQQty
while(futureLTQSum < totalOfFutureLTQQty):
if dataFile.matrix[currentRowIndex][colNumberOfData.MsgCode].upper() == "T":
currentLTPValue = float(dataFile.matrix[currentRowIndex][colNumberOfData.LTP])
currentLTQValue = int(dataFile.matrix[currentRowIndex][colNumberOfData.NewQ])
queueOfCellValueInFutureNLTQs.append( [ (currentLTPValue * currentLTQValue) , currentLTQValue , currentLTPValue , 0 ] )
futureLTQSum = futureLTQSum + currentLTQValue
WeightedLTPSum = WeightedLTPSum + queueOfCellValueInFutureNLTQs[-1][0]
lLastTradeIndex = currentRowIndex
currentRowIndex = currentRowIndex + 1
if futureLTQSum > totalOfFutureLTQQty :
currentLTQValueToBeUsed = currentLTQValue - ( futureLTQSum - totalOfFutureLTQQty )
currentTickPriceToBeAdded = currentLTPValue * currentLTQValueToBeUsed
LTQValueNotAdded = ( futureLTQSum - totalOfFutureLTQQty )
WeightedLTPSum = WeightedLTPSum - queueOfCellValueInFutureNLTQs[-1][0] + currentTickPriceToBeAdded
queueOfCellValueInFutureNLTQs[-1] = [ currentTickPriceToBeAdded , currentLTQValueToBeUsed , currentLTPValue , LTQValueNotAdded ]
currentRowCount = 0
for dataRow in dataFile.matrix:
attribute.aList[currentRowCount][0] = common.getTimeStamp(dataFile.matrix[currentRowCount],colNumberOfData.TimeStamp)
if (dataFile.matrix[currentRowCount][colNumberOfData.MsgCode].upper() == "T") :
currentWtLTQToBeSubtracted = queueOfCellValueInFutureNLTQs.popleft()
WeightedLTPSum = WeightedLTPSum - currentWtLTQToBeSubtracted[0]
LTQSubtracted = currentWtLTQToBeSubtracted[1]
if len(queueOfCellValueInFutureNLTQs) != 0:
LTQWhichCanBeAddedWithoutNewTradeMessage = queueOfCellValueInFutureNLTQs[-1][3]
LTQAddedWithoutNewTradeMessage = min( LTQSubtracted , LTQWhichCanBeAddedWithoutNewTradeMessage)
currentTickPriceToBeAdded = ( LTQAddedWithoutNewTradeMessage * queueOfCellValueInFutureNLTQs[-1][2] ) + queueOfCellValueInFutureNLTQs[-1][0]
LTQValueNotAddedInLastElementOfQueue = queueOfCellValueInFutureNLTQs[-1][3] - LTQAddedWithoutNewTradeMessage
WeightedLTPSum = WeightedLTPSum - queueOfCellValueInFutureNLTQs[-1][0] + currentTickPriceToBeAdded
queueOfCellValueInFutureNLTQs[-1][0] = currentTickPriceToBeAdded
queueOfCellValueInFutureNLTQs[-1][1] = queueOfCellValueInFutureNLTQs[-1][1] + LTQAddedWithoutNewTradeMessage
queueOfCellValueInFutureNLTQs[-1][3] = LTQValueNotAddedInLastElementOfQueue
else:
LTQAddedWithoutNewTradeMessage = 0
print "Last element in queue = " , currentWtLTQToBeSubtracted
if LTQSubtracted > LTQAddedWithoutNewTradeMessage:
LTQToBeAddedFromNewTrade = LTQSubtracted - LTQAddedWithoutNewTradeMessage
if (lNoMoreTradesFound == 0):
startIndexToFindNextTrade = lLastTradeIndex + 1
while( startIndexToFindNextTrade < len(dataFile.matrix)):
if dataFile.matrix[startIndexToFindNextTrade][colNumberOfData.MsgCode].upper() == "T":
lLastTradeIndex = startIndexToFindNextTrade
futureLTPValue = float(dataFile.matrix[startIndexToFindNextTrade][colNumberOfData.LTP])
futureLTQValue = int(dataFile.matrix[startIndexToFindNextTrade][colNumberOfData.NewQ])
queueOfCellValueInFutureNLTQs.append( [ (futureLTPValue * futureLTQValue) , futureLTQValue , futureLTPValue , 0 ] )
LTQToBeAddedFromNewTrade = LTQToBeAddedFromNewTrade - futureLTQValue
WeightedLTPSum = WeightedLTPSum + queueOfCellValueInFutureNLTQs[-1][0]
if LTQToBeAddedFromNewTrade <= 0 :
if LTQToBeAddedFromNewTrade < 0 :
LTQValueToBeUsed = futureLTQValue + LTQToBeAddedFromNewTrade
currentTickPriceToBeAdded = LTQValueToBeUsed * futureLTPValue
LTQValueNotAdded = -1 * LTQToBeAddedFromNewTrade
WeightedLTPSum = WeightedLTPSum - queueOfCellValueInFutureNLTQs[-1][0] + currentTickPriceToBeAdded
queueOfCellValueInFutureNLTQs[-1] = [ currentTickPriceToBeAdded , LTQValueToBeUsed , futureLTPValue , LTQValueNotAdded ]
break
startIndexToFindNextTrade = startIndexToFindNextTrade + 1
if startIndexToFindNextTrade == len(dataFile.matrix):
if LTQToBeAddedFromNewTrade > 0 :
totalLTPQty = totalLTPQty - LTQToBeAddedFromNewTrade
lNoMoreTradesFound = 1
else:
totalLTPQty = totalLTPQty - LTQToBeAddedFromNewTrade
currentBidP0 = float(dataFile.matrix[currentRowCount][colNumberOfData.BidP0])
currentAskP0 = float(dataFile.matrix[currentRowCount][colNumberOfData.AskP0])
currentLTP = float(dataFile.matrix[currentRowCount][colNumberOfData.LTP])
currentMsgCode = dataFile.matrix[currentRowCount][colNumberOfData.MsgCode]
currentLTQ = int(dataFile.matrix[currentRowCount][colNumberOfData.NewQ])
lClassOfTargetVariable = 0
valueInCurrentRow = 0
lClassifiedInOneClass = 0
if totalLTPQty != 0:
WALTPOfFutureNQty = WeightedLTPSum / totalLTPQty
if (WALTPOfFutureNQty - (currentBidP0 + lPipSize)) > lMargin1 :
if (WALTPOfFutureNQty - (currentBidP0 + lPipSize)) > lMargin2:
lClassOfTargetVariable = 2
lClassifiedInOneClass = 1
else:
lClassOfTargetVariable = 1
lClassifiedInOneClass = 1
if ((currentAskP0 - lPipSize) - WALTPOfFutureNQty ) > lMargin1 :
if lClassifiedInOneClass == 1 :
lClassOfTargetVariable = 0
elif ((currentAskP0 - lPipSize) - WALTPOfFutureNQty ) > lMargin2 :
lClassOfTargetVariable = -2
lClassifiedInOneClass = 1
else:
lClassOfTargetVariable = -1
lClassifiedInOneClass = 1
attribute.aList[currentRowCount][1] = lClassOfTargetVariable
attribute.aList[currentRowCount][2] = currentBidP0
attribute.aList[currentRowCount][3] = ";".join([ str(currentAskP0) , str(WALTPOfFutureNQty) , str(currentLTP) , str(currentLTQ) ,\
str(currentMsgCode) , str(WeightedLTPSum) , str(totalLTPQty)
])
currentRowCount = currentRowCount + 1
if(currentRowCount % 1000 == 0):
print "Processed row number " + str(currentRowCount)
lNameOfTarget = "tClassOfTargetVariableInFuture"+ str(args.n) + "TradeTicks"
return ["TimeStamp",lNameOfTarget,"CurrentBid","CurrentAsk","ValueOfTargetVariableFromCurrentToNextNthTickTaken","CurrentLTP","CurrentLTQ","CurrentMsgCode"\
"WeightedLTPSum" , "totalLTPQtys"]
