def scriptWrapperForDayWiseCorrelation(pTrainingDay):
lDate = os.path.basename(os.path.abspath(pTrainingDay))
lFileName = l_exp_dir + "/corr-date-" + lDate + "-td." + os.path.basename(
os.path.abspath(args.td)
) + "-dt." + args.dt + attribute.generateExtension() + ".r"
utility.runCommand([lFileName, '-d', pTrainingDay], args.run,
args.sequence)
def scripWrapperForTradingCommand(predictionDirAfterLastTD):
global tradeEngine
for lQty in lQtyList:
utility.runCommand([tradeEngine,"-e",args.e1, "-e1",args.e,"-skipT",args.skipT,"-a",args.a,"-entryCL",entrylist,"-exitCL",\
exitlist,"-entryCL2",entrycl2list,"-exitCL2",exitcl2list,"-orderQty",lQty,'-dt',args.dt,"-targetClass",args.targetClass,"-td",args.td , "-pd",predictionDirAfterLastTD,\
'-tickSize',args.tickSize,'-wt',args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP , "-ls", args.ls],args.run,args.sequence)
def scriptWrapperForFeatureGeneration(trainingDirectory):
utility.runCommand([
"aGenForE.py", "-e", args.e, "-d", trainingDirectory, "-g", args.g,
"-run", args.run, "-sequence", args.sequence, '-tickSize',
args.tickSize, "-iT", args.iT, "-oT", args.oT, "-sP", args.sP
], args.run, args.sequence)
pass
def scriptWrapperForGeneratingOrderBook(trainingDirectory):
utility.runCommand([
"generate_orderbook_with_bands_6level.py", "-td",
trainingDirectory, "-iT", args.iT, "-oT", args.oT, "-sP", args.sP,
'-bGap', args.bGap, '-uGE', 'no'
], args.run, args.sequence)
pass
def scriptWrapper(experimentName):
if args.targetClass == "multinomial" :
# utility.runCommand(["cMatrixGen.py","-d",args.pd,"-e",experimentName,"-a",algo],args.run,args.sequence)
utility.runCommand(["./ob/quality/tradeE5.py","-e",experimentName,"-skipT",args.skipT,"-a",algo,"-entryCL","55;90;60;50","-exitCL","45;50;40;25","-orderQty","500",\
'-dt',args.dt,"-targetClass",args.targetClass,"-td",args.td , "-pd",args.pd,'-tickSize',args.tickSize,'-wt',args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
else:
utility.runCommand(["./ob/quality/tradeE7Optimized.py","-e",experimentName,"-skipT",args.skipT,"-a",algo,"-entryCL",entrylist,"-exitCL",exitlist,"-orderQty","300", '-dt',args.dt,"-targetClass",args.targetClass,"-td",args.td , "-pd",args.pd,'-tickSize',args.tickSize,'-wt',args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
pass
def scripWrapperForTradingCommand(predictionDirAfterLastTD):
for lQty in lQtyList:
# utility.runCommand(["./ob/quality/tradeE7Optimized.py","-e",lExperimentFolderName,"-skipT",args.skipT,"-a",args.a,"-entryCL",entrylist,"-exitCL",exitlist,"-orderQty",lQty,'-dt',args.dt,"-targetClass",args.targetClass,"-td",args.td , "-pd",predictionDirAfterLastTD,'-tickSize',args.tickSize,'-wt',args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP ],args.run,args.sequence)
utility.runCommand(["./ob/quality/tradeE7WithMultipleProbabilitiesAndQuickExit.py","-e",lExperimentFolderName,"-skipT",args.skipT,"-a",args.a,"-entryCL",entrylist,"-exitCL",\
exitlist,"-entryCL2",entrycl2list,"-exitCL2",exitcl2list,"-orderQty",lQty,'-dt',args.dt,"-targetClass",args.targetClass,"-td",args.td , "-pd",predictionDirAfterLastTD,\
'-tickSize',args.tickSize,'-wt',args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP ],args.run,args.sequence)
def scripWrapperForTradingCommand(predictionDirAfterLastTD):
for lQty in lQtyList:
# utility.runCommand(["./ob/quality/tradeE7Optimized.py","-e",lExperimentFolderName,"-skipT",args.skipT,"-a",args.a,"-entryCL",entrylist,"-exitCL",exitlist,"-orderQty",lQty,'-dt',args.dt,"-targetClass",args.targetClass,"-td",args.td , "-pd",predictionDirAfterLastTD,'-tickSize',args.tickSize,'-wt',args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP ],args.run,args.sequence)
utility.runCommand(["./ob/quality/tradeE7WithMultipleProbabilitiesAndQuickExit.py","-e",lExperimentFolderName,"-skipT",args.skipT,"-a",args.a,"-entryCL",entrylist,"-exitCL",\
exitlist,"-entryCL2",entrycl2list,"-exitCL2",exitcl2list,"-orderQty",lQty,'-dt',args.dt,"-targetClass",args.targetClass,"-td",args.td , "-pd",predictionDirAfterLastTD,\
'-tickSize',args.tickSize,'-wt',args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP ],args.run,args.sequence)
def scriptWrapper(index):
utility.runCommand([
"./ob/quality/tradeE7BuySellMixMatchOptimized.py", "-es",
sellExperimentList[index], "-eb", buyExperimentList[index], "-skipT",
args.skipT, "-a", args.a, "-entryCL", args.entryCL, "-exitCL",
args.exitCL, "-orderQty", args.orderQty, '-dt', args.dt,
"-targetClass", args.targetClass, "-td", args.td, "-pd", args.pd,
'-tickSize', args.tickSize, '-wt', args.wt, "-iT", args.iT, "-oT",
args.oT, "-sP", args.sP
], args.run, args.sequence)
def scriptWrapper(experimentName):
if args.targetClass == "multinomial":
# utility.runCommand(["cMatrixGen.py","-d",args.pd,"-e",experimentName,"-a",algo],args.run,args.sequence)
utility.runCommand(["./ob/quality/tradeE5.py","-e",experimentName,"-skipT",args.skipT,"-a",algo,"-entryCL","55;90;60;50","-exitCL","45;50;40;25","-orderQty","500",\
'-dt',args.dt,"-targetClass",args.targetClass,"-td",args.td , "-pd",args.pd,'-tickSize',args.tickSize,'-wt',args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
else:
utility.runCommand([
"./ob/quality/tradeE7Optimized.py", "-e", experimentName, "-skipT",
args.skipT, "-a", algo, "-entryCL", entrylist, "-exitCL", exitlist,
"-orderQty", "300", '-dt', args.dt, "-targetClass",
args.targetClass, "-td", args.td, "-pd", args.pd, '-tickSize',
args.tickSize, '-wt', args.wt, "-iT", args.iT, "-oT", args.oT,
"-sP", args.sP
], args.run, args.sequence)
pass
def bot_loop():
while connected:
response = s.recv(1024).decode("utf-8")
if response == "PING :tmi.twitch.tv\r\n":
s.send("PONG :tmi.twitch.tv\r\n".encode("utf-8"))
print("Pong")
else:
username = re.search(r"\w+", response).group(0)
message = CHAT_MSG.sub("", response)
print(username + ": " + message)
# Ban pattern check
for pat in pattern.BAN_PAT:
if re.match(pat, message):
utility.ban(s, username)
utility.chat(s,"Tap, tap, tap. Nevermore. " + username + " banned")
break
# Time out pattern check
for pat in pattern.TO_PAT:
if re.match(pat, message):
utility.timeout(s, username)
utility.chat(s,"Caw caw! " + username + " silence! You know what you've done...")
break
# Command check
if re.match(r'^(![A-Z,a-z])\w', message):
# Check if command exists
if message.strip() in command_list:
utility.runCommand(s, message.strip(), command_list[message.strip()])
else:
# New command check
if re.match(r'^(![A-Z,a-z])\w+\s([A-Z,a-z])', message):
command = message.split(" ", 1)[0]
action = message.split(" ", 1)[1]
utility.newCommand(s, command, action, command_list.keys())
command_list[command] = action
print("A new command: " + command + " action: " + action)
else:
utility.chat(s, "Command doesn't exists")
time.sleep(1 / config.RATE)
pool = multiprocessing.Pool(
) # this will return the number of CPU's
results = pool.map(scriptWrapperForPredictRProgramGeneration,
predictionDaysDirectory)
results = pool.map(scriptWrapperForPredictProgramRun,
predictionDaysDirectory)
results = pool.map(scripWrapperForTradingCommand,
predictionDaysDirectory)
else:
print "None"
# results = map(scriptWrapperForPredictRProgramGeneration,predictionDaysDirectory)
# results = map(scriptWrapperForPredictRProgramGenerationDepth,predictionDaysDirectory)
# results = map(scriptWrapperForPredictProgramRun,predictionDaysDirectory)
# results = map(scriptWrapperForPredictProgramRunDepth,predictionDaysDirectory)
results = map(scripWrapperForTradingCommand,
predictionDaysDirectory)
if args.iT is not None:
message = args.iT + "_ForExperimnet_" + lExperimentFolderName + "-Results. With DEPTH Pred"
else:
message = "ml_experiments"
utility.runCommand([
"accumulate_results.py", "-e", args.e1, "-a", args.a, "-t", args.t,
"-td", args.td, "-dt", "1", '-nD',
str(args.nDays), "-m", message, "-f", "1", "-iT", args.iT, "-oT",
args.oT, "-sP", args.sP
], args.run, args.sequence)
if args.sequence == "dp":
print dp.printGroupStatus()
args.skipM = "yes"
if args.skipP == None:
args.skipP = "yes"
if args.dt == None:
args.dt = "1"
if args.wt == None:
args.wt = "default"
if args.a is not None:
allAlgos = [args.a]
else:
allAlgos = ['logitr','glmnet','randomForest']
config = ConfigObj(args.e+"/design.ini")
targetAttributes = attribute.getTargetVariableKeys(config)
one_feature_attributes = attribute.getFeatureVariableKeys(config , targetAttributes.keys()[0])
totalNumberOfFeatures = len(one_feature_attributes)
i = 2
for algo in allAlgos:
while i <= totalNumberOfFeatures:
if args.mpMearge == "yes":
utility.runCommand(["mpRGenForAllSubE.py","-e",args.e,"-a",algo,"-targetClass",args.targetClass,"-pd",args.pd ,"-skipP",args.skipP ,"-skipM",args.skipM,"-td",args.td,\
"-dt" , args.dt , '-wt' , args.wt , "-s",args.e+"/s/"+str(i)+"c","-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
else:
utility.runCommand(["mRGenForAllSubE.py","-e",args.e,"-a",algo,"-targetClass",args.targetClass,"-td" , args.td , "-dt", args.dt,"-skipM",args.skipM,\
'-wt' , args.wt,"-s",args.e+"/s/"+str(i)+"c","-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
utility.runCommand(["pRGenForAllSubE.py","-e",args.e,"-a",algo,"-skipP",args.skipP ,"-targetClass",args.targetClass,"-pd",args.pd ,"-td" , args.td ,\
"-dt", args.dt, "-wt", args.wt , "-s",args.e+"/s/"+str(i)+"c","-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
i +=1
def scriptWrapperForPredictProgramRun(predictionDirAfterLastTD):
scriptName=lExperimentFolderName+"/predict" + args.a + "-td." + os.path.basename(os.path.abspath(args.td)) + "-dt." + args.dt +"-pd." +\
os.path.basename(os.path.abspath(predictionDirAfterLastTD)) + "-wt." + args.wt + attribute.generateExtension() +".r"
dirName = predictionDirAfterLastTD.replace('/ro/', '/wf/')
utility.runCommand([scriptName, "-d", dirName], args.run,
args.sequence)
if args.targetClass == None:
args.targetClass = "binomial"
print "Since no class of target variable is specified so taking binomial class of target variable"
def scriptWrapperForFeatureGeneration(trainingDirectory):
utility.runCommand(["aGenForE.py","-e",args.e,"-d",trainingDirectory,"-g",args.g,"-run",args.run,"-sequence",args.sequence,'-tickSize',args.tickSize],args.run,args.sequence)
trainingDirectory = attribute.getTrainDirFromPredictDir( args.dt , args.pd , args.pType )
lListOfTrainingDirectories = attribute.getListOfTrainingDirectoriesNames(args.dt,trainingDirectory)
lListOfTrainPredictDirectories = lListOfTrainingDirectories
lListOfTrainPredictDirectories.append(args.pd)
results = map(scriptWrapperForFeatureGeneration,lListOfTrainPredictDirectories)
utility.runCommand(["rGenForE.py","-e",args.e,"-a",algo,"-sequence",args.sequence,"-targetClass",args.targetClass,"-skipM",args.skipM,\
'-dt',args.dt,'-pd',args.pd,"-td",trainingDirectory,"-skipP",args.skipP, '-wt' , args.wt],args.run,args.sequence)
utility.runCommand(["runAllRScriptsForE.py","-td",trainingDirectory,"-pd",args.pd,"-dt",args.dt,"-e",args.e,"-a",algo,"-run",args.run,\
'-wt' , args.wt,"-sequence",args.sequence],args.run,args.sequence)
utility.runCommand(["./ob/quality/tradeE7Optimized.py","-e",args.e,"-a",algo,"-entryCL",args.entryCL,"-exitCL",args.exitCL,"-orderQty",args.orderQty,\
'-dt',args.dt,"-targetClass",args.targetClass,"-td",trainingDirectory , "-pd",args.pd,'-tickSize',args.tickSize,'-wt',args.wt],args.run,args.sequence)
instrGroupList = args.instrGroups.split(";")
if args.pType.lower()== "same":
modelValueFileName = args.e+'/'+algo+ '-td.' + os.path.basename(os.path.abspath(trainingDirectory)) + '-dt.' + args.dt + '-targetClass.' + \
args.targetClass + "-wt." + args.wt +'.coef'
modelFp = open(modelValueFileName,"r")
lines = modelFp.readlines()
modelIniFile = args.e + '/' + "model-parameters.ini"
modelIniFP = open(modelIniFile,"w")
writeOutputToRemotePC.clear_file_from_remote_PC('/home/', "model-parameters.ini", ('1.ps.eo.spalgo.com', 'root', 'omshriganeshaya'))
for n in instrGroupList:
def scriptWrapperForTradeGeneration(TargetNumber):
utility.runCommand(["./ob/quality/tradeE7OnTargetVariable.py", "-orderQty", args.orderQty, "-d", args.d,"-startTime", args.startTime ,"-endTime",args.endTime ,"-tickSize",args.tickSize,\
"-targetType",TargetNumber,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP,"-e",args.e],args.run,args.sequence)
args.tickSize, "-iT", args.iT, "-oT", args.oT, "-sP", args.sP
])
utility.runCommandList(commandList, args)
print dp.printGroupStatus()
else:
if args.sequence == 'lp':
# to run it in local parallel mode
pool = multiprocessing.Pool() # this will return the number of CPU's
results = map(scriptWrapperForFeatureGeneration,
lListOfTrainPredictDirectories)
else:
results = map(scriptWrapperForFeatureGeneration,
lListOfTrainPredictDirectories)
utility.runCommand(["rGenForE.py","-e",args.e,"-a",algo,"-sequence",args.sequence,"-targetClass",args.targetClass,"-skipM",args.skipM,\
'-dt',args.dt,'-pd',predictionDirectory,"-td",args.td,"-skipP",args.skipP, '-wt' , args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
utility.runCommand(["runAllRScriptsForE.py","-td",args.td,"-pd",predictionDirectory,"-dt",args.dt,"-e",args.e,"-a",algo,"-run",args.run,\
'-wt' , args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP,"-sequence",args.sequence],args.run,args.sequence)
if args.targetClass == "multinomial":
utility.runCommand(
["cMatrixGen.py", "-d", predictionDirectory, "-e", args.e, "-a", algo],
args.run, args.sequence)
utility.runCommand(["./ob/quality/tradeE5.py","-e",args.e,"-a",algo,"-entryCL",args.entryCL,"-exitCL",args.exitCL,"-orderQty",args.orderQty,"-skipT",args.skipT,\
'-dt',args.dt,"-targetClass",args.targetClass,"-td",args.td , "-pd",predictionDirectory,'-tickSize',args.tickSize,'-wt',args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
else:
utility.runCommand([
"./ob/quality/tradeE7Optimized.py", "-e", args.e, "-a", algo,
"-entryCL", args.entryCL, "-exitCL", args.exitCL, "-orderQty",
args.orderQty, "-skipT", args.skipT, '-dt', args.dt, "-targetClass",
args.targetClass, "-td", args.td, "-pd", predictionDirectory,
'-tickSize', args.tickSize, '-wt', args.wt, "-iT", args.iT, "-oT",
parser.add_argument('-iT',required=False,help='Instrument name')
parser.add_argument('-sP',required=False,help='Strike price of instrument')
parser.add_argument('-oT',required=False,help='Options Type')
args = parser.parse_args()
if(args.sequence == "dp"):
import dp
attribute.initializeInstDetails(args.iT,args.sP,args.oT)
commandList = []
allDataDirectories = attribute.getListOfTrainingDirectoriesNames( int(args.nDays) , args.d,args.iT )
for directories in allDataDirectories:
commandList.append(['src/aGenForEWithTargetOnly.py','-d',directories,'-g','ob/generators/','-run',args.run,'-sequence',args.sequence,'-tickSize',args.tickSize,'-e',args.e ,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP])
pass
if(args.sequence != "dp"):
utility.runCommandList(commandList,args)
pass
else:
utility.runListOfCommandsWithMaxUtlilizationOfWorkers(commandList,args,"Target Generation",int(args.nComputers))
commandList = []
for directories in allDataDirectories:
commandList.append(['src/targetVariableRun.py','-orderQty',args.orderQty,'-d',directories,'-tickSize',args.tickSize,'-targetType',args.targetType,'-e',args.e,'-run',args.run,'-sequence',args.sequence,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP])
pass
if(args.sequence != "dp"):
utility.runCommandList(commandList,args)
else:
utility.runListOfCommandsWithMaxUtlilizationOfWorkers(commandList,args,"Trade OnTarget Variable",int(args.nComputers))
utility.runCommand(["src/accumulate_results_for_target_testing.py","-e",args.e,"-t",args.t,"-d",args.d, '-nD' , str(args.nDays) , "-m" , "ResultOfNewTargetVariableWhereWeForBuyWeCheckGreaterThanAskAndSellLessThanBid","-iT",args.iT,"-sP",args.sP,"-oT",args.oT],args.run,args.sequence)
if(args.sequence == 'dp'):
print dp.printGroupStatus()
def scriptWrapperForGeneratingOrderBook(trainingDirectory):
utility.runCommand(["generate_orderbook_with_bands_6level.py","-td",trainingDirectory,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP,'-bGap',args.bGap,'-uGE','no'],args.run,args.sequence)
pass
parser.add_argument('-sequence', required=True,help='lp / dp / serial')
parser.add_argument('-dt',required=False,help='No of day from start for which it is to be trained ')
parser.add_argument('-wt',required=True,help="default/exp , weight type to be given to different days")
parser.add_argument('-iT',required=False,help='Instrument name')
parser.add_argument('-sP',required=False,help='Strike price of instrument')
parser.add_argument('-oT',required=False,help='Options Type')
args = parser.parse_args()
attribute.initializeInstDetails(args.iT,args.sP,args.oT)
if args.a is not None:
algo = args.a
else:
algo = 'glmnet'
if args.dt == None:
args.dt = "1"
dirName = args.td.replace('/ro/','/wf/')
scriptName = args.e+"/train" + algo + "-td." + os.path.basename(os.path.abspath(args.td)) + "-dt." + args.dt + "-wt." + args.wt + attribute.generateExtension() +".r"
trainingDataList = attribute.getListOfTrainingDirectoriesNames(args.dt,dirName,args.iT)
trainingDataListString = ";".join(trainingDataList)
utility.runCommand([scriptName,"-d",trainingDataListString],args.run,args.sequence)
dirName = args.pd.replace('/ro/','/wf/')
scriptName=args.e+"/predict" + algo + "-td." + os.path.basename(os.path.abspath(args.td)) + "-dt." + args.dt +\
"-pd." + os.path.basename(os.path.abspath(args.pd)) + "-wt." + args.wt + attribute.generateExtension() +".r"
utility.runCommand([scriptName,"-d",dirName],args.run,args.sequence)
def scriptWrapper(index):
utility.runCommand(["./ob/quality/tradeE7BuySellMixMatchOptimized.py","-es",sellExperimentList[index],"-eb",buyExperimentList[index],"-skipT",args.skipT,"-a",args.a,"-entryCL",args.entryCL,"-exitCL",args.exitCL,"-orderQty",args.orderQty,'-dt',args.dt,"-targetClass",args.targetClass,"-td",args.td ,"-pd",args.pd,'-tickSize',args.tickSize,'-wt',args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
required=True,
help="default/exp , weight type to be given to different days")
parser.add_argument('-iT', required=False, help='Instrument name')
parser.add_argument('-sP', required=False, help='Strike price of instrument')
parser.add_argument('-oT', required=False, help='Options Type')
args = parser.parse_args()
attribute.initializeInstDetails(args.iT, args.sP, args.oT)
if args.a is not None:
algo = args.a
else:
algo = 'glmnet'
if args.dt == None:
args.dt = "1"
dirName = args.td.replace('/ro/', '/wf/')
scriptName = args.e + "/train" + algo + "-td." + os.path.basename(
os.path.abspath(args.td)
) + "-dt." + args.dt + "-wt." + args.wt + attribute.generateExtension() + ".r"
trainingDataList = attribute.getListOfTrainingDirectoriesNames(
args.dt, dirName, args.iT)
trainingDataListString = ";".join(trainingDataList)
utility.runCommand([scriptName, "-d", trainingDataListString], args.run,
args.sequence)
dirName = args.pd.replace('/ro/', '/wf/')
scriptName=args.e+"/predict" + algo + "-td." + os.path.basename(os.path.abspath(args.td)) + "-dt." + args.dt +\
"-pd." + os.path.basename(os.path.abspath(args.pd)) + "-wt." + args.wt + attribute.generateExtension() +".r"
utility.runCommand([scriptName, "-d", dirName], args.run, args.sequence)
generatorsFolder = args.g
commandList = []
for directories in lListOfTrainPredictDirectories:
commandList.append([
"aGenForE.py", "-e", experimentFolder, "-d", directories, "-g",
args.g, "-run", args.run, "-sequence", args.sequence, '-tickSize',
args.tickSize, "-iT", args.iT, "-oT", args.oT, "-sP", args.sP
])
utility.runCommandList(commandList, args)
print dp.printGroupStatus()
else:
results = map(scriptWrapperForFeatureGeneration,
lListOfTrainPredictDirectories)
utility.runCommand(["rGenForAllSubE.py","-e",args.e,"-a",algo,"-run",args.run,"-sequence",args.sequence,"-targetClass",args.targetClass,"-td",args.td , \
"-pd",args.pd,"-skipM",args.skipM,"-skipP",args.skipP,"-mpMearge",args.mpMearge,'-dt',args.dt, '-wt' , args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
if (args.sequence == "dp"):
print dp.printGroupStatus()
if (args.sequence == "dp"):
import runAllRScriptsForAllSubE
attribute.initializeInstDetails(args.iT, args.sP, args.oT)
if args.mpMearge.lower() == "yes":
commandList = runAllRScriptsForAllSubE.getTrainPredictCommandList(
args.e, args.a, args.td, args.pd, args.dt, args.wt)
utility.runCommandList(commandList, args)
print dp.printGroupStatus()
else:
commandList = runAllRScriptsForAllSubE.getTrainCommandList(
args.e, args.a, args.td, args.dt, args.wt)
utility.runCommandList(commandList, args)
def scriptWrapperForPredictRProgramGeneration(predictionDirAfterLastTD):
utility.runCommand(["pRGenForE.py","-e",args.e,"-s",lExperimentFolderName,"-a",args.a,"-skipP",args.skipP,"-td",args.td , "-pd" , predictionDirAfterLastTD , "-dt" , args.dt ,\
"-targetClass" , args.targetClass , '-wt' , args.wt ,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
else:
def scriptWrapperForFeatureGeneration(trainingDirectory):
utility.runCommand([
"aGenForE.py", "-e", l_exp_dir, "-d", trainingDirectory, "-g",
args.g, "-run", args.run, "-sequence", args.sequence, '-tickSize',
str(tickSize), "-iT", args.iT, "-oT", args.oT, "-sP", args.sP
], args.run, args.sequence)
pass
#results = map(scriptWrapperForFeatureGeneration,allDataDirectories)
pass
#==========R Code formation to find correlation between features and target file ==============================0
utility.runCommand([
"corrRGenForEForAllDays.py", "-e", l_exp_dir, "-td", args.td, "-dt",
args.dt, "-iT", args.iT, "-oT", args.oT, "-sP", args.sP
], args.run, args.sequence)
if args.sequence == "dp":
print dp.printGroupStatus()
#========Running the correlation R program=========================
allWorkingFileDirectories = attribute.getListOfTrainingDirectoriesNames(
int(args.nDays), args.td.replace('/ro/', '/wf/'), args.iT)
allWorkingFileDirectoriesString = ";".join(allWorkingFileDirectories)
lCorrCommandList = []
if args.sequence == "dp":
for l_training_day in allWorkingFileDirectories:
lDate = os.path.basename(os.path.abspath(l_training_day))
lFileName = l_exp_dir + "/corr-date-" + lDate + "-td." + os.path.basename(
os.path.abspath(args.td)
) + "-dt." + args.dt + attribute.generateExtension() + ".r"
def scriptWrapperForGeneratingOrderBook(trainingDirectory):
utility.runCommand(["generate_orderbook_from_nsedata_v15JanY14.py","-td",trainingDirectory,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP,'-uGE','no','-insType','Opt'],args.run,args.sequence)
generatorsFolder = args.g
commandList = []
for directories in lListOfTrainPredictDirectories:
commandList.append(["aGenForE.py","-e",experimentFolder,"-d",directories,"-g",args.g,"-run",args.run,"-sequence",args.sequence,'-tickSize',args.tickSize,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP])
utility.runCommandList( commandList ,args)
print dp.printGroupStatus()
else:
if args.sequence == 'lp':
# to run it in local parallel mode
pool = multiprocessing.Pool() # this will return the number of CPU's
results = map(scriptWrapperForFeatureGeneration,lListOfTrainPredictDirectories)
else:
results = map(scriptWrapperForFeatureGeneration,lListOfTrainPredictDirectories)
utility.runCommand(["rGenForE.py","-e",args.e,"-a",algo,"-sequence",args.sequence,"-targetClass",args.targetClass,"-skipM",args.skipM,\
'-dt',args.dt,'-pd',predictionDirectory,"-td",args.td,"-skipP",args.skipP, '-wt' , args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
utility.runCommand(["runAllRScriptsForE.py","-td",args.td,"-pd",predictionDirectory,"-dt",args.dt,"-e",args.e,"-a",algo,"-run",args.run,\
'-wt' , args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP,"-sequence",args.sequence],args.run,args.sequence)
if args.targetClass == "multinomial" :
utility.runCommand(["cMatrixGen.py","-d",predictionDirectory,"-e",args.e,"-a",algo],args.run,args.sequence)
utility.runCommand(["./ob/quality/tradeE5.py","-e",args.e,"-a",algo,"-entryCL",args.entryCL,"-exitCL",args.exitCL,"-orderQty",args.orderQty,"-skipT",args.skipT,\
'-dt',args.dt,"-targetClass",args.targetClass,"-td",args.td , "-pd",predictionDirectory,'-tickSize',args.tickSize,'-wt',args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
else:
utility.runCommand(["./ob/quality/tradeE7Optimized.py","-e",args.e,"-a",algo,"-entryCL",args.entryCL,"-exitCL",args.exitCL,"-orderQty",args.orderQty,"-skipT",args.skipT,'-dt',args.dt,"-targetClass",args.targetClass,"-td",args.td , "-pd",predictionDirectory,'-tickSize',args.tickSize,'-wt',args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
pass
if((args.e).find("nsefut") >= 0):
utility.runCommand(["accumulate_results.py","-e",args.e,"-a",algo,"-t",args.t,"-td",args.td, "-dt" , str(args.dt) ,"-pd",args.pd, "-m" , "NEW_FWATURE_TRY " , "-f" , "1","-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
print "NSEFUT"
else:
utility.runCommand(["accumulate_results.py","-e",args.e,"-a",algo,"-t",args.t,"-td","ob/data/ro/nsecur/20140903/", "-dt" , str(args.dt) ,"-nD", "26", "-m" , "NSE_CURRENCY_RESULTS_FOR_AB_And_SmartPrice" , "-f" , "1","-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
def wrapper(pCommandName): # we need this wrapper since pool.map has problems taking multiple arguments.
utility.runCommand(pCommandName,args.run,args.sequence)
def scriptWrapperForDayWiseCorrelation(pTrainingDay):
lDate = os.path.basename(os.path.abspath(pTrainingDay))
lFileName = l_exp_dir + "/corr-date-" + lDate + "-td." + os.path.basename(os.path.abspath(args.td)) + "-dt." + args.dt + attribute.generateExtension() +".r"
utility.runCommand([lFileName,'-d',pTrainingDay],args.run,args.sequence)
lListOfTrainPredictDirectories.append(args.pd)
if(args.sequence == "dp"):
experimentFolder = args.e
dataFolder = args.td
generatorsFolder = args.g
commandList = []
for directories in lListOfTrainPredictDirectories:
commandList.append(["aGenForE.py","-e",experimentFolder,"-d",directories,"-g",args.g,"-run",args.run,"-sequence",args.sequence,'-tickSize',args.tickSize,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP])
utility.runCommandList( commandList ,args)
print dp.printGroupStatus()
else:
results = map(scriptWrapperForFeatureGeneration,lListOfTrainPredictDirectories)
utility.runCommand(["rGenForAllSubE.py","-e",args.e,"-a",algo,"-run",args.run,"-sequence",args.sequence,"-targetClass",args.targetClass,"-td",args.td , \
"-pd",args.pd,"-skipM",args.skipM,"-skipP",args.skipP,"-mpMearge",args.mpMearge,'-dt',args.dt, '-wt' , args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
if(args.sequence == "dp"):
print dp.printGroupStatus()
if(args.sequence == "dp"):
import runAllRScriptsForAllSubE
attribute.initializeInstDetails(args.iT,args.sP,args.oT)
if args.mpMearge.lower() == "yes":
commandList = runAllRScriptsForAllSubE.getTrainPredictCommandList(args.e,args.a,args.td,args.pd,args.dt,args.wt)
utility.runCommandList(commandList,args)
print dp.printGroupStatus()
else:
commandList = runAllRScriptsForAllSubE.getTrainCommandList(args.e,args.a,args.td,args.dt,args.wt)
utility.runCommandList(commandList,args)
print dp.printGroupStatus()
commandList = []
# Seperate into 2 different list one for aGen and another for operateOnAttribute
if args.sequence=="dp":
for directories in allDataDirectories :
commandList.append(["aGenForE.py","-e",l_exp_dir,"-d",directories,"-g",args.g,"-run",args.run,"-sequence",args.sequence,'-tickSize',str(tickSize),"-iT",args.iT,"-oT",args.oT,"-sP",args.sP])
pass
utility.runListOfCommandsWithMaxUtlilizationOfWorkers(commandList,args,"CorrelationFeature Generation",int(args.nComputers))
else:
def scriptWrapperForFeatureGeneration(trainingDirectory):
utility.runCommand(["aGenForE.py","-e",l_exp_dir,"-d",trainingDirectory,"-g",args.g,"-run",args.run,"-sequence",args.sequence,'-tickSize',str(tickSize),"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
pass
#results = map(scriptWrapperForFeatureGeneration,allDataDirectories)
pass
#==========R Code formation to find correlation between features and target file ==============================0
utility.runCommand(["corrRGenForEForAllDays.py","-e",l_exp_dir,"-td",args.td,"-dt",args.dt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
if args.sequence=="dp":
print dp.printGroupStatus()
#========Running the correlation R program=========================
allWorkingFileDirectories = attribute.getListOfTrainingDirectoriesNames( int(args.nDays) , args.td.replace('/ro/','/wf/') ,args.iT)
allWorkingFileDirectoriesString = ";".join(allWorkingFileDirectories)
lCorrCommandList = []
if args.sequence == "dp":
for l_training_day in allWorkingFileDirectories:
lDate = os.path.basename(os.path.abspath(l_training_day))
lFileName = l_exp_dir + "/corr-date-" + lDate + "-td." + os.path.basename(os.path.abspath(args.td)) + "-dt." + args.dt + attribute.generateExtension() +".r"
lCorrCommandList.append([lFileName,'-d',l_training_day])
utility.runListOfCommandsWithMaxUtlilizationOfWorkers(lCorrCommandList,args,"Day-wise Correlation",int(args.nComputers))
else:
def scriptWrapperForDayWiseCorrelation(pTrainingDay):
def scriptWrapper(experimentName):
try:
utility.runCommand(["./ob/quality/tradeE7.py","-e",experimentName,"-skipT",args.skipT,"-a",algo,"-entryCL",entrylist,"-exitCL",exitlist,"-orderQty",args.orderQty,\
'-dt',args.dt,"-targetClass",args.targetClass,"-td",args.td , "-pd",args.pd,'-tickSize',args.tickSize,'-wt',args.wt],args.run,args.sequence)
except:
pass
def scriptWrapperForPredictRProgramGeneration(
predictionDirAfterLastTD):
utility.runCommand(["pRGenForE.py","-e",args.e,"-s",lExperimentFolderName,"-a",args.a,"-skipP",args.skipP,"-td",args.td , "-pd" , predictionDirAfterLastTD , "-dt" , args.dt ,\
"-targetClass" , args.targetClass , '-wt' , args.wt ,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
utility.runCommandList(lSubModelList,args)
print dp.printGroupStatus()
utility.runCommandList(lPGenRCodeList,args)
print dp.printGroupStatus()
if args.treeUsed.lower() == "yes":
for chunkNum in range(0,len(lTreeTrainingList),totalModelsWhichCanBeScheduled):
lSubTreeList = lTreeTrainingList[chunkNum:chunkNum+totalModelsWhichCanBeScheduled]
#utility.runCommandList(lSubTreeList,args)
#print dp.printGroupStatus()
if args.treeUsed.lower() == "yes":
for chunkNum in range(0,len(lTradingCommandList),totalModelsWhichCanBeScheduled):
lSubTradingList = lTradingCommandList[chunkNum:chunkNum+totalModelsWhichCanBeScheduled]
utility.runCommandList(lSubTradingList,args)
print dp.printGroupStatus()
else:
utility.runCommandList(lTradingCommandList,args)
print dp.printGroupStatus()
utility.runCommand(["accumulate_results.py","-e",args.e,"-a",algo,"-t",args.t,"-td",dataFolder, "-dt" , str(args.dt) , '-nD' , str(args.nDays) , "-m" ,"ICICI_BANK_experiments" , "-f" , "1","-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
indexOfFeatures = indexOfFeatures + 1
for i in range(len(allDataDirectories)-int(args.dt)):
args.td = allDataDirectories[i]
predictionDirLastTD = allDataDirectories[i + int(args.dt) - 1]
predictionDirAfterLastTD = allDataDirectories[i + int(args.dt)]
# utility.runCommand(["src/rsTradeBuySellMixMatch.py","-e",args.e,"-skipT",args.skipT,"-a",algo,"-entryCL", entrylist ,"-exitCL",exitlist,"-orderQty","300",'-dt',args.dt,"-targetClass",args.targetClass,"-td",args.td , "-pd",predictionDirAfterLastTD,'-tickSize',args.tickSize,'-wt',wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP,"-run",args.run,"-sequence",args.sequence],args.run,args.sequence)
# print dp.printGroupStatus()
# utility.runCommand(["accumulate_results.py","-e",args.e,"-a",algo,"-t",args.t,"-td",dataFolder, "-dt" , str(args.dt) , '-nD' , str(args.nDays) , "-m" ,"LiveExperimentTestingInCurrentMonthsData" , "-f" , "1","-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
def scriptWrapperForGeneratingOrderBook(trainingDirectory):
utility.runCommand([
"generate_orderbook_from_nsedata_v15JanY14.py", "-td",
trainingDirectory, "-iT", args.iT, "-oT", args.oT, "-sP", args.sP,
'-uGE', 'no', '-insType', 'Opt'
], args.run, args.sequence)
def scriptWrapperForPredictProgramRun(predictionDirAfterLastTD):
scriptName=lExperimentFolderName+"/predict" + args.a + "-td." + os.path.basename(os.path.abspath(args.td)) + "-dt." + args.dt +"-pd." +\
os.path.basename(os.path.abspath(predictionDirAfterLastTD)) + "-wt." + args.wt + attribute.generateExtension() +".r"
dirName = predictionDirAfterLastTD.replace('/ro/','/wf/')
utility.runCommand([scriptName,"-d",dirName],args.run,args.sequence)
def scriptWrapperForFeatureGeneration(trainingDirectory):
utility.runCommand(["aGenForE.py","-e",args.e,"-d",trainingDirectory,"-g",args.g,"-run",args.run,"-sequence",args.sequence,'-tickSize',args.tickSize,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
pass
def wrapper(
pCommandName
): # we need this wrapper since pool.map has problems taking multiple arguments.
utility.runCommand(pCommandName, args.run, args.sequence)
parser.add_argument(
'-wt',
required=False,
help="default/exp , weight type to be given to different days")
parser.add_argument('-iT', required=False, help='Instrument name')
parser.add_argument('-sP', required=False, help='Strike price of instrument')
parser.add_argument('-oT', required=False, help='Options Type')
args = parser.parse_args()
if args.skipM == None:
args.skipM = "yes"
if args.skipP == None:
args.skipP = "yes"
if args.dt == None:
args.dt = "1"
if args.wt == None:
args.wt = "default"
if args.a is not None:
allAlgos = [args.a]
else:
allAlgos = ['logitr', 'glmnet', 'randomForest']
for algo in allAlgos:
utility.runCommand(["mRGenForE.py","-e",args.e,"-a",algo,"-targetClass",args.targetClass,"-skipM",args.skipM,"-td",args.td, "-dt" , args.dt ,\
'-wt' , args.wt ,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
if algo <> 'bigRandomForest':
utility.runCommand(["pRGenForE.py","-e",args.e,"-a",algo,"-skipP",args.skipP,"-td",args.td , "-pd" , args.pd , "-dt" , args.dt ,\
"-targetClass" , args.targetClass , '-wt' , args.wt ,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
def scriptWrapper(experimentName):
try:
utility.runCommand(["./ob/quality/tradeE7.py","-e",experimentName,"-skipT",args.skipT,"-a",algo,"-entryCL",entrylist,"-exitCL",exitlist,"-orderQty",args.orderQty,\
'-dt',args.dt,"-targetClass",args.targetClass,"-td",args.td , "-pd",args.pd,'-tickSize',args.tickSize,'-wt',args.wt],args.run,args.sequence)
except:
pass
