def scriptWrapperForDayWiseCorrelation(pTrainingDay): lDate = os.path.basename(os.path.abspath(pTrainingDay)) lFileName = l_exp_dir + "/corr-date-" + lDate + "-td." + os.path.basename( os.path.abspath(args.td) ) + "-dt." + args.dt + attribute.generateExtension() + ".r" utility.runCommand([lFileName, '-d', pTrainingDay], args.run, args.sequence)
def scripWrapperForTradingCommand(predictionDirAfterLastTD): global tradeEngine for lQty in lQtyList: utility.runCommand([tradeEngine,"-e",args.e1, "-e1",args.e,"-skipT",args.skipT,"-a",args.a,"-entryCL",entrylist,"-exitCL",\ exitlist,"-entryCL2",entrycl2list,"-exitCL2",exitcl2list,"-orderQty",lQty,'-dt',args.dt,"-targetClass",args.targetClass,"-td",args.td , "-pd",predictionDirAfterLastTD,\ '-tickSize',args.tickSize,'-wt',args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP , "-ls", args.ls],args.run,args.sequence)
def scriptWrapperForFeatureGeneration(trainingDirectory): utility.runCommand([ "aGenForE.py", "-e", args.e, "-d", trainingDirectory, "-g", args.g, "-run", args.run, "-sequence", args.sequence, '-tickSize', args.tickSize, "-iT", args.iT, "-oT", args.oT, "-sP", args.sP ], args.run, args.sequence) pass
def scriptWrapperForGeneratingOrderBook(trainingDirectory): utility.runCommand([ "generate_orderbook_with_bands_6level.py", "-td", trainingDirectory, "-iT", args.iT, "-oT", args.oT, "-sP", args.sP, '-bGap', args.bGap, '-uGE', 'no' ], args.run, args.sequence) pass
def scriptWrapper(experimentName): if args.targetClass == "multinomial" : # utility.runCommand(["cMatrixGen.py","-d",args.pd,"-e",experimentName,"-a",algo],args.run,args.sequence) utility.runCommand(["./ob/quality/tradeE5.py","-e",experimentName,"-skipT",args.skipT,"-a",algo,"-entryCL","55;90;60;50","-exitCL","45;50;40;25","-orderQty","500",\ '-dt',args.dt,"-targetClass",args.targetClass,"-td",args.td , "-pd",args.pd,'-tickSize',args.tickSize,'-wt',args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence) else: utility.runCommand(["./ob/quality/tradeE7Optimized.py","-e",experimentName,"-skipT",args.skipT,"-a",algo,"-entryCL",entrylist,"-exitCL",exitlist,"-orderQty","300", '-dt',args.dt,"-targetClass",args.targetClass,"-td",args.td , "-pd",args.pd,'-tickSize',args.tickSize,'-wt',args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence) pass
def scripWrapperForTradingCommand(predictionDirAfterLastTD): for lQty in lQtyList: # utility.runCommand(["./ob/quality/tradeE7Optimized.py","-e",lExperimentFolderName,"-skipT",args.skipT,"-a",args.a,"-entryCL",entrylist,"-exitCL",exitlist,"-orderQty",lQty,'-dt',args.dt,"-targetClass",args.targetClass,"-td",args.td , "-pd",predictionDirAfterLastTD,'-tickSize',args.tickSize,'-wt',args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP ],args.run,args.sequence) utility.runCommand(["./ob/quality/tradeE7WithMultipleProbabilitiesAndQuickExit.py","-e",lExperimentFolderName,"-skipT",args.skipT,"-a",args.a,"-entryCL",entrylist,"-exitCL",\ exitlist,"-entryCL2",entrycl2list,"-exitCL2",exitcl2list,"-orderQty",lQty,'-dt',args.dt,"-targetClass",args.targetClass,"-td",args.td , "-pd",predictionDirAfterLastTD,\ '-tickSize',args.tickSize,'-wt',args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP ],args.run,args.sequence)
def scriptWrapper(index): utility.runCommand([ "./ob/quality/tradeE7BuySellMixMatchOptimized.py", "-es", sellExperimentList[index], "-eb", buyExperimentList[index], "-skipT", args.skipT, "-a", args.a, "-entryCL", args.entryCL, "-exitCL", args.exitCL, "-orderQty", args.orderQty, '-dt', args.dt, "-targetClass", args.targetClass, "-td", args.td, "-pd", args.pd, '-tickSize', args.tickSize, '-wt', args.wt, "-iT", args.iT, "-oT", args.oT, "-sP", args.sP ], args.run, args.sequence)
def scriptWrapper(experimentName): if args.targetClass == "multinomial": # utility.runCommand(["cMatrixGen.py","-d",args.pd,"-e",experimentName,"-a",algo],args.run,args.sequence) utility.runCommand(["./ob/quality/tradeE5.py","-e",experimentName,"-skipT",args.skipT,"-a",algo,"-entryCL","55;90;60;50","-exitCL","45;50;40;25","-orderQty","500",\ '-dt',args.dt,"-targetClass",args.targetClass,"-td",args.td , "-pd",args.pd,'-tickSize',args.tickSize,'-wt',args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence) else: utility.runCommand([ "./ob/quality/tradeE7Optimized.py", "-e", experimentName, "-skipT", args.skipT, "-a", algo, "-entryCL", entrylist, "-exitCL", exitlist, "-orderQty", "300", '-dt', args.dt, "-targetClass", args.targetClass, "-td", args.td, "-pd", args.pd, '-tickSize', args.tickSize, '-wt', args.wt, "-iT", args.iT, "-oT", args.oT, "-sP", args.sP ], args.run, args.sequence) pass
def bot_loop(): while connected: response = s.recv(1024).decode("utf-8") if response == "PING :tmi.twitch.tv\r\n": s.send("PONG :tmi.twitch.tv\r\n".encode("utf-8")) print("Pong") else: username = re.search(r"\w+", response).group(0) message = CHAT_MSG.sub("", response) print(username + ": " + message) # Ban pattern check for pat in pattern.BAN_PAT: if re.match(pat, message): utility.ban(s, username) utility.chat(s,"Tap, tap, tap. Nevermore. " + username + " banned") break # Time out pattern check for pat in pattern.TO_PAT: if re.match(pat, message): utility.timeout(s, username) utility.chat(s,"Caw caw! " + username + " silence! You know what you've done...") break # Command check if re.match(r'^(![A-Z,a-z])\w', message): # Check if command exists if message.strip() in command_list: utility.runCommand(s, message.strip(), command_list[message.strip()]) else: # New command check if re.match(r'^(![A-Z,a-z])\w+\s([A-Z,a-z])', message): command = message.split(" ", 1)[0] action = message.split(" ", 1)[1] utility.newCommand(s, command, action, command_list.keys()) command_list[command] = action print("A new command: " + command + " action: " + action) else: utility.chat(s, "Command doesn't exists") time.sleep(1 / config.RATE)
pool = multiprocessing.Pool( ) # this will return the number of CPU's results = pool.map(scriptWrapperForPredictRProgramGeneration, predictionDaysDirectory) results = pool.map(scriptWrapperForPredictProgramRun, predictionDaysDirectory) results = pool.map(scripWrapperForTradingCommand, predictionDaysDirectory) else: print "None" # results = map(scriptWrapperForPredictRProgramGeneration,predictionDaysDirectory) # results = map(scriptWrapperForPredictRProgramGenerationDepth,predictionDaysDirectory) # results = map(scriptWrapperForPredictProgramRun,predictionDaysDirectory) # results = map(scriptWrapperForPredictProgramRunDepth,predictionDaysDirectory) results = map(scripWrapperForTradingCommand, predictionDaysDirectory) if args.iT is not None: message = args.iT + "_ForExperimnet_" + lExperimentFolderName + "-Results. With DEPTH Pred" else: message = "ml_experiments" utility.runCommand([ "accumulate_results.py", "-e", args.e1, "-a", args.a, "-t", args.t, "-td", args.td, "-dt", "1", '-nD', str(args.nDays), "-m", message, "-f", "1", "-iT", args.iT, "-oT", args.oT, "-sP", args.sP ], args.run, args.sequence) if args.sequence == "dp": print dp.printGroupStatus()
args.skipM = "yes" if args.skipP == None: args.skipP = "yes" if args.dt == None: args.dt = "1" if args.wt == None: args.wt = "default" if args.a is not None: allAlgos = [args.a] else: allAlgos = ['logitr','glmnet','randomForest'] config = ConfigObj(args.e+"/design.ini") targetAttributes = attribute.getTargetVariableKeys(config) one_feature_attributes = attribute.getFeatureVariableKeys(config , targetAttributes.keys()[0]) totalNumberOfFeatures = len(one_feature_attributes) i = 2 for algo in allAlgos: while i <= totalNumberOfFeatures: if args.mpMearge == "yes": utility.runCommand(["mpRGenForAllSubE.py","-e",args.e,"-a",algo,"-targetClass",args.targetClass,"-pd",args.pd ,"-skipP",args.skipP ,"-skipM",args.skipM,"-td",args.td,\ "-dt" , args.dt , '-wt' , args.wt , "-s",args.e+"/s/"+str(i)+"c","-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence) else: utility.runCommand(["mRGenForAllSubE.py","-e",args.e,"-a",algo,"-targetClass",args.targetClass,"-td" , args.td , "-dt", args.dt,"-skipM",args.skipM,\ '-wt' , args.wt,"-s",args.e+"/s/"+str(i)+"c","-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence) utility.runCommand(["pRGenForAllSubE.py","-e",args.e,"-a",algo,"-skipP",args.skipP ,"-targetClass",args.targetClass,"-pd",args.pd ,"-td" , args.td ,\ "-dt", args.dt, "-wt", args.wt , "-s",args.e+"/s/"+str(i)+"c","-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence) i +=1
def scriptWrapperForPredictProgramRun(predictionDirAfterLastTD): scriptName=lExperimentFolderName+"/predict" + args.a + "-td." + os.path.basename(os.path.abspath(args.td)) + "-dt." + args.dt +"-pd." +\ os.path.basename(os.path.abspath(predictionDirAfterLastTD)) + "-wt." + args.wt + attribute.generateExtension() +".r" dirName = predictionDirAfterLastTD.replace('/ro/', '/wf/') utility.runCommand([scriptName, "-d", dirName], args.run, args.sequence)
if args.targetClass == None: args.targetClass = "binomial" print "Since no class of target variable is specified so taking binomial class of target variable" def scriptWrapperForFeatureGeneration(trainingDirectory): utility.runCommand(["aGenForE.py","-e",args.e,"-d",trainingDirectory,"-g",args.g,"-run",args.run,"-sequence",args.sequence,'-tickSize',args.tickSize],args.run,args.sequence) trainingDirectory = attribute.getTrainDirFromPredictDir( args.dt , args.pd , args.pType ) lListOfTrainingDirectories = attribute.getListOfTrainingDirectoriesNames(args.dt,trainingDirectory) lListOfTrainPredictDirectories = lListOfTrainingDirectories lListOfTrainPredictDirectories.append(args.pd) results = map(scriptWrapperForFeatureGeneration,lListOfTrainPredictDirectories) utility.runCommand(["rGenForE.py","-e",args.e,"-a",algo,"-sequence",args.sequence,"-targetClass",args.targetClass,"-skipM",args.skipM,\ '-dt',args.dt,'-pd',args.pd,"-td",trainingDirectory,"-skipP",args.skipP, '-wt' , args.wt],args.run,args.sequence) utility.runCommand(["runAllRScriptsForE.py","-td",trainingDirectory,"-pd",args.pd,"-dt",args.dt,"-e",args.e,"-a",algo,"-run",args.run,\ '-wt' , args.wt,"-sequence",args.sequence],args.run,args.sequence) utility.runCommand(["./ob/quality/tradeE7Optimized.py","-e",args.e,"-a",algo,"-entryCL",args.entryCL,"-exitCL",args.exitCL,"-orderQty",args.orderQty,\ '-dt',args.dt,"-targetClass",args.targetClass,"-td",trainingDirectory , "-pd",args.pd,'-tickSize',args.tickSize,'-wt',args.wt],args.run,args.sequence) instrGroupList = args.instrGroups.split(";") if args.pType.lower()== "same": modelValueFileName = args.e+'/'+algo+ '-td.' + os.path.basename(os.path.abspath(trainingDirectory)) + '-dt.' + args.dt + '-targetClass.' + \ args.targetClass + "-wt." + args.wt +'.coef' modelFp = open(modelValueFileName,"r") lines = modelFp.readlines() modelIniFile = args.e + '/' + "model-parameters.ini" modelIniFP = open(modelIniFile,"w") writeOutputToRemotePC.clear_file_from_remote_PC('/home/', "model-parameters.ini", ('1.ps.eo.spalgo.com', 'root', 'omshriganeshaya')) for n in instrGroupList:
def scriptWrapperForTradeGeneration(TargetNumber): utility.runCommand(["./ob/quality/tradeE7OnTargetVariable.py", "-orderQty", args.orderQty, "-d", args.d,"-startTime", args.startTime ,"-endTime",args.endTime ,"-tickSize",args.tickSize,\ "-targetType",TargetNumber,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP,"-e",args.e],args.run,args.sequence)
args.tickSize, "-iT", args.iT, "-oT", args.oT, "-sP", args.sP ]) utility.runCommandList(commandList, args) print dp.printGroupStatus() else: if args.sequence == 'lp': # to run it in local parallel mode pool = multiprocessing.Pool() # this will return the number of CPU's results = map(scriptWrapperForFeatureGeneration, lListOfTrainPredictDirectories) else: results = map(scriptWrapperForFeatureGeneration, lListOfTrainPredictDirectories) utility.runCommand(["rGenForE.py","-e",args.e,"-a",algo,"-sequence",args.sequence,"-targetClass",args.targetClass,"-skipM",args.skipM,\ '-dt',args.dt,'-pd',predictionDirectory,"-td",args.td,"-skipP",args.skipP, '-wt' , args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence) utility.runCommand(["runAllRScriptsForE.py","-td",args.td,"-pd",predictionDirectory,"-dt",args.dt,"-e",args.e,"-a",algo,"-run",args.run,\ '-wt' , args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP,"-sequence",args.sequence],args.run,args.sequence) if args.targetClass == "multinomial": utility.runCommand( ["cMatrixGen.py", "-d", predictionDirectory, "-e", args.e, "-a", algo], args.run, args.sequence) utility.runCommand(["./ob/quality/tradeE5.py","-e",args.e,"-a",algo,"-entryCL",args.entryCL,"-exitCL",args.exitCL,"-orderQty",args.orderQty,"-skipT",args.skipT,\ '-dt',args.dt,"-targetClass",args.targetClass,"-td",args.td , "-pd",predictionDirectory,'-tickSize',args.tickSize,'-wt',args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence) else: utility.runCommand([ "./ob/quality/tradeE7Optimized.py", "-e", args.e, "-a", algo, "-entryCL", args.entryCL, "-exitCL", args.exitCL, "-orderQty", args.orderQty, "-skipT", args.skipT, '-dt', args.dt, "-targetClass", args.targetClass, "-td", args.td, "-pd", predictionDirectory, '-tickSize', args.tickSize, '-wt', args.wt, "-iT", args.iT, "-oT",
parser.add_argument('-iT',required=False,help='Instrument name') parser.add_argument('-sP',required=False,help='Strike price of instrument') parser.add_argument('-oT',required=False,help='Options Type') args = parser.parse_args() if(args.sequence == "dp"): import dp attribute.initializeInstDetails(args.iT,args.sP,args.oT) commandList = [] allDataDirectories = attribute.getListOfTrainingDirectoriesNames( int(args.nDays) , args.d,args.iT ) for directories in allDataDirectories: commandList.append(['src/aGenForEWithTargetOnly.py','-d',directories,'-g','ob/generators/','-run',args.run,'-sequence',args.sequence,'-tickSize',args.tickSize,'-e',args.e ,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP]) pass if(args.sequence != "dp"): utility.runCommandList(commandList,args) pass else: utility.runListOfCommandsWithMaxUtlilizationOfWorkers(commandList,args,"Target Generation",int(args.nComputers)) commandList = [] for directories in allDataDirectories: commandList.append(['src/targetVariableRun.py','-orderQty',args.orderQty,'-d',directories,'-tickSize',args.tickSize,'-targetType',args.targetType,'-e',args.e,'-run',args.run,'-sequence',args.sequence,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP]) pass if(args.sequence != "dp"): utility.runCommandList(commandList,args) else: utility.runListOfCommandsWithMaxUtlilizationOfWorkers(commandList,args,"Trade OnTarget Variable",int(args.nComputers)) utility.runCommand(["src/accumulate_results_for_target_testing.py","-e",args.e,"-t",args.t,"-d",args.d, '-nD' , str(args.nDays) , "-m" , "ResultOfNewTargetVariableWhereWeForBuyWeCheckGreaterThanAskAndSellLessThanBid","-iT",args.iT,"-sP",args.sP,"-oT",args.oT],args.run,args.sequence) if(args.sequence == 'dp'): print dp.printGroupStatus()
def scriptWrapperForGeneratingOrderBook(trainingDirectory): utility.runCommand(["generate_orderbook_with_bands_6level.py","-td",trainingDirectory,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP,'-bGap',args.bGap,'-uGE','no'],args.run,args.sequence) pass
parser.add_argument('-sequence', required=True,help='lp / dp / serial') parser.add_argument('-dt',required=False,help='No of day from start for which it is to be trained ') parser.add_argument('-wt',required=True,help="default/exp , weight type to be given to different days") parser.add_argument('-iT',required=False,help='Instrument name') parser.add_argument('-sP',required=False,help='Strike price of instrument') parser.add_argument('-oT',required=False,help='Options Type') args = parser.parse_args() attribute.initializeInstDetails(args.iT,args.sP,args.oT) if args.a is not None: algo = args.a else: algo = 'glmnet' if args.dt == None: args.dt = "1" dirName = args.td.replace('/ro/','/wf/') scriptName = args.e+"/train" + algo + "-td." + os.path.basename(os.path.abspath(args.td)) + "-dt." + args.dt + "-wt." + args.wt + attribute.generateExtension() +".r" trainingDataList = attribute.getListOfTrainingDirectoriesNames(args.dt,dirName,args.iT) trainingDataListString = ";".join(trainingDataList) utility.runCommand([scriptName,"-d",trainingDataListString],args.run,args.sequence) dirName = args.pd.replace('/ro/','/wf/') scriptName=args.e+"/predict" + algo + "-td." + os.path.basename(os.path.abspath(args.td)) + "-dt." + args.dt +\ "-pd." + os.path.basename(os.path.abspath(args.pd)) + "-wt." + args.wt + attribute.generateExtension() +".r" utility.runCommand([scriptName,"-d",dirName],args.run,args.sequence)
def scriptWrapper(index): utility.runCommand(["./ob/quality/tradeE7BuySellMixMatchOptimized.py","-es",sellExperimentList[index],"-eb",buyExperimentList[index],"-skipT",args.skipT,"-a",args.a,"-entryCL",args.entryCL,"-exitCL",args.exitCL,"-orderQty",args.orderQty,'-dt',args.dt,"-targetClass",args.targetClass,"-td",args.td ,"-pd",args.pd,'-tickSize',args.tickSize,'-wt',args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
required=True, help="default/exp , weight type to be given to different days") parser.add_argument('-iT', required=False, help='Instrument name') parser.add_argument('-sP', required=False, help='Strike price of instrument') parser.add_argument('-oT', required=False, help='Options Type') args = parser.parse_args() attribute.initializeInstDetails(args.iT, args.sP, args.oT) if args.a is not None: algo = args.a else: algo = 'glmnet' if args.dt == None: args.dt = "1" dirName = args.td.replace('/ro/', '/wf/') scriptName = args.e + "/train" + algo + "-td." + os.path.basename( os.path.abspath(args.td) ) + "-dt." + args.dt + "-wt." + args.wt + attribute.generateExtension() + ".r" trainingDataList = attribute.getListOfTrainingDirectoriesNames( args.dt, dirName, args.iT) trainingDataListString = ";".join(trainingDataList) utility.runCommand([scriptName, "-d", trainingDataListString], args.run, args.sequence) dirName = args.pd.replace('/ro/', '/wf/') scriptName=args.e+"/predict" + algo + "-td." + os.path.basename(os.path.abspath(args.td)) + "-dt." + args.dt +\ "-pd." + os.path.basename(os.path.abspath(args.pd)) + "-wt." + args.wt + attribute.generateExtension() +".r" utility.runCommand([scriptName, "-d", dirName], args.run, args.sequence)
generatorsFolder = args.g commandList = [] for directories in lListOfTrainPredictDirectories: commandList.append([ "aGenForE.py", "-e", experimentFolder, "-d", directories, "-g", args.g, "-run", args.run, "-sequence", args.sequence, '-tickSize', args.tickSize, "-iT", args.iT, "-oT", args.oT, "-sP", args.sP ]) utility.runCommandList(commandList, args) print dp.printGroupStatus() else: results = map(scriptWrapperForFeatureGeneration, lListOfTrainPredictDirectories) utility.runCommand(["rGenForAllSubE.py","-e",args.e,"-a",algo,"-run",args.run,"-sequence",args.sequence,"-targetClass",args.targetClass,"-td",args.td , \ "-pd",args.pd,"-skipM",args.skipM,"-skipP",args.skipP,"-mpMearge",args.mpMearge,'-dt',args.dt, '-wt' , args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence) if (args.sequence == "dp"): print dp.printGroupStatus() if (args.sequence == "dp"): import runAllRScriptsForAllSubE attribute.initializeInstDetails(args.iT, args.sP, args.oT) if args.mpMearge.lower() == "yes": commandList = runAllRScriptsForAllSubE.getTrainPredictCommandList( args.e, args.a, args.td, args.pd, args.dt, args.wt) utility.runCommandList(commandList, args) print dp.printGroupStatus() else: commandList = runAllRScriptsForAllSubE.getTrainCommandList( args.e, args.a, args.td, args.dt, args.wt) utility.runCommandList(commandList, args)
def scriptWrapperForPredictRProgramGeneration(predictionDirAfterLastTD): utility.runCommand(["pRGenForE.py","-e",args.e,"-s",lExperimentFolderName,"-a",args.a,"-skipP",args.skipP,"-td",args.td , "-pd" , predictionDirAfterLastTD , "-dt" , args.dt ,\ "-targetClass" , args.targetClass , '-wt' , args.wt ,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
else: def scriptWrapperForFeatureGeneration(trainingDirectory): utility.runCommand([ "aGenForE.py", "-e", l_exp_dir, "-d", trainingDirectory, "-g", args.g, "-run", args.run, "-sequence", args.sequence, '-tickSize', str(tickSize), "-iT", args.iT, "-oT", args.oT, "-sP", args.sP ], args.run, args.sequence) pass #results = map(scriptWrapperForFeatureGeneration,allDataDirectories) pass #==========R Code formation to find correlation between features and target file ==============================0 utility.runCommand([ "corrRGenForEForAllDays.py", "-e", l_exp_dir, "-td", args.td, "-dt", args.dt, "-iT", args.iT, "-oT", args.oT, "-sP", args.sP ], args.run, args.sequence) if args.sequence == "dp": print dp.printGroupStatus() #========Running the correlation R program========================= allWorkingFileDirectories = attribute.getListOfTrainingDirectoriesNames( int(args.nDays), args.td.replace('/ro/', '/wf/'), args.iT) allWorkingFileDirectoriesString = ";".join(allWorkingFileDirectories) lCorrCommandList = [] if args.sequence == "dp": for l_training_day in allWorkingFileDirectories: lDate = os.path.basename(os.path.abspath(l_training_day)) lFileName = l_exp_dir + "/corr-date-" + lDate + "-td." + os.path.basename( os.path.abspath(args.td) ) + "-dt." + args.dt + attribute.generateExtension() + ".r"
def scriptWrapperForGeneratingOrderBook(trainingDirectory): utility.runCommand(["generate_orderbook_from_nsedata_v15JanY14.py","-td",trainingDirectory,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP,'-uGE','no','-insType','Opt'],args.run,args.sequence)
generatorsFolder = args.g commandList = [] for directories in lListOfTrainPredictDirectories: commandList.append(["aGenForE.py","-e",experimentFolder,"-d",directories,"-g",args.g,"-run",args.run,"-sequence",args.sequence,'-tickSize',args.tickSize,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP]) utility.runCommandList( commandList ,args) print dp.printGroupStatus() else: if args.sequence == 'lp': # to run it in local parallel mode pool = multiprocessing.Pool() # this will return the number of CPU's results = map(scriptWrapperForFeatureGeneration,lListOfTrainPredictDirectories) else: results = map(scriptWrapperForFeatureGeneration,lListOfTrainPredictDirectories) utility.runCommand(["rGenForE.py","-e",args.e,"-a",algo,"-sequence",args.sequence,"-targetClass",args.targetClass,"-skipM",args.skipM,\ '-dt',args.dt,'-pd',predictionDirectory,"-td",args.td,"-skipP",args.skipP, '-wt' , args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence) utility.runCommand(["runAllRScriptsForE.py","-td",args.td,"-pd",predictionDirectory,"-dt",args.dt,"-e",args.e,"-a",algo,"-run",args.run,\ '-wt' , args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP,"-sequence",args.sequence],args.run,args.sequence) if args.targetClass == "multinomial" : utility.runCommand(["cMatrixGen.py","-d",predictionDirectory,"-e",args.e,"-a",algo],args.run,args.sequence) utility.runCommand(["./ob/quality/tradeE5.py","-e",args.e,"-a",algo,"-entryCL",args.entryCL,"-exitCL",args.exitCL,"-orderQty",args.orderQty,"-skipT",args.skipT,\ '-dt',args.dt,"-targetClass",args.targetClass,"-td",args.td , "-pd",predictionDirectory,'-tickSize',args.tickSize,'-wt',args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence) else: utility.runCommand(["./ob/quality/tradeE7Optimized.py","-e",args.e,"-a",algo,"-entryCL",args.entryCL,"-exitCL",args.exitCL,"-orderQty",args.orderQty,"-skipT",args.skipT,'-dt',args.dt,"-targetClass",args.targetClass,"-td",args.td , "-pd",predictionDirectory,'-tickSize',args.tickSize,'-wt',args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence) pass if((args.e).find("nsefut") >= 0): utility.runCommand(["accumulate_results.py","-e",args.e,"-a",algo,"-t",args.t,"-td",args.td, "-dt" , str(args.dt) ,"-pd",args.pd, "-m" , "NEW_FWATURE_TRY " , "-f" , "1","-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence) print "NSEFUT" else: utility.runCommand(["accumulate_results.py","-e",args.e,"-a",algo,"-t",args.t,"-td","ob/data/ro/nsecur/20140903/", "-dt" , str(args.dt) ,"-nD", "26", "-m" , "NSE_CURRENCY_RESULTS_FOR_AB_And_SmartPrice" , "-f" , "1","-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
def wrapper(pCommandName): # we need this wrapper since pool.map has problems taking multiple arguments. utility.runCommand(pCommandName,args.run,args.sequence)
def scriptWrapperForDayWiseCorrelation(pTrainingDay): lDate = os.path.basename(os.path.abspath(pTrainingDay)) lFileName = l_exp_dir + "/corr-date-" + lDate + "-td." + os.path.basename(os.path.abspath(args.td)) + "-dt." + args.dt + attribute.generateExtension() +".r" utility.runCommand([lFileName,'-d',pTrainingDay],args.run,args.sequence)
lListOfTrainPredictDirectories.append(args.pd) if(args.sequence == "dp"): experimentFolder = args.e dataFolder = args.td generatorsFolder = args.g commandList = [] for directories in lListOfTrainPredictDirectories: commandList.append(["aGenForE.py","-e",experimentFolder,"-d",directories,"-g",args.g,"-run",args.run,"-sequence",args.sequence,'-tickSize',args.tickSize,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP]) utility.runCommandList( commandList ,args) print dp.printGroupStatus() else: results = map(scriptWrapperForFeatureGeneration,lListOfTrainPredictDirectories) utility.runCommand(["rGenForAllSubE.py","-e",args.e,"-a",algo,"-run",args.run,"-sequence",args.sequence,"-targetClass",args.targetClass,"-td",args.td , \ "-pd",args.pd,"-skipM",args.skipM,"-skipP",args.skipP,"-mpMearge",args.mpMearge,'-dt',args.dt, '-wt' , args.wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence) if(args.sequence == "dp"): print dp.printGroupStatus() if(args.sequence == "dp"): import runAllRScriptsForAllSubE attribute.initializeInstDetails(args.iT,args.sP,args.oT) if args.mpMearge.lower() == "yes": commandList = runAllRScriptsForAllSubE.getTrainPredictCommandList(args.e,args.a,args.td,args.pd,args.dt,args.wt) utility.runCommandList(commandList,args) print dp.printGroupStatus() else: commandList = runAllRScriptsForAllSubE.getTrainCommandList(args.e,args.a,args.td,args.dt,args.wt) utility.runCommandList(commandList,args) print dp.printGroupStatus()
commandList = [] # Seperate into 2 different list one for aGen and another for operateOnAttribute if args.sequence=="dp": for directories in allDataDirectories : commandList.append(["aGenForE.py","-e",l_exp_dir,"-d",directories,"-g",args.g,"-run",args.run,"-sequence",args.sequence,'-tickSize',str(tickSize),"-iT",args.iT,"-oT",args.oT,"-sP",args.sP]) pass utility.runListOfCommandsWithMaxUtlilizationOfWorkers(commandList,args,"CorrelationFeature Generation",int(args.nComputers)) else: def scriptWrapperForFeatureGeneration(trainingDirectory): utility.runCommand(["aGenForE.py","-e",l_exp_dir,"-d",trainingDirectory,"-g",args.g,"-run",args.run,"-sequence",args.sequence,'-tickSize',str(tickSize),"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence) pass #results = map(scriptWrapperForFeatureGeneration,allDataDirectories) pass #==========R Code formation to find correlation between features and target file ==============================0 utility.runCommand(["corrRGenForEForAllDays.py","-e",l_exp_dir,"-td",args.td,"-dt",args.dt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence) if args.sequence=="dp": print dp.printGroupStatus() #========Running the correlation R program========================= allWorkingFileDirectories = attribute.getListOfTrainingDirectoriesNames( int(args.nDays) , args.td.replace('/ro/','/wf/') ,args.iT) allWorkingFileDirectoriesString = ";".join(allWorkingFileDirectories) lCorrCommandList = [] if args.sequence == "dp": for l_training_day in allWorkingFileDirectories: lDate = os.path.basename(os.path.abspath(l_training_day)) lFileName = l_exp_dir + "/corr-date-" + lDate + "-td." + os.path.basename(os.path.abspath(args.td)) + "-dt." + args.dt + attribute.generateExtension() +".r" lCorrCommandList.append([lFileName,'-d',l_training_day]) utility.runListOfCommandsWithMaxUtlilizationOfWorkers(lCorrCommandList,args,"Day-wise Correlation",int(args.nComputers)) else: def scriptWrapperForDayWiseCorrelation(pTrainingDay):
def scriptWrapper(experimentName): try: utility.runCommand(["./ob/quality/tradeE7.py","-e",experimentName,"-skipT",args.skipT,"-a",algo,"-entryCL",entrylist,"-exitCL",exitlist,"-orderQty",args.orderQty,\ '-dt',args.dt,"-targetClass",args.targetClass,"-td",args.td , "-pd",args.pd,'-tickSize',args.tickSize,'-wt',args.wt],args.run,args.sequence) except: pass
def scriptWrapperForPredictRProgramGeneration( predictionDirAfterLastTD): utility.runCommand(["pRGenForE.py","-e",args.e,"-s",lExperimentFolderName,"-a",args.a,"-skipP",args.skipP,"-td",args.td , "-pd" , predictionDirAfterLastTD , "-dt" , args.dt ,\ "-targetClass" , args.targetClass , '-wt' , args.wt ,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
utility.runCommandList(lSubModelList,args) print dp.printGroupStatus() utility.runCommandList(lPGenRCodeList,args) print dp.printGroupStatus() if args.treeUsed.lower() == "yes": for chunkNum in range(0,len(lTreeTrainingList),totalModelsWhichCanBeScheduled): lSubTreeList = lTreeTrainingList[chunkNum:chunkNum+totalModelsWhichCanBeScheduled] #utility.runCommandList(lSubTreeList,args) #print dp.printGroupStatus() if args.treeUsed.lower() == "yes": for chunkNum in range(0,len(lTradingCommandList),totalModelsWhichCanBeScheduled): lSubTradingList = lTradingCommandList[chunkNum:chunkNum+totalModelsWhichCanBeScheduled] utility.runCommandList(lSubTradingList,args) print dp.printGroupStatus() else: utility.runCommandList(lTradingCommandList,args) print dp.printGroupStatus() utility.runCommand(["accumulate_results.py","-e",args.e,"-a",algo,"-t",args.t,"-td",dataFolder, "-dt" , str(args.dt) , '-nD' , str(args.nDays) , "-m" ,"ICICI_BANK_experiments" , "-f" , "1","-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence) indexOfFeatures = indexOfFeatures + 1 for i in range(len(allDataDirectories)-int(args.dt)): args.td = allDataDirectories[i] predictionDirLastTD = allDataDirectories[i + int(args.dt) - 1] predictionDirAfterLastTD = allDataDirectories[i + int(args.dt)] # utility.runCommand(["src/rsTradeBuySellMixMatch.py","-e",args.e,"-skipT",args.skipT,"-a",algo,"-entryCL", entrylist ,"-exitCL",exitlist,"-orderQty","300",'-dt',args.dt,"-targetClass",args.targetClass,"-td",args.td , "-pd",predictionDirAfterLastTD,'-tickSize',args.tickSize,'-wt',wt,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP,"-run",args.run,"-sequence",args.sequence],args.run,args.sequence) # print dp.printGroupStatus() # utility.runCommand(["accumulate_results.py","-e",args.e,"-a",algo,"-t",args.t,"-td",dataFolder, "-dt" , str(args.dt) , '-nD' , str(args.nDays) , "-m" ,"LiveExperimentTestingInCurrentMonthsData" , "-f" , "1","-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)
def scriptWrapperForGeneratingOrderBook(trainingDirectory): utility.runCommand([ "generate_orderbook_from_nsedata_v15JanY14.py", "-td", trainingDirectory, "-iT", args.iT, "-oT", args.oT, "-sP", args.sP, '-uGE', 'no', '-insType', 'Opt' ], args.run, args.sequence)
def scriptWrapperForPredictProgramRun(predictionDirAfterLastTD): scriptName=lExperimentFolderName+"/predict" + args.a + "-td." + os.path.basename(os.path.abspath(args.td)) + "-dt." + args.dt +"-pd." +\ os.path.basename(os.path.abspath(predictionDirAfterLastTD)) + "-wt." + args.wt + attribute.generateExtension() +".r" dirName = predictionDirAfterLastTD.replace('/ro/','/wf/') utility.runCommand([scriptName,"-d",dirName],args.run,args.sequence)
def scriptWrapperForFeatureGeneration(trainingDirectory): utility.runCommand(["aGenForE.py","-e",args.e,"-d",trainingDirectory,"-g",args.g,"-run",args.run,"-sequence",args.sequence,'-tickSize',args.tickSize,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence) pass
def wrapper( pCommandName ): # we need this wrapper since pool.map has problems taking multiple arguments. utility.runCommand(pCommandName, args.run, args.sequence)
parser.add_argument( '-wt', required=False, help="default/exp , weight type to be given to different days") parser.add_argument('-iT', required=False, help='Instrument name') parser.add_argument('-sP', required=False, help='Strike price of instrument') parser.add_argument('-oT', required=False, help='Options Type') args = parser.parse_args() if args.skipM == None: args.skipM = "yes" if args.skipP == None: args.skipP = "yes" if args.dt == None: args.dt = "1" if args.wt == None: args.wt = "default" if args.a is not None: allAlgos = [args.a] else: allAlgos = ['logitr', 'glmnet', 'randomForest'] for algo in allAlgos: utility.runCommand(["mRGenForE.py","-e",args.e,"-a",algo,"-targetClass",args.targetClass,"-skipM",args.skipM,"-td",args.td, "-dt" , args.dt ,\ '-wt' , args.wt ,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence) if algo <> 'bigRandomForest': utility.runCommand(["pRGenForE.py","-e",args.e,"-a",algo,"-skipP",args.skipP,"-td",args.td , "-pd" , args.pd , "-dt" , args.dt ,\ "-targetClass" , args.targetClass , '-wt' , args.wt ,"-iT",args.iT,"-oT",args.oT,"-sP",args.sP],args.run,args.sequence)