def getSymbolDataToFileInParallel(self, symbolArray, outFileArray, errorFile, numProcesses, numThreads):
"""
getSymbolDataToFileInParallel() : Using a linear extractor, launches multiple threads to extract data in parallel
Parameters are:
symbolArray: An array of symbols to download
outFileArray: An array of filenames (with path) where symbol data is to be saved
errorFile: An error file where any and all runtime errors are captured.
numThreads: An integer value between 1-32 of the number of threads to launch in parallel
"""
if (numProcesses > settings.MAX_PROCESSES):
numThreads = settings.MAX_PROCESSES
if (numThreads > settings.MAX_THREADS):
numThreads = settings.MAX_THREADS
if (len(symbolArray) < 1 or len(outFileArray) < 1 or len(symbolArray) != len(outFileArray)) :
raise Exception("Incorrect size of symbol or outputfile arrays")
print "Total symbols to download are : "+str(len(symbolArray))+'\n'
sliceSize = len(symbolArray)/numProcesses if len(symbolArray)%numProcesses == 0 else len(symbolArray)/(numProcesses-1)
myprocs = []
for x in range(numProcesses):
procName = "Process"+str(x)
start = sliceSize * x
if (x==numProcesses-1 and len(symbolArray)%numProcesses != 0):
print "Last Process will go with a length of : ",len(symbolArray)%(numProcesses-1)," instead of the sliceSize of ",sliceSize, "\n"
end = start + len(symbolArray)%(numProcesses-1) - 1
else:
end = start + sliceSize -1
ppr = ParallelProcessRunner(self.LinearExtractor)
pr = Process(target= ppr.launchThreadsForProcess, args=(symbolArray, outFileArray, errorFile+"_"+str(x), x, numThreads, start, end))
pr.procID = x
print "Launching Process number "+str(x) + "from "+str(start)+" to "+str(end)+'\n'
myprocs.append(pr)
for i in myprocs:
i.start()
""" Awaiting the threads to complete execution """
for i in myprocs:
i.join()
try:
final_error = open(errorFile, "w")
for i in myprocs:
err = open(errorFile+"_"+str(i.procID), "r")
final_error.write(err.read())
except:
print "Error writing output to the final error file \n"
