def writeSingleTile(options): """Writes a single tile according to the options""" # Determine the name of the tile we need to write tileName = generateTileName(options.pixelStartX, options.pixelStartY, options.pixelStopX, options.pixelStopY) tilePath = os.path.join(options.workDir, tileName) # Just call the command for a single tile! cmd = ['mapproject', '--t_pixelwin', str(options.pixelStartX), str(options.pixelStartY), str(options.pixelStopX), str(options.pixelStopY), options.demPath, options.imagePath, tilePath] cmd = cmd + options.extraArgs # Append other options IrgSystemFunctions.executeCommand(cmd, options.suppressOutput) if options.convertTiles: # Make uint8 version of the tile for debugging tilePathU8 = os.path.splitext(tilePath)[0] + 'U8.tif' cmd = ['gdal_translate', '-ot', 'byte', '-scale', tilePath, tilePathU8] IrgSystemFunctions.executeCommand(cmd, options.suppressOutput) return 0
def handleArguments(args): """Split up arguments into required and optional lists which will be passed to subprocess""" requiredList = [] optionsList = [] # Loop through all entries. iterable = iter(range(0, len(args))) for i in iterable: a = args[i] if (i < len(args)-1): # Don't load the next value when we are at the end! n = args[i+1] else: n = '-' # This will just cause us to get out of the loop if IrgSystemFunctions.isCmdOption(a): # This is the start of an option. optionsList.append(a) # Record this entry. if IrgSystemFunctions.isCmdOption(n): # The next entry is the start of another option so this one has no values. continue optionsList.append(n) # Otherwise record the next entry as a value. iterable.next() # Skip the next entry in the loop. if (a == '--t_projwin') or (a == '--t_pixelwin'): # These arguments have four values, not just one. optionsList.append(args[i+2]) # Add the additional three arguments and skip them in the loop. optionsList.append(args[i+3]) optionsList.append(args[i+4]) iterable.next() iterable.next() iterable.next() else: # This is one of the three positional arguments requiredList.append(a) # Return the two lists return (requiredList, optionsList)
def main(argsIn): try: usage = "usage: pbs_parallel_stereo.py (same inputs as parallel_stereo plus new options)" parser = IrgSystemFunctions.PassThroughOptionParser( usage=usage) # Use parser that ignores unknown options parser.add_option( "--num-correlation-nodes", dest="numCorrelationNodes", type='int', default=1, help="Number of nodes to use for the two correlation steps") parser.add_option( "--num-triangulation-nodes", dest="numTriangulationNodes", type='int', default=1, help="Number of nodes to use for the triangulation steps") parser.add_option( '--node-type', dest='nodeType', default='wes', help='Type of processing node to request (wes, san, or ivy)') parser.add_option('--group-id', dest='groupId', default='', help='GID to charge the hours to [REQUIRED]') # This call handles all the specific options for this code. (options, args) = parser.parse_args(argsIn) # 'args' contains everything for parallel_stereo # Check the required positional arguments. if not options.groupId: parser.error("Must input a group ID to charge to!") # Any additional arguments need to be forwarded to the mapproject function options.extraArgs = args except optparse.OptionError, msg: raise Usage(msg)
':model=' + options.nodeType + ' -W depend=afterok:$subjob2 -- ' + scriptCalls[2] + ')') # Triangulation stage pbsStrings.append(corePbsString + ' -N pbs_stereo4 -l walltime="' + stepHours[3] + '"' + ' -e ' + errLogPaths[3] + ' -o ' + stdLogPaths[3] + ' -l select=' + str(options.numTriangulationNodes) + ':ncpus=' + str(cpusPerNode) + ':model=' + options.nodeType + ' -W depend=afterok:$subjob3 -- ' + scriptCalls[3]) # Set up the command line for parallel_stereo commandList = ['parallel_stereo', '--nodes-list', '$PBS_NODEFILE'] commandList = commandList + options.extraArgs # Append other options commandString = IrgSystemFunctions.argListToString(commandList) phases = [ ' --entry-point 0 --stop-point 1', # Init ' --entry-point 1 --stop-point 3', # Correlation ' --entry-point 3 --stop-point 4', # Filtering ' --entry-point 4 --stop-point 6' ] # Triangulation # Generate a set of four script files for i in range(4): print 'Writing script file ' + scriptCalls[i] scriptFile = open(scriptCalls[i], 'w') scriptFile.write('#!/bin/bash\n\n') thisCommandString = commandString + phases[i] scriptFile.write(thisCommandString)
argumentFile.close() # Indicate to GNU Parallel that there are multiple tab-seperated variables in the text file we just wrote parallelArgs = ['--colsep', "\\t"] # TODO: Move this to the lower level functions # If using multiple computers, need to make sure the same program paths are available if options.nodesListPath: parallelArgs = parallelArgs + ['--env', 'PATH', '--env', 'PYTHONPATH', '--env', 'ISISROOT', '--env', 'ISIS3DATA'] # Get the number of available nodes and CPUs per node numNodes = IrgPbsFunctions.getNumNodesInList(options.nodesListPath) # We assume all machines have the same number of CPUs (cores) cpusPerNode = IrgSystemFunctions.get_num_cpus() # We don't actually know the best number here! threadsPerCpu = 2 # Set the optimal number of processes if the user did not specify if not options.numProcesses: options.numProcesses = numNodes * cpusPerNode * threadsPerCpu # Note: mapproject can run with multiple threads on non-ISIS data but we don't use that # functionality here since we call mapproject with one tile at a time. # No need for more processes than their are tiles! if options.numProcesses > numTiles: options.numProcesses = numTiles
# Filtering stage pbsStrings.append('subjob3=$( ' + corePbsString + ' -N pbs_stereo3 -l walltime="'+stepHours[2]+'"' + ' -e '+ errLogPaths[2] +' -o '+ stdLogPaths[2] + ' -l select='+str(1)+':ncpus='+str(cpusPerNode)+':model='+options.nodeType + ' -W depend=afterok:$subjob2 -- '+ scriptCalls[2] +')') # Triangulation stage pbsStrings.append(corePbsString + ' -N pbs_stereo4 -l walltime="'+stepHours[3]+'"' + ' -e '+ errLogPaths[3] +' -o '+ stdLogPaths[3] + ' -l select='+str(options.numTriangulationNodes)+':ncpus='+str(cpusPerNode)+':model='+options.nodeType + ' -W depend=afterok:$subjob3 -- '+ scriptCalls[3]) # Set up the command line for parallel_stereo commandList = ['parallel_stereo', '--nodes-list', '$PBS_NODEFILE'] commandList = commandList + options.extraArgs # Append other options commandString = IrgSystemFunctions.argListToString(commandList) phases = [' --entry-point 0 --stop-point 1', # Init ' --entry-point 1 --stop-point 3', # Correlation ' --entry-point 3 --stop-point 4', # Filtering ' --entry-point 4 --stop-point 6'] # Triangulation # Generate a set of four script files for i in range(4): print 'Writing script file ' + scriptCalls[i] scriptFile = open(scriptCalls[i], 'w') scriptFile.write('#!/bin/bash\n\n') thisCommandString = commandString + phases[i] scriptFile.write(thisCommandString) scriptFile.close()