def getOutputFileList(self, fileType): '''Return a list containing all the output DEM files in the run and the associated frames''' batchList = self.getBatchFolderList() output = [] for batch in batchList: frames = icebridge_common.getFrameRangeFromBatchFolder(batch) path = os.path.join(batch, fileType) output.append((path, frames)) return output
def getOutputFileList(self, fileType): '''Return a list containing all the output DEM files in the run and the associated frames''' batchList = self.getBatchFolderList() output = [] for batch in batchList: frames = icebridge_common.getFrameRangeFromBatchFolder(batch) path = os.path.join(batch, fileType) output.append((path, frames)) return output
def packAndSendCompletedRun(run, logger): '''Assembles and compresses the deliverable parts of the run''' logger.info('Getting ready to pack up run ' + str(run)) cwd = os.getcwd() os.chdir(run.parentFolder) runFolder = str(run) # Use symlinks to assemble a fake file structure to tar up assemblyFolder = run.getAssemblyFolder() batchFolders = run.getBatchFolderList() os.system('mkdir -p ' + assemblyFolder) # Wipe any dead symlinks, as maybe this is not the first time the assembly is made pattern = os.path.join(assemblyFolder, '*') currFiles = glob.glob(pattern) for filename in currFiles: if not os.path.exists(os.path.realpath(filename)): logger.info("Will wipe dead link: " + filename) os.system("rm -f " + filename) # For each batch folder, start adding links to files that we want in the tarball for batch in batchFolders: # Skip folders where we did not produce final output finalDemFile = os.path.join(batch, icebridge_common.blendFileName()) if not os.path.exists(finalDemFile): continue # Need to change the name of these files when they go in the output folder (startFrame, stopFrame) = icebridge_common.getFrameRangeFromBatchFolder(batch) # We respect the convention below in push_to_nsidc.py. prefix = ('F_%05d_%05d' % (startFrame, stopFrame)) prefix = os.path.join(assemblyFolder, prefix) target = prefix + '_DEM.tif' try: if os.path.exists(target): os.system("rm -f " + target) # to wipe whatever was there os.symlink(finalDemFile, target) except Exception, e: logger.info( str(e) + " when doing: ln -s " + finalDemFile + " " + target)
def packAndSendCompletedRun(run, logger): '''Assembles and compresses the deliverable parts of the run''' logger.info('Getting ready to pack up run ' + str(run)) cwd = os.getcwd() os.chdir(run.parentFolder) runFolder = str(run) # Use symlinks to assemble a fake file structure to tar up assemblyFolder = run.getAssemblyFolder() batchFolders = run.getBatchFolderList() os.system('mkdir -p ' + assemblyFolder) # Wipe any dead symlinks, as maybe this is not the first time the assembly is made pattern = os.path.join(assemblyFolder, '*') currFiles = glob.glob(pattern) for filename in currFiles: if not os.path.exists(os.path.realpath(filename)): logger.info("Will wipe dead link: " + filename) os.system("rm -f " + filename) # For each batch folder, start adding links to files that we want in the tarball for batch in batchFolders: # Skip folders where we did not produce final output finalDemFile = os.path.join(batch, icebridge_common.blendFileName()) if not os.path.exists(finalDemFile): continue # Need to change the name of these files when they go in the output folder (startFrame, stopFrame) = icebridge_common.getFrameRangeFromBatchFolder(batch) # We respect the convention below in push_to_nsidc.py. prefix = ('F_%05d_%05d' % (startFrame, stopFrame)) prefix = os.path.join(assemblyFolder, prefix) target = prefix + '_DEM.tif' try: if os.path.exists(target): os.system("rm -f " + target) # to wipe whatever was there os.symlink(finalDemFile, target) except Exception as e: logger.info(str(e) + " when doing: ln -s " + finalDemFile + " " + target) # Tar up the assembled files and send them at the same time using the shiftc command # - No need to use a compression algorithm here fileName = run.getOutputTarName() lfePath = os.path.join(REMOTE_OUTPUT_FOLDER, fileName) logger.info('Sending run to lfe...') cmd = "ssh lfe 'rm -f " + stripHost(lfePath) + "' 2>/dev/null" logger.info(cmd) os.system(cmd) cmd = 'shiftc --wait -d -r --dereference --create-tar ' + \ os.path.join(runFolder, os.path.basename(assemblyFolder)) + ' ' + lfePath # Command to transfer the files as they are in the batch dirs without symlink #cmd = 'shiftc --wait -d -r --include=\'^.*?' + icebridge_common.blendFileName() + \ # '$\' --create-tar ' + runFolder + \ # ' ' + lfePath try: robust_shiftc(cmd, logger) except Exception as e: logger.info(str(e)) raise Exception('Failed to pack/send results for run ' + str(run) + \ '. Maybe not all sym links are valid.') os.chdir(cwd) logger.info('Finished sending run to lfe.')
def main(argsIn): try: usage = '''usage: multi_process_command_runner.py ...''' parser = argparse.ArgumentParser(usage=usage) # Data selection optios parser.add_argument('--start-frame', dest='startFrame', type=int, default=icebridge_common.getSmallestFrame(), help="Frame to start with. Leave this and stop-frame blank to " + \ "process all frames.") parser.add_argument('--stop-frame', dest='stopFrame', type=int, default=icebridge_common.getLargestFrame(), help='Frame to stop on. This last one will not be processed.') parser.add_argument('--num-processes', dest='numProcesses', type=int, default=-1, help='How many processes to start at the same time.') parser.add_argument("--command-file-path", dest="commandFilePath", default=None, help="The file from where to read the commands to process.") parser.add_argument("--force-redo-these-frames", dest="redoFrameList", default="", help="For each frame in this file (stored one per line) within the current frame range, delete the batch folder and redo the batch.") options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) icebridge_common.switchWorkDir() os.system("ulimit -c 0") # disable core dumps os.system("umask 022") # enforce files be readable by others if not os.path.exists(options.commandFilePath): print('Error: File ' + options.commandFilePath + ' does not exist!') return -1 # TODO: Write to a log? print('Starting processing pool with ' + str(options.numProcesses) +' processes.') pool = multiprocessing.Pool(options.numProcesses) taskHandles = [] framesToDo = set() if options.redoFrameList != "" and os.path.exists(options.redoFrameList): with open(options.redoFrameList, 'r') as f: text = f.read() for line in text.split('\n'): line = line.strip() if line == "": continue framesToDo.add(int(line)) # Open the file and loop through all the lines # - Count the lines as we go so we only process the desired lines print('Opening command file ' + options.commandFilePath) text = '' with open(options.commandFilePath, 'r') as f: text = f.read() for line in text.split('\n'): if line == "": continue (begFrame, endFrame) = icebridge_common.getFrameRangeFromBatchFolder(line) # Check line indices if begFrame >= options.startFrame and begFrame < options.stopFrame: if options.redoFrameList != "": if begFrame in framesToDo: folderName = icebridge_common.getBatchFolderFromBatchLine(line) if os.path.exists(folderName): print("will wipe " + folderName) cmd = "rm -rf " + folderName print(cmd) try: os.system(cmd) except Exception as e: pass else: print("Could not find " + folderName) else: print("Will skip frame: " + str(begFrame)) continue # Add the command to the task pool taskHandles.append(pool.apply_async(runCommand, (line,))) # Wait for all the tasks to complete print('Finished adding ' + str(len(taskHandles)) + ' tasks to the pool.') icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, interactive=False) # All tasks should be finished, clean up the processing pool icebridge_common.stopTaskPool(pool) print('Jobs finished.')
framesToDo.add(int(line)) # Open the file and loop through all the lines # - Count the lines as we go so we only process the desired lines print('Opening command file ' + options.commandFilePath) text = '' with open(options.commandFilePath, 'r') as f: text = f.read() for line in text.split('\n'): if line == "": continue (begFrame, endFrame) = icebridge_common.getFrameRangeFromBatchFolder(line) # Check line indices if begFrame >= options.startFrame and begFrame < options.stopFrame: if options.redoFrameList != "": if begFrame in framesToDo: folderName = icebridge_common.getBatchFolderFromBatchLine( line) if os.path.exists(folderName): print("will wipe " + folderName) cmd = "rm -rf " + folderName print(cmd) try: os.system(cmd) except Exception, e:
def main(argsIn): try: usage = '''usage: multi_process_command_runner.py ...''' parser = argparse.ArgumentParser(usage=usage) # Data selection options parser.add_argument('--start-frame', dest='startFrame', type=int, default=icebridge_common.getSmallestFrame(), help="Frame to start with. Leave this and stop-frame blank to " + \ "process all frames. Set both to None to blindly run all commands.") parser.add_argument( '--stop-frame', dest='stopFrame', type=int, default=icebridge_common.getLargestFrame(), help='Frame to stop on. This last one will not be processed.') parser.add_argument( '--num-processes', dest='numProcesses', type=int, default=-1, help='How many processes to start at the same time.') parser.add_argument( "--command-file-path", dest="commandFilePath", default=None, help="The file from where to read the commands to process.") parser.add_argument( "--force-redo-these-frames", dest="redoFrameList", default="", help= "For each frame in this file (stored one per line) within the current frame range, delete the batch folder and redo the batch." ) options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) icebridge_common.switchWorkDir() os.system("ulimit -c 0") # disable core dumps os.system("umask 022") # enforce files be readable by others if not os.path.exists(options.commandFilePath): print('Error: File ' + options.commandFilePath + ' does not exist!') return -1 # TODO: Write to a log? print('Starting processing pool with ' + str(options.numProcesses) + ' processes.') pool = multiprocessing.Pool(options.numProcesses) taskHandles = [] framesToDo = set() if options.redoFrameList != "" and os.path.exists(options.redoFrameList): with open(options.redoFrameList, 'r') as f: text = f.read() for line in text.split('\n'): line = line.strip() if line == "": continue framesToDo.add(int(line)) # Open the file and loop through all the lines # - Count the lines as we go so we only process the desired lines print('Opening command file ' + options.commandFilePath) text = '' with open(options.commandFilePath, 'r') as f: text = f.read() for line in text.split('\n'): if line == "": continue # If the frame range is turned off, just run the commands as-is. if (options.startFrame == None and options.stopFrame == None): # Add the command to the task pool taskHandles.append(pool.apply_async(runCommand, (line, ))) continue (begFrame, endFrame) = icebridge_common.getFrameRangeFromBatchFolder(line) # Check line indices if begFrame >= options.startFrame and begFrame < options.stopFrame: if options.redoFrameList != "": if begFrame in framesToDo: folderName = icebridge_common.getBatchFolderFromBatchLine( line) if os.path.exists(folderName): print("will wipe " + folderName) cmd = "rm -rf " + folderName print(cmd) try: os.system(cmd) except Exception as e: pass else: print("Could not find " + folderName) else: print("Will skip frame: " + str(begFrame)) continue # Add the command to the task pool taskHandles.append(pool.apply_async(runCommand, (line, ))) # Wait for all the tasks to complete print('Finished adding ' + str(len(taskHandles)) + ' tasks to the pool.') icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, interactive=False) # All tasks should be finished, clean up the processing pool icebridge_common.stopTaskPool(pool) print('Jobs finished.')