def getOutputFileList(self, fileType):
'''Return a list containing all the output DEM files in the run and the associated frames'''
batchList = self.getBatchFolderList()
output = []
for batch in batchList:
frames = icebridge_common.getFrameRangeFromBatchFolder(batch)
path = os.path.join(batch, fileType)
output.append((path, frames))
return output
def getOutputFileList(self, fileType):
'''Return a list containing all the output DEM files in the run and the associated frames'''
batchList = self.getBatchFolderList()
output = []
for batch in batchList:
frames = icebridge_common.getFrameRangeFromBatchFolder(batch)
path = os.path.join(batch, fileType)
output.append((path, frames))
return output
def packAndSendCompletedRun(run, logger):
'''Assembles and compresses the deliverable parts of the run'''
logger.info('Getting ready to pack up run ' + str(run))
cwd = os.getcwd()
os.chdir(run.parentFolder)
runFolder = str(run)
# Use symlinks to assemble a fake file structure to tar up
assemblyFolder = run.getAssemblyFolder()
batchFolders = run.getBatchFolderList()
os.system('mkdir -p ' + assemblyFolder)
# Wipe any dead symlinks, as maybe this is not the first time the assembly is made
pattern = os.path.join(assemblyFolder, '*')
currFiles = glob.glob(pattern)
for filename in currFiles:
if not os.path.exists(os.path.realpath(filename)):
logger.info("Will wipe dead link: " + filename)
os.system("rm -f " + filename)
# For each batch folder, start adding links to files that we want in the tarball
for batch in batchFolders:
# Skip folders where we did not produce final output
finalDemFile = os.path.join(batch, icebridge_common.blendFileName())
if not os.path.exists(finalDemFile):
continue
# Need to change the name of these files when they go in the output folder
(startFrame,
stopFrame) = icebridge_common.getFrameRangeFromBatchFolder(batch)
# We respect the convention below in push_to_nsidc.py.
prefix = ('F_%05d_%05d' % (startFrame, stopFrame))
prefix = os.path.join(assemblyFolder, prefix)
target = prefix + '_DEM.tif'
try:
if os.path.exists(target):
os.system("rm -f " + target) # to wipe whatever was there
os.symlink(finalDemFile, target)
except Exception, e:
logger.info(
str(e) + " when doing: ln -s " + finalDemFile + " " + target)
def packAndSendCompletedRun(run, logger):
'''Assembles and compresses the deliverable parts of the run'''
logger.info('Getting ready to pack up run ' + str(run))
cwd = os.getcwd()
os.chdir(run.parentFolder)
runFolder = str(run)
# Use symlinks to assemble a fake file structure to tar up
assemblyFolder = run.getAssemblyFolder()
batchFolders = run.getBatchFolderList()
os.system('mkdir -p ' + assemblyFolder)
# Wipe any dead symlinks, as maybe this is not the first time the assembly is made
pattern = os.path.join(assemblyFolder, '*')
currFiles = glob.glob(pattern)
for filename in currFiles:
if not os.path.exists(os.path.realpath(filename)):
logger.info("Will wipe dead link: " + filename)
os.system("rm -f " + filename)
# For each batch folder, start adding links to files that we want in the tarball
for batch in batchFolders:
# Skip folders where we did not produce final output
finalDemFile = os.path.join(batch, icebridge_common.blendFileName())
if not os.path.exists(finalDemFile):
continue
# Need to change the name of these files when they go in the output folder
(startFrame, stopFrame) = icebridge_common.getFrameRangeFromBatchFolder(batch)
# We respect the convention below in push_to_nsidc.py.
prefix = ('F_%05d_%05d' % (startFrame, stopFrame))
prefix = os.path.join(assemblyFolder, prefix)
target = prefix + '_DEM.tif'
try:
if os.path.exists(target):
os.system("rm -f " + target) # to wipe whatever was there
os.symlink(finalDemFile, target)
except Exception as e:
logger.info(str(e) + " when doing: ln -s " + finalDemFile + " " + target)
# Tar up the assembled files and send them at the same time using the shiftc command
# - No need to use a compression algorithm here
fileName = run.getOutputTarName()
lfePath = os.path.join(REMOTE_OUTPUT_FOLDER, fileName)
logger.info('Sending run to lfe...')
cmd = "ssh lfe 'rm -f " + stripHost(lfePath) + "' 2>/dev/null"
logger.info(cmd)
os.system(cmd)
cmd = 'shiftc --wait -d -r --dereference --create-tar ' + \
os.path.join(runFolder, os.path.basename(assemblyFolder)) + ' ' + lfePath
# Command to transfer the files as they are in the batch dirs without symlink
#cmd = 'shiftc --wait -d -r --include=\'^.*?' + icebridge_common.blendFileName() + \
# '$\' --create-tar ' + runFolder + \
# ' ' + lfePath
try:
robust_shiftc(cmd, logger)
except Exception as e:
logger.info(str(e))
raise Exception('Failed to pack/send results for run ' + str(run) + \
'. Maybe not all sym links are valid.')
os.chdir(cwd)
logger.info('Finished sending run to lfe.')
def main(argsIn):
try:
usage = '''usage: multi_process_command_runner.py ...'''
parser = argparse.ArgumentParser(usage=usage)
# Data selection optios
parser.add_argument('--start-frame', dest='startFrame', type=int,
default=icebridge_common.getSmallestFrame(),
help="Frame to start with. Leave this and stop-frame blank to " + \
"process all frames.")
parser.add_argument('--stop-frame', dest='stopFrame', type=int,
default=icebridge_common.getLargestFrame(),
help='Frame to stop on. This last one will not be processed.')
parser.add_argument('--num-processes', dest='numProcesses', type=int,
default=-1,
help='How many processes to start at the same time.')
parser.add_argument("--command-file-path", dest="commandFilePath", default=None,
help="The file from where to read the commands to process.")
parser.add_argument("--force-redo-these-frames", dest="redoFrameList", default="",
help="For each frame in this file (stored one per line) within the current frame range, delete the batch folder and redo the batch.")
options = parser.parse_args(argsIn)
except argparse.ArgumentError as msg:
parser.error(msg)
icebridge_common.switchWorkDir()
os.system("ulimit -c 0") # disable core dumps
os.system("umask 022") # enforce files be readable by others
if not os.path.exists(options.commandFilePath):
print('Error: File ' + options.commandFilePath + ' does not exist!')
return -1
# TODO: Write to a log?
print('Starting processing pool with ' + str(options.numProcesses) +' processes.')
pool = multiprocessing.Pool(options.numProcesses)
taskHandles = []
framesToDo = set()
if options.redoFrameList != "" and os.path.exists(options.redoFrameList):
with open(options.redoFrameList, 'r') as f:
text = f.read()
for line in text.split('\n'):
line = line.strip()
if line == "":
continue
framesToDo.add(int(line))
# Open the file and loop through all the lines
# - Count the lines as we go so we only process the desired lines
print('Opening command file ' + options.commandFilePath)
text = ''
with open(options.commandFilePath, 'r') as f:
text = f.read()
for line in text.split('\n'):
if line == "":
continue
(begFrame, endFrame) = icebridge_common.getFrameRangeFromBatchFolder(line)
# Check line indices
if begFrame >= options.startFrame and begFrame < options.stopFrame:
if options.redoFrameList != "":
if begFrame in framesToDo:
folderName = icebridge_common.getBatchFolderFromBatchLine(line)
if os.path.exists(folderName):
print("will wipe " + folderName)
cmd = "rm -rf " + folderName
print(cmd)
try:
os.system(cmd)
except Exception as e:
pass
else:
print("Could not find " + folderName)
else:
print("Will skip frame: " + str(begFrame))
continue
# Add the command to the task pool
taskHandles.append(pool.apply_async(runCommand, (line,)))
# Wait for all the tasks to complete
print('Finished adding ' + str(len(taskHandles)) + ' tasks to the pool.')
icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, interactive=False)
# All tasks should be finished, clean up the processing pool
icebridge_common.stopTaskPool(pool)
print('Jobs finished.')
framesToDo.add(int(line))
# Open the file and loop through all the lines
# - Count the lines as we go so we only process the desired lines
print('Opening command file ' + options.commandFilePath)
text = ''
with open(options.commandFilePath, 'r') as f:
text = f.read()
for line in text.split('\n'):
if line == "":
continue
(begFrame,
endFrame) = icebridge_common.getFrameRangeFromBatchFolder(line)
# Check line indices
if begFrame >= options.startFrame and begFrame < options.stopFrame:
if options.redoFrameList != "":
if begFrame in framesToDo:
folderName = icebridge_common.getBatchFolderFromBatchLine(
line)
if os.path.exists(folderName):
print("will wipe " + folderName)
cmd = "rm -rf " + folderName
print(cmd)
try:
os.system(cmd)
except Exception, e:
def main(argsIn):
try:
usage = '''usage: multi_process_command_runner.py ...'''
parser = argparse.ArgumentParser(usage=usage)
# Data selection options
parser.add_argument('--start-frame', dest='startFrame', type=int,
default=icebridge_common.getSmallestFrame(),
help="Frame to start with. Leave this and stop-frame blank to " + \
"process all frames. Set both to None to blindly run all commands.")
parser.add_argument(
'--stop-frame',
dest='stopFrame',
type=int,
default=icebridge_common.getLargestFrame(),
help='Frame to stop on. This last one will not be processed.')
parser.add_argument(
'--num-processes',
dest='numProcesses',
type=int,
default=-1,
help='How many processes to start at the same time.')
parser.add_argument(
"--command-file-path",
dest="commandFilePath",
default=None,
help="The file from where to read the commands to process.")
parser.add_argument(
"--force-redo-these-frames",
dest="redoFrameList",
default="",
help=
"For each frame in this file (stored one per line) within the current frame range, delete the batch folder and redo the batch."
)
options = parser.parse_args(argsIn)
except argparse.ArgumentError as msg:
parser.error(msg)
icebridge_common.switchWorkDir()
os.system("ulimit -c 0") # disable core dumps
os.system("umask 022") # enforce files be readable by others
if not os.path.exists(options.commandFilePath):
print('Error: File ' + options.commandFilePath + ' does not exist!')
return -1
# TODO: Write to a log?
print('Starting processing pool with ' + str(options.numProcesses) +
' processes.')
pool = multiprocessing.Pool(options.numProcesses)
taskHandles = []
framesToDo = set()
if options.redoFrameList != "" and os.path.exists(options.redoFrameList):
with open(options.redoFrameList, 'r') as f:
text = f.read()
for line in text.split('\n'):
line = line.strip()
if line == "":
continue
framesToDo.add(int(line))
# Open the file and loop through all the lines
# - Count the lines as we go so we only process the desired lines
print('Opening command file ' + options.commandFilePath)
text = ''
with open(options.commandFilePath, 'r') as f:
text = f.read()
for line in text.split('\n'):
if line == "":
continue
# If the frame range is turned off, just run the commands as-is.
if (options.startFrame == None and options.stopFrame == None):
# Add the command to the task pool
taskHandles.append(pool.apply_async(runCommand, (line, )))
continue
(begFrame,
endFrame) = icebridge_common.getFrameRangeFromBatchFolder(line)
# Check line indices
if begFrame >= options.startFrame and begFrame < options.stopFrame:
if options.redoFrameList != "":
if begFrame in framesToDo:
folderName = icebridge_common.getBatchFolderFromBatchLine(
line)
if os.path.exists(folderName):
print("will wipe " + folderName)
cmd = "rm -rf " + folderName
print(cmd)
try:
os.system(cmd)
except Exception as e:
pass
else:
print("Could not find " + folderName)
else:
print("Will skip frame: " + str(begFrame))
continue
# Add the command to the task pool
taskHandles.append(pool.apply_async(runCommand, (line, )))
# Wait for all the tasks to complete
print('Finished adding ' + str(len(taskHandles)) + ' tasks to the pool.')
icebridge_common.waitForTaskCompletionOrKeypress(taskHandles,
interactive=False)
# All tasks should be finished, clean up the processing pool
icebridge_common.stopTaskPool(pool)
print('Jobs finished.')