def main():
try:
parser = MyOptParser()
parser.add_argument("-p","--processFile", dest="processFile", default="" ,
help="""Json file with process description.""", metavar="<path>", required=True)
opts= AttrMap(vars(parser.parse_args()))
if(opts.processFile):
processFile = cF.jsonLoad(opts.processFile);
# load process module/class
mod, ProcessClass = iH.importClassFromModule( **processFile["processClass"] )
process = ProcessClass( processFile, jobGenModules = {"importHelpers":iH, "commonFunctions" : cF} )
process.doProcessing()
return 0
except Exception as e:
sys.stdout.flush()
print("====================================================================", file=sys.stderr)
print("Exception occured here: " + str(e), file=sys.stderr)
print("====================================================================", file=sys.stderr)
traceback.print_exc(file=sys.stderr)
return 1
def main():
""" {old validatation file infos} is compared
to { new validation file infos} ==> outputs new file validation info
"""
parser = MyOptParser()
parser.add_argument("-s", "--searchDirNew", dest="searchDirNew",
help="""This is the search directory where it is looked for output files (.tiff,.exr,.rib.gz). """,
metavar="<path>", default=None, required=False)
parser.add_argument("--valFileInfoGlobNew", dest="valFileInfoGlobNew",
help="""
The globbing expression for all input xmls with file status which
are consolidated into a new file info under --output. The found and validated files in --searchDir (if specified) are
added to the set of new files.
""", default=None, metavar="<glob>", required=False)
parser.add_argument("--valFileInfoGlobOld", dest="valFileInfoGlobOld",
help="""
The globbing expression for all old input xmls with file status which
are consolidated with the new files into a combined file info under --output.
""", default=None, metavar="<glob>", required=False)
parser.add_argument("--pipelineSpecs", dest="pipelineSpecs", default="",
help="""Json file with info about the pipeline, fileValidation, fileValidationTools.
""", metavar="<string>", required=True)
parser.add_argument("--statusFolder", dest="statusFolder", default=None,
help="""The output status folder which contains links to files which are finished, or can be recovered.
""", metavar="<string>", required=False)
parser.add_argument("--validateOnlyLastModified", dest="validateOnlyLastModified", type=cF.toBool, default=True,
help="""The file with the moset recent modified time is only validated, all others are set to finished!.""", required=False)
parser.add_argument("-o", "--output", dest="output",
help="""The output xml which is written, which proivides validation info for each file found""", metavar="<path>", required=True)
try:
print("====================== FileValidation ===========================")
opts= AttrMap(vars(parser.parse_args()))
if not opts.searchDirNew and not opts.valFileInfoGlobNew:
raise ValueError("You need to define either searchDirNew or valFileInfoGlobNew!")
if opts.valFileInfoGlobOld == "":
opts.valFileInfoGlobOld = None
print("searchDir: %s" % opts.searchDirNew)
print("valFileInfoGlobNew: %s" % opts.valFileInfoGlobNew)
print("valFileInfoGlobOld: %s" % opts.valFileInfoGlobOld)
print("output: %s" % opts.output)
d = cF.jsonLoad(opts.pipelineSpecs)
pipelineTools = d["pipelineTools"]
fileValidationSpecs = d["fileValidationSpecs"]
fileValidationTools = d["fileValidationTools"]
valDataAllNew = dict()
deleteFiles = []
# load new validataion datas
if opts.valFileInfoGlobNew is not None:
print("Load new validation files")
valDataAllNew , valFilesNew = loadValidationFiles(opts.valFileInfoGlobNew)
preferGlobalPaths(valDataAllNew)
# add searchDir files to new set
# search files ============================================================================
if opts.searchDirNew is not None:
print("Validate all files in: %s with pipeLineSpecs: %s" % (opts.searchDirNew , opts.pipelineSpecs) )
allFiles = searchFiles(opts.searchDirNew, opts, fileValidationSpecs,fileValidationTools,pipelineTools)
for ha, f in allFiles.items():
if ha in valDataAllNew:
print("""WARNING: File %s already found in validation data set
from globbing expr. %s """ % (f["absPath"], opts.valFileInfoGlobNew))
else:
valDataAllNew[ha] = f
# ===============================================================================================
# load old validation datas
if opts.valFileInfoGlobOld is not None:
print("Load old validation files")
valDataAllOld , valFilesOld = loadValidationFiles(opts.valFileInfoGlobOld)
preferGlobalPaths(valDataAllOld)
# add old to new validatation infos
for ha, valInfo in valDataAllOld.items():
if ha not in valDataAllNew:
# this old file hash is not in our current list, so add it!
# check absPath if it exists otherwise try to extent the relPath with dir of this validation file.
if not os.path.exists(valInfo["absPath"]):
absPath = os.path.join( os.path.dirname(valInfo["validatationInfoPath"]) , valInfo["relPath"] )
if not os.path.exists(absPath):
print(valInfo["validatationInfoPath"])
raise NameError("""File path in valid. info file: %s
does not exist, extended rel. path to: %s does also not exist!""" % (valInfo["absPath"],absPath))
else:
print("Replacing inexisting path %s with %s", valInfo["absPath"], absPath)
valInfo["absPath"] = absPath
# copy element to new file info
valDataAllNew[ha] = valInfo
else:
# we have the same hash in the new info
# take our new one which is better!
# delete old file if it is not linked to by new file
if os.path.realpath(valDataAllNew[ha]["absPath"]) != os.path.realpath(valInfo["absPath"]):
deleteFiles.append(valInfo["absPath"])
# make final list
finalFiles = [ f for f in valDataAllNew.values() ]
printSummary(finalFiles,pipelineTools,False)
print("Make output validation file")
f = open(opts.output,"w+")
cF.jsonDump(finalFiles,f, sort_keys=True)
f.close();
# Renew status folder, move over new xml info
if opts.statusFolder is not None:
print("Renew status folder:")
finished = os.path.join(opts.statusFolder,"finished")
recover = os.path.join(opts.statusFolder,"recover")
cF.makeDirectory(finished,interact=False, defaultMakeEmpty=True)
cF.makeDirectory(recover ,interact=False, defaultMakeEmpty=True)
# make symlinks for all files in the appropriate folder:
paths = {"recover": recover, "finished": finished}
for f in finalFiles:
h = f["hash"]
p = os.path.relpath(f["absPath"],start=paths[f["status"]])
filename = os.path.basename(p)
head,ext = os.path.splitext(filename)
os.symlink(p, os.path.join( paths[f["status"]] , head+"-uuid-"+h+ext ) );
print("=================================================================")
except Exception as e:
print("====================================================================")
print("Exception occured: " + str(e))
print("====================================================================")
traceback.print_exc(file=sys.stdout)
parser.print_help()
return 1
def main():
global subProcess,shutdown
parser = MyOptParser()
try:
parser.add_argument("-c", "--renderCommand", dest="renderCommand", default="prman -t:1" ,
help="""The render command to call.""", metavar="<path>", required=True)
parser.add_argument("-p", "--processFile", dest="processFile", default="./RenderProcess.xml" ,
help="""Input file xml path with a list of files for rendering""", metavar="<path>", required=True)
opts= AttrMap(vars(parser.parse_args()))
print("================== Rendering Frames =========================")
print("Script: " + __file__ )
if(opts.processFile):
frames = cF.jsonLoad(opts.processFile);
if not frames:
raise ValueError("No frames specified in xml %s" % opts.processFile)
for f in frames:
if shutDown:
break
print("Render frame: %s " % f["inputFile"])
command = opts.renderCommand.split(" ") + [f["inputFile"]]
print("Command: %s" % str(" ".join(command)))
sys.stdout.flush()
try:
subProcess = subprocess.Popen(command, stderr=subprocess.STDOUT)
# Waiting for process
subProcess.wait()
subProcess = None
except Exception as c:
raise NameError("Rendering Process Error: for file: %s with render output: %s " % (f,c.output) )
if shutDown:
print("Render Loop shutdown")
print("============================================================")
except Exception as e:
print("====================================================================", file=sys.stderr)
print("Exception occured: " + str(e), file=sys.stderr)
print("====================================================================", file=sys.stderr)
traceback.print_exc(file=sys.stderr)
return 1
print("================== Rendering finished ==================== ")
return 0
def main():
parser = MyOptParser()
parser.add_argument("--pipelineSpecs", dest="pipelineSpecs", default="" ,
help="""Json file with info about the pipeline, fileValidation, fileValidationTools.""", metavar="<path>", required=True)
parser.add_argument("--validationFileInfo", dest="validationFileInfo", default="" ,
help="""XML file with info about render output files.""", metavar="<path>", required=False)
parser.add_argument("-p", "--processes", type=int, dest="processes", default=int(1),
help="The number of processes for the cluster render", metavar="<integer>", required=True)
try:
print("================== Prepare for Cluster Pipeline Job============")
opts= AttrMap(vars(parser.parse_args()))
pipelineSpecs = cF.jsonLoad(opts.pipelineSpecs)
pipelineTools = pipelineSpecs["pipelineTools"]
# tool1 ---> tool2 ----> tool3
# : dependency on tool2
# :tool3 is parent
# define parents and dependencies for all tools
for toolName,tool in pipelineTools.items():
if "dependencies" not in tool:
tool["dependencies"]=set()
tool["parents"]=set()
for toolName,tool in pipelineTools.items():
for dep in tool["dependencies"]:
t = pipelineTools[dep]
t["parents"].add(toolName)
frameGenerator = pipelineSpecs["frameGenerator"]
# fileValidationSpecs = d["fileValidationSpecs"]
# fileValidationTools = d["fileValidationTools"]
# Important job modules to hand over to frameGenerators and processFileWriters
importantModules = {"importHelpers":iH, "commonFunctions" : cF, "getSimFileInfos" : getSimFileInfos}
# Generate Frames =====================================================
mod, frameGenerator["generator"] = iH.importClassFromModuleString(frameGenerator["generator"])
# hand over some modules to the frame generator!
fgen = frameGenerator["generator"](pipelineSpecs, jobGenModules = importantModules )
allFrames,framesPerIdx, framesToDistribute = fgen(**frameGenerator["arguments"])
# =====================================================================
# Formatting frames ========================================================
# format strings in all settings (if possible) in allFrames again with itself
for i,fr in enumerate(allFrames):
allFrames[i] = cF.formatAll(fr,fr,formatter=StrExpr)
# Filter Frames =======================================================
recoverFrames(opts,allFrames,framesPerIdx,pipelineTools)
#======================================================================
# make a list of all frames which are not-completely finished
# (that are frames where all tools with no parent (the last one) are not finished, we need at least one )
notcompleted = lambda frame: sum( 1 if frame["tools"][toolName]["status"] != STATUS_FINISHED
else 0 for toolName,tool in pipelineTools.items() if len(tool["parents"])==0 ) > 0
framesCount = len(allFrames);
allFrames = list(filter(notcompleted, allFrames))
framesToDistribute = list(filter(notcompleted, framesToDistribute))
print("Removed %d finished frames!" % (framesCount - len(allFrames)) )
#count number of frames to render
totalFrames = len(framesToDistribute);
print("Number of frames to compute %i" % totalFrames)
if(totalFrames == 0):
print("No frames to distribute -> exit")
return 0
# Distribute the processes over the number of processes ===============
processFrames = distributeFrames(opts,framesToDistribute)
#======================================================================
# Write for each tool in the pipeline the process file, for each process a seperate one
for toolName,tool in pipelineTools.items():
# load the class and module for the tools processFileWriter
print("Load processFileGenerator for tool: %s" % toolName )
mod, tool["processFileGenerator"]["generator"] = iH.importClassFromModuleString(tool["processFileGenerator"]["generator"])
tool["processFileGenerator"]["generator"](pipelineSpecs, jobGenModules = importantModules).write(processFrames, **tool["processFileGenerator"]["arguments"])
# if we have some info file generator , produce the output
if "infoFileGenerator" in tool:
print("Load infoFileGenerator for tool: %s" % toolName )
mod, tool["infoFileGenerator"]["generator"] = iH.importClassFromModuleString(tool["infoFileGenerator"]["generator"])
tool["infoFileGenerator"]["generator"](pipelineSpecs, jobGenModules = importantModules).write(processFrames, **tool["infoFileGenerator"]["arguments"])
# Write FileMover process file =======================================
writeFileMoverProcessFile(pipelineSpecs,processFrames)
# =====================================================================
return 0
except Exception as e:
print("====================================================================")
print("Exception occured: " + str(e))
print("====================================================================")
traceback.print_exc(file=sys.stdout)
parser.print_help()
return 1
def recoverFrames(opts,allFrames,framesPerIdx, pipelineTools):
def addFile(frame,file,parent=None):
if "usedFile" in file and file["usedFile"] :
#this file has already been used
return
if file["status"]==STATUS_RECOVER:
print("added File: %s (recover)" % file["relPath"])
# add a file move to recover this file
frame["fileMover"].append(file["fileMoveRecover"])
# mark file as used
file["usedFile"] = True
elif file["status"]==STATUS_FINISHED:
if parent:
print("added File: %s (finished, dependent)" % file["relPath"])
# add a file move to recover this file
frame["fileMover"].append(file["fileMoveDependent"])
#print("id", id(frame), frame["fileMover"] )
# mark file as used
file["usedFile"] = True
def addTool(frame,toolName, visitedTools, parentToolName=None):
if toolName in visitedTools:
return
visitedTools.add(toolName);
frameTool = frame["tools"][toolName]
# if tool is not finished
if frameTool["status"] != STATUS_FINISHED:
# add all this tools checkpoint files
for outFileProp in frameTool["outputFiles"]:
if not outFileProp["cpFile"] == None:
addFile(frame,outFileProp["cpFile"],parentToolName)
# add all dependent tools
depTools = pipelineTools[toolName]["dependencies"]
if depTools:
for depTool in depTools:
addTool(frame,depTool,visitedTools,toolName)
else:
# we are finished, but
# if we have a parent tool (always not finished),
# we add our finished checkpoint files
if parentToolName:
# add all its checkpoint files of output files
for outFileProp in frameTool["outputFiles"]:
if outFileProp["cpFile"] == None:
raise ValueError("""Trying to add non existant checkpoint file of output file %s in tool
%s!""" % (str(outFileProp),toolName) )
addFile(frame,outFileProp["cpFile"],parentToolName)
#else:
# if no parent given, dont do anything
# get all file info
if opts.validationFileInfo:
print("Setup recovery from file info===============================")
print("Using validation file: %s", opts.validationFileInfo)
checkpointFiles = cF.jsonLoad(opts.validationFileInfo);
cpFiles = { "hash": {}, "all" : []}
for f in checkpointFiles:
tool = f["tool"]
fileId = f["hash"]
frameIdx = int(f["frameIdx"])
ha = f["hash"]
cpFiles["all"].append(f)
if ha in cpFiles["hash"]:
raise ValueError("File %s and %s have the same hash!" % (f["absPath"], cpFiles["hash"][ha]["absPath"] ) )
else:
cpFiles["hash"][ha] = f
print("===========================================================")
print("Determine status of all tools =============================")
# move over all frames, for each tool and match cpFiles
for frameIdx,frame in framesPerIdx.items():
finished = False;
for toolName,tool in frame["tools"].items():
# if there are checkpoint files corresponding to outputfiles of this tool
finishedOutFiles = 0
for outFileProp in tool["outputFiles"]:
ha = cF.makeUUID(outFileProp["hashString"])
outFileProp["hash"] = ha
if ha in cpFiles["hash"]: # we found checkpoint file
cpFile = cpFiles["hash"][ha]
absP = cpFile["absPath"]
print("Frame: %i " % frameIdx +
" checkpoint file matched:\n\t%s\n\thash: %s\n\tstatus: %s " % ( absP[:10]+'...'+absP[-20:] if len(absP) > 70 else absP ,ha, cpFile["status"] ))
outFileProp["cpFile"] = cpFile
if outFileProp["cpFile"]["status"] == STATUS_FINISHED:
finishedOutFiles += 1
else:
outFileProp["cpFile"] = None
# if all output files are finished -> tool is finished
if finishedOutFiles == len(tool["outputFiles"]):
tool["status"] = STATUS_FINISHED
print("Tool: %s -> finished" % toolName)
#print("Dependency check===========================================")
## for each frameIdx file list,
## travel dependency of each file and if some files are missing
## silently remove this file from the cpFilesPerFrame because this frameIdx can not be recovered!
#invalidFrameIdx = set() # list for not recoverable frames! (should not happen)
#for frameIdx,frame in framesPerIdx.items():
#if frameIdx not in cpFilesPerFrame:
#continue
#stack = cpFilesPerFrame[frameIdx]["all"][:] # shallow copy (remove files from stack)
#invalidFrame = walkDependencies(stack,cpFilesPerFrame[frameIdx])
#if invalidFrame:
#print("Invalid frameIdx: %i for recovery!" % frameIdx)
#invalidFrameIdx.add(frameIdx)
## continue to next frame
#continue
##endfor
## remove all files from all tools for invalid frames
#for k in invalidFrameIdx:
#for toolName,tool in framesPerIdx[k].items():
#if toolName in pipelineTools.keys():
#tool["checkpointFiles"] = []
#print("===========================================================")
# setup recovery for all frames
print("Setup pipeline tools with file info ========================")
for frame in allFrames:
# walk all tools in pipeline (visit all once!)
for tool in pipelineTools.keys():
addTool(frame,tool,set())
print("===============================================================")
def main():
parser = MyOptParser()
try:
parser.add_argument("-p", "--processFile", dest="processFile", default="./FileMove.xml" ,
help="""Input .json file with a list of files for moving into place""", metavar="<path>", required=True)
opts= AttrMap(vars(parser.parse_args()))
print("========================= Move Files ===========================")
if opts.processFile:
tasks = cF.jsonLoad(opts.processFile);
for task in tasks:
try:
# avoiding links to links (get the real path resolving all symlinks)
# if that is necessary is not known
if "from" in task:
task["realfrom"] = os.path.realpath(task["from"])
print("Task:", task)
if task["type"] == "hardlink":
try:
os.link(task["realfrom"],task["to"] )
continue
except:
task["type"] ="symlink"
if task["type"] == "symlink":
try:
os.symlink(task["realfrom"],task["to"] )
continue
except:
task["type"] ="copy"
if task["type"] == "copy":
shutil.copy2(task["from"],task["to"])
continue
if task["type"] == "makeDirs" :
os.makedirs(task["dir"],exist_ok=True)
continue
raise ValueError("Type: %s not supported!" % str(task))
except:
raise ValueError("Could not execute task: %s" % str(task))
print("================================================================")
except Exception as e:
print("====================================================================")
print("Exception occured: " + str(e))
print("====================================================================")
traceback.print_exc(file=sys.stdout)
return 1
return 0