Ejemplo n.º 1
0
def main():

    try: 
        parser = MyOptParser()
        parser.add_argument("-p","--processFile", dest="processFile", default="" ,
                help="""Json file with process description.""", metavar="<path>", required=True)    
        
        
        opts= AttrMap(vars(parser.parse_args()))
          
        if(opts.processFile):
            
            processFile = cF.jsonLoad(opts.processFile);

            # load process module/class
            mod, ProcessClass = iH.importClassFromModule( **processFile["processClass"] )

            process = ProcessClass( processFile, jobGenModules = {"importHelpers":iH, "commonFunctions" : cF} )
            process.doProcessing()
           
        return 0
            
    except Exception as e:
        sys.stdout.flush()
        print("====================================================================", file=sys.stderr)
        print("Exception occured here: " + str(e), file=sys.stderr)
        print("====================================================================", file=sys.stderr)
        traceback.print_exc(file=sys.stderr)
        return 1
def main():
    
    """ {old validatation file infos}  is compared 
        to { new validation file infos}  ==> outputs new file validation info
    """
    
    parser = MyOptParser()
    
    parser.add_argument("-s", "--searchDirNew", dest="searchDirNew",
            help="""This is the search directory where it is looked for output files (.tiff,.exr,.rib.gz). """, 
            metavar="<path>", default=None, required=False)
    
    
    parser.add_argument("--valFileInfoGlobNew", dest="valFileInfoGlobNew",
            help="""
            The globbing expression for all input xmls with file status which 
            are consolidated into a new file info under --output. The found and validated files in --searchDir (if specified) are
            added to the set of new files.
            """, default=None, metavar="<glob>", required=False)
    
    parser.add_argument("--valFileInfoGlobOld", dest="valFileInfoGlobOld",
            help="""
            The globbing expression for all old input xmls with file status which 
            are consolidated with the new files into a combined file info under --output.
            """, default=None, metavar="<glob>", required=False)
    
    parser.add_argument("--pipelineSpecs", dest="pipelineSpecs", default="",
            help="""Json file with info about the pipeline, fileValidation, fileValidationTools.                 
                 """, metavar="<string>", required=True)
    
    parser.add_argument("--statusFolder", dest="statusFolder", default=None,
            help="""The output status folder which contains links to files which are finished, or can be recovered.                
                 """, metavar="<string>", required=False)
    
                                                       
    parser.add_argument("--validateOnlyLastModified", dest="validateOnlyLastModified", type=cF.toBool, default=True,
            help="""The file with the moset recent modified time is only validated, all others are set to finished!.""", required=False)
                         

    parser.add_argument("-o", "--output", dest="output",
            help="""The output xml which is written, which proivides validation info for each file found""", metavar="<path>", required=True)
    
    
    try:
        
        print("====================== FileValidation ===========================")
        
        opts= AttrMap(vars(parser.parse_args()))
        if not opts.searchDirNew and not opts.valFileInfoGlobNew:
            raise ValueError("You need to define either searchDirNew or valFileInfoGlobNew!")
        
        if opts.valFileInfoGlobOld == "":
            opts.valFileInfoGlobOld = None
        
        print("searchDir: %s" % opts.searchDirNew)
        print("valFileInfoGlobNew: %s" % opts.valFileInfoGlobNew)
        print("valFileInfoGlobOld: %s" % opts.valFileInfoGlobOld)
        print("output: %s" % opts.output)
        
        
        d = cF.jsonLoad(opts.pipelineSpecs)
        pipelineTools = d["pipelineTools"]
        fileValidationSpecs = d["fileValidationSpecs"]
        fileValidationTools = d["fileValidationTools"]
        
        valDataAllNew = dict()
        deleteFiles = []
        
        # load new validataion datas
        if opts.valFileInfoGlobNew is not None:
            print("Load new validation files")
            valDataAllNew , valFilesNew  = loadValidationFiles(opts.valFileInfoGlobNew)
            
            preferGlobalPaths(valDataAllNew)
            
        
        # add searchDir files to new set
        # search files ============================================================================
        if opts.searchDirNew is not None:
            print("Validate all files in: %s with pipeLineSpecs: %s" % (opts.searchDirNew , opts.pipelineSpecs) )
            allFiles = searchFiles(opts.searchDirNew, opts, fileValidationSpecs,fileValidationTools,pipelineTools)
            for ha, f in allFiles.items():
              if ha in valDataAllNew:
                  print("""WARNING: File %s already found in validation data set 
                           from globbing expr. %s """ % (f["absPath"], opts.valFileInfoGlobNew))
              else:
                valDataAllNew[ha] = f
        # ===============================================================================================
        
        
        
        # load old validation datas
        if opts.valFileInfoGlobOld is not None:
            print("Load old validation files")
            valDataAllOld , valFilesOld  = loadValidationFiles(opts.valFileInfoGlobOld)
            preferGlobalPaths(valDataAllOld)
            
            # add old to new validatation infos 
            for ha, valInfo in valDataAllOld.items():
              
                if ha not in valDataAllNew:
                    # this old file hash is not in our current list, so add it!
                    
                    # check absPath if it exists otherwise try to extent the relPath with dir of this validation file.
                    if not os.path.exists(valInfo["absPath"]):
                      absPath = os.path.join( os.path.dirname(valInfo["validatationInfoPath"]) , valInfo["relPath"] )
                      if not os.path.exists(absPath):
                         print(valInfo["validatationInfoPath"])
                         raise NameError("""File path in valid. info file: %s 
                                            does not exist, extended rel. path to: %s does also not exist!""" % (valInfo["absPath"],absPath))
                      else:
                         print("Replacing inexisting path %s with %s", valInfo["absPath"], absPath)
                         valInfo["absPath"] = absPath
                      
                    # copy element to new file info
                    valDataAllNew[ha] = valInfo
                else:
                    # we have the same hash in the new info
                    # take our new one which is better!
                    # delete old file if it is not linked to by new file

                    if  os.path.realpath(valDataAllNew[ha]["absPath"]) !=  os.path.realpath(valInfo["absPath"]):
                        deleteFiles.append(valInfo["absPath"])
     

        # make final list
        finalFiles = [ f for f in valDataAllNew.values() ]
        
        printSummary(finalFiles,pipelineTools,False)
        
        print("Make output validation file")
        f = open(opts.output,"w+")
        cF.jsonDump(finalFiles,f, sort_keys=True)
        f.close();
        
        # Renew status folder, move over new xml info
        if opts.statusFolder is not None:
          
            print("Renew status folder:")
            finished = os.path.join(opts.statusFolder,"finished")
            recover = os.path.join(opts.statusFolder,"recover")
            
            cF.makeDirectory(finished,interact=False, defaultMakeEmpty=True)
            cF.makeDirectory(recover ,interact=False, defaultMakeEmpty=True)
            # make symlinks for all files in the appropriate folder:
            paths = {"recover": recover, "finished": finished}           
            
            for f in finalFiles:
                h = f["hash"]
                p = os.path.relpath(f["absPath"],start=paths[f["status"]])
                filename = os.path.basename(p)
                head,ext = os.path.splitext(filename)
                
                os.symlink(p, os.path.join( paths[f["status"]] , head+"-uuid-"+h+ext ) );


        print("=================================================================")
        
    except Exception as e:
        print("====================================================================")
        print("Exception occured: " + str(e))
        print("====================================================================")
        traceback.print_exc(file=sys.stdout)
        parser.print_help()
        return 1
Ejemplo n.º 3
0
def main():
    
    global subProcess,shutdown
    
    parser = MyOptParser()
    
    try:
        
        parser.add_argument("-c", "--renderCommand", dest="renderCommand", default="prman -t:1" ,
                help="""The render command to call.""", metavar="<path>", required=True)         
        
        parser.add_argument("-p", "--processFile", dest="processFile", default="./RenderProcess.xml" ,
                help="""Input file xml path with a list of files for rendering""", metavar="<path>", required=True)
        
           
        opts= AttrMap(vars(parser.parse_args()))
      
        print("================== Rendering Frames =========================")
        print("Script: " + __file__ )
        
        if(opts.processFile):
            frames = cF.jsonLoad(opts.processFile);

            if not frames:
                raise ValueError("No frames specified in xml %s" % opts.processFile)
            
            for f in frames:
              
                if shutDown:
                  break
                  
                print("Render frame: %s " % f["inputFile"])
                command = opts.renderCommand.split(" ") + [f["inputFile"]]
                print("Command: %s" % str(" ".join(command)))
                sys.stdout.flush()
                
                try:
                  subProcess = subprocess.Popen(command, stderr=subprocess.STDOUT)
                  # Waiting for process
                  subProcess.wait()
                  subProcess = None
                  
                except Exception as c:
                  raise NameError("Rendering Process Error: for file: %s with render output: %s " % (f,c.output) )
                  
        if shutDown:
            print("Render Loop shutdown")
        
        print("============================================================")        
            

    except Exception as e:
      
        print("====================================================================", file=sys.stderr)
        print("Exception occured: " + str(e), file=sys.stderr)
        print("====================================================================", file=sys.stderr)
        traceback.print_exc(file=sys.stderr)
        return 1
    
    print("================== Rendering finished ==================== ")
    
    return 0
def main():
         
    
    parser = MyOptParser()

    parser.add_argument("--pipelineSpecs", dest="pipelineSpecs", default="" ,
            help="""Json file with info about the pipeline, fileValidation, fileValidationTools.""", metavar="<path>", required=True)    
    
    parser.add_argument("--validationFileInfo", dest="validationFileInfo", default="" ,
            help="""XML file with info about render output files.""", metavar="<path>", required=False)
                                                                
    parser.add_argument("-p", "--processes", type=int, dest="processes", default=int(1),
            help="The number of processes for the cluster render", metavar="<integer>", required=True)
    
            
    try:
        
        print("================== Prepare for Cluster Pipeline Job============")
        
        opts= AttrMap(vars(parser.parse_args()))
        
        pipelineSpecs = cF.jsonLoad(opts.pipelineSpecs)
        

        pipelineTools = pipelineSpecs["pipelineTools"]
        
        # tool1 ---> tool2 ----> tool3
        #                        : dependency on tool2
        #            :tool3 is parent
                                 
        # define parents and dependencies for all tools
        for toolName,tool in pipelineTools.items():
            if "dependencies" not in tool:
                tool["dependencies"]=set()
            
            tool["parents"]=set()
            
        for toolName,tool in pipelineTools.items():
            for dep in tool["dependencies"]:
                t = pipelineTools[dep]
                t["parents"].add(toolName)

        
        frameGenerator = pipelineSpecs["frameGenerator"]
#        fileValidationSpecs = d["fileValidationSpecs"]
#        fileValidationTools = d["fileValidationTools"]
        
        # Important job modules to hand over to frameGenerators and processFileWriters
        importantModules = {"importHelpers":iH, "commonFunctions" : cF, "getSimFileInfos" : getSimFileInfos}
        
        # Generate Frames =====================================================
        mod, frameGenerator["generator"] = iH.importClassFromModuleString(frameGenerator["generator"])
        # hand over some modules to the frame generator!
        fgen = frameGenerator["generator"](pipelineSpecs, jobGenModules =  importantModules )
        allFrames,framesPerIdx, framesToDistribute = fgen(**frameGenerator["arguments"])
        # =====================================================================
        
            
        # Formatting frames ========================================================
        # format strings in all settings (if possible) in allFrames again with itself     
        for i,fr in enumerate(allFrames):
            allFrames[i] = cF.formatAll(fr,fr,formatter=StrExpr)
        
        # Filter Frames =======================================================
        recoverFrames(opts,allFrames,framesPerIdx,pipelineTools)
        #======================================================================
                  
        # make a list of all frames which are not-completely finished 
        # (that are frames where all tools with no parent (the last one) are not finished, we need  at least one )
        notcompleted = lambda frame:   sum( 1 if frame["tools"][toolName]["status"] != STATUS_FINISHED 
                                                else 0 for toolName,tool in pipelineTools.items() if len(tool["parents"])==0 ) > 0
        framesCount = len(allFrames);
        allFrames = list(filter(notcompleted, allFrames))
        framesToDistribute = list(filter(notcompleted, framesToDistribute))
        print("Removed %d finished frames!" % (framesCount - len(allFrames)) )
        

        #count number of frames to render
        totalFrames = len(framesToDistribute);
        print("Number of frames to compute %i" % totalFrames)
        if(totalFrames == 0):
          print("No frames to distribute -> exit")
          return 0
        
        # Distribute the processes over the number of processes ===============
        processFrames = distributeFrames(opts,framesToDistribute)
        #======================================================================

        
        # Write for each tool in the pipeline the process file, for each process a seperate one
        for toolName,tool in pipelineTools.items():
            
            # load the class and module for the tools processFileWriter
            print("Load processFileGenerator for tool: %s" % toolName )
            mod, tool["processFileGenerator"]["generator"] = iH.importClassFromModuleString(tool["processFileGenerator"]["generator"])
            tool["processFileGenerator"]["generator"](pipelineSpecs, jobGenModules = importantModules).write(processFrames, **tool["processFileGenerator"]["arguments"])
            
            # if we have some info file generator , produce the output
            
            if "infoFileGenerator" in tool:
                print("Load infoFileGenerator for tool: %s" % toolName )
                mod, tool["infoFileGenerator"]["generator"] = iH.importClassFromModuleString(tool["infoFileGenerator"]["generator"])
                tool["infoFileGenerator"]["generator"](pipelineSpecs, jobGenModules = importantModules).write(processFrames, **tool["infoFileGenerator"]["arguments"])
            
        
        # Write FileMover process file  =======================================
        writeFileMoverProcessFile(pipelineSpecs,processFrames)
        # =====================================================================
        return 0
         
    except Exception as e:
        print("====================================================================")
        print("Exception occured: " + str(e))
        print("====================================================================")
        traceback.print_exc(file=sys.stdout)
        parser.print_help()
        return 1
def recoverFrames(opts,allFrames,framesPerIdx, pipelineTools):
    
    
    def addFile(frame,file,parent=None):
        
        if "usedFile" in file and file["usedFile"] :
            #this file has already been used
            return
        
        if file["status"]==STATUS_RECOVER:
            
            print("added File: %s (recover)" % file["relPath"])
            # add a file move to recover this file
            frame["fileMover"].append(file["fileMoveRecover"])
            # mark file as used
            file["usedFile"] = True
            
        elif file["status"]==STATUS_FINISHED:
            if parent:
                print("added File: %s (finished, dependent)" % file["relPath"])
                # add a file move to recover this file
                frame["fileMover"].append(file["fileMoveDependent"])
                #print("id", id(frame), frame["fileMover"] )
                # mark file as used
                file["usedFile"] = True

    def addTool(frame,toolName, visitedTools,  parentToolName=None):        
        
       if toolName in visitedTools:
           return
           
       visitedTools.add(toolName);
       frameTool = frame["tools"][toolName]
       # if tool is not finished
       if frameTool["status"] != STATUS_FINISHED:    
           
            # add all this tools checkpoint files
            for outFileProp in frameTool["outputFiles"]:
                   
                if not outFileProp["cpFile"] == None:
                    addFile(frame,outFileProp["cpFile"],parentToolName)
          
            # add all dependent tools
            depTools = pipelineTools[toolName]["dependencies"]
            if depTools:
                for depTool in depTools:
                   addTool(frame,depTool,visitedTools,toolName)
                
       else:
            # we are finished, but
            # if we have a parent tool (always not finished), 
            # we add our finished checkpoint files
            if parentToolName:
               # add all its checkpoint files of output files
                for outFileProp in frameTool["outputFiles"]:
                   
                    if outFileProp["cpFile"] == None:
                        raise ValueError("""Trying to add non existant checkpoint file of output file %s in tool 
                        %s!""" % (str(outFileProp),toolName) )
                   
                    addFile(frame,outFileProp["cpFile"],parentToolName)
                   
            #else:
            # if no parent given, dont do anything

    # get all file info
    if opts.validationFileInfo:
        print("Setup recovery from file info===============================")
        print("Using validation file: %s", opts.validationFileInfo)
        checkpointFiles = cF.jsonLoad(opts.validationFileInfo);
        
        cpFiles = { "hash": {}, "all" : []}
        
        for f in checkpointFiles:
            tool = f["tool"]
            fileId = f["hash"]
            frameIdx = int(f["frameIdx"])
            ha = f["hash"]
            
            
            cpFiles["all"].append(f)
            
            if ha in cpFiles["hash"]:
                raise ValueError("File %s and %s have the same hash!" % (f["absPath"], cpFiles["hash"][ha]["absPath"] ) ) 
            else:
                cpFiles["hash"][ha] = f

        print("===========================================================")
        
        print("Determine status of all tools =============================")
        # move over all frames, for each tool and match cpFiles 
        for frameIdx,frame in framesPerIdx.items():
            finished = False;
            
            for toolName,tool in frame["tools"].items():
                
                # if there are checkpoint files corresponding to outputfiles of this tool
                finishedOutFiles = 0
                for outFileProp in tool["outputFiles"]:

                    ha =  cF.makeUUID(outFileProp["hashString"])
                    
                    
                    outFileProp["hash"] = ha
                    if ha in cpFiles["hash"]: # we found checkpoint file
                        cpFile = cpFiles["hash"][ha]
                        absP = cpFile["absPath"]
                        
                        print("Frame: %i " % frameIdx + 
                        " checkpoint file matched:\n\t%s\n\thash: %s\n\tstatus: %s " % ( absP[:10]+'...'+absP[-20:] if len(absP) > 70 else absP ,ha, cpFile["status"] ))
                        
                        outFileProp["cpFile"] = cpFile
                        
                        if outFileProp["cpFile"]["status"] == STATUS_FINISHED:
                            finishedOutFiles += 1
                            
                    else:
                        outFileProp["cpFile"] = None
                    

                # if all output files are finished -> tool is finished
                if finishedOutFiles == len(tool["outputFiles"]):
                    
                    tool["status"] =  STATUS_FINISHED
                    print("Tool: %s -> finished" % toolName)
                
        
        #print("Dependency check===========================================")
        
        
        ## for each frameIdx file list, 
        ## travel dependency of each file and if some files are missing       
        ## silently remove this file from the cpFilesPerFrame because this frameIdx can not be recovered!
        
        #invalidFrameIdx = set() # list for not recoverable frames! (should not happen)
        #for frameIdx,frame in framesPerIdx.items():
            
            #if frameIdx not in cpFilesPerFrame:
                #continue
            #stack = cpFilesPerFrame[frameIdx]["all"][:] # shallow copy (remove files from stack)
            #invalidFrame = walkDependencies(stack,cpFilesPerFrame[frameIdx])
            
            #if invalidFrame:
                #print("Invalid frameIdx: %i for recovery!" % frameIdx)
                #invalidFrameIdx.add(frameIdx)
                ## continue to next frame
                #continue
        ##endfor
                
        ## remove all files from all tools for invalid frames
        #for k in invalidFrameIdx:
            #for toolName,tool in framesPerIdx[k].items():
                #if toolName in pipelineTools.keys():
                    #tool["checkpointFiles"] = []
                
        #print("===========================================================")
        
        
        # setup recovery for all frames
        print("Setup pipeline tools with file info ========================")
        for frame in allFrames:
            
            # walk all tools in pipeline (visit all once!)
            for tool in pipelineTools.keys():
               addTool(frame,tool,set())
                       

        print("===============================================================")
Ejemplo n.º 6
0
def main():
         
    
    parser = MyOptParser()
    
    try:

        parser.add_argument("-p", "--processFile", dest="processFile", default="./FileMove.xml" ,
                help="""Input .json file with a list of files for moving into place""", metavar="<path>", required=True)
        opts= AttrMap(vars(parser.parse_args()))
      
        print("========================= Move Files ===========================")
        
        if opts.processFile:
            tasks = cF.jsonLoad(opts.processFile);
            for task in tasks:
                try:
                    
                    # avoiding links to links (get the real path resolving all symlinks)
                    # if that is necessary is not known
                    if "from" in task:
                        task["realfrom"] = os.path.realpath(task["from"])
                    
                    print("Task:", task)

                    if task["type"] == "hardlink":
                        try:
                            os.link(task["realfrom"],task["to"] )
                            continue
                        except:
                          task["type"] ="symlink"
                           
                    if task["type"] == "symlink":
                        try:
                            os.symlink(task["realfrom"],task["to"] )
                            continue
                        except:
                            task["type"] ="copy"
                    
                    if task["type"] == "copy":
                        shutil.copy2(task["from"],task["to"])
                        continue
                        
                    if task["type"] == "makeDirs" :
                        os.makedirs(task["dir"],exist_ok=True)
                        continue
                    
                    
                    raise ValueError("Type: %s not supported!" % str(task))
                    
                except:
                    raise ValueError("Could not execute task: %s" % str(task))
                    
        print("================================================================")  

    except Exception as e:
        print("====================================================================")
        print("Exception occured: " + str(e))
        print("====================================================================")
        traceback.print_exc(file=sys.stdout)
        return 1

    return 0