def writeFileMoverProcessFile(pipelineSpecs,processFrames): pipelineTools = pipelineSpecs["pipelineTools"] toolTasks = [] # Set up all input/output folders for each tool for toolName,tool in pipelineTools.items(): toolTasks.append( {"type":"makeDirs", "dir" : tool["outputDir"]} ) toolTasks.append( {"type":"makeDirs", "dir" : tool["inputDir"]} ) # Set up simlinks (make link to outputs in input folders of parent tools) if "linkAllTools" in pipelineSpecs["fileMover"] and pipelineSpecs["fileMover"]["linkAllTools"]: for toolName,tool in pipelineTools.items(): for parentName in tool["parents"]: d = pipelineTools[parentName]["inputDir"] toolTasks.append( {"type":"symlink", "to" : os.path.join(d, toolName), "from" : tool["outputDir"]} ) # Set up all other dirs in makeDirectories if "additionalTasks" in pipelineSpecs["fileMover"]: toolTasks += pipelineSpecs["fileMover"]["additionalTasks"] # set up all folders and links for each process for procIdx, fList in enumerate(processFrames): o = open(pipelineSpecs["fileMover"]["fileMoverProcessFile"].format(procIdx),"w+") c=[] # add for this process all tool tasks c += toolTasks # Set up all file move stuff for all frames (recover,dependend...) for fr in fList: for m in fr["fileMover"]: c.append(m) cF.jsonDump(c,o,indent=4) o.close() print("Wrote file mover pre-process files for all ranks for local directory")
def main(): """ {old validatation file infos} is compared to { new validation file infos} ==> outputs new file validation info """ parser = MyOptParser() parser.add_argument("-s", "--searchDirNew", dest="searchDirNew", help="""This is the search directory where it is looked for output files (.tiff,.exr,.rib.gz). """, metavar="<path>", default=None, required=False) parser.add_argument("--valFileInfoGlobNew", dest="valFileInfoGlobNew", help=""" The globbing expression for all input xmls with file status which are consolidated into a new file info under --output. The found and validated files in --searchDir (if specified) are added to the set of new files. """, default=None, metavar="<glob>", required=False) parser.add_argument("--valFileInfoGlobOld", dest="valFileInfoGlobOld", help=""" The globbing expression for all old input xmls with file status which are consolidated with the new files into a combined file info under --output. """, default=None, metavar="<glob>", required=False) parser.add_argument("--pipelineSpecs", dest="pipelineSpecs", default="", help="""Json file with info about the pipeline, fileValidation, fileValidationTools. """, metavar="<string>", required=True) parser.add_argument("--statusFolder", dest="statusFolder", default=None, help="""The output status folder which contains links to files which are finished, or can be recovered. """, metavar="<string>", required=False) parser.add_argument("--validateOnlyLastModified", dest="validateOnlyLastModified", type=cF.toBool, default=True, help="""The file with the moset recent modified time is only validated, all others are set to finished!.""", required=False) parser.add_argument("-o", "--output", dest="output", help="""The output xml which is written, which proivides validation info for each file found""", metavar="<path>", required=True) try: print("====================== FileValidation ===========================") opts= AttrMap(vars(parser.parse_args())) if not opts.searchDirNew and not opts.valFileInfoGlobNew: raise ValueError("You need to define either searchDirNew or valFileInfoGlobNew!") if opts.valFileInfoGlobOld == "": opts.valFileInfoGlobOld = None print("searchDir: %s" % opts.searchDirNew) print("valFileInfoGlobNew: %s" % opts.valFileInfoGlobNew) print("valFileInfoGlobOld: %s" % opts.valFileInfoGlobOld) print("output: %s" % opts.output) d = cF.jsonLoad(opts.pipelineSpecs) pipelineTools = d["pipelineTools"] fileValidationSpecs = d["fileValidationSpecs"] fileValidationTools = d["fileValidationTools"] valDataAllNew = dict() deleteFiles = [] # load new validataion datas if opts.valFileInfoGlobNew is not None: print("Load new validation files") valDataAllNew , valFilesNew = loadValidationFiles(opts.valFileInfoGlobNew) preferGlobalPaths(valDataAllNew) # add searchDir files to new set # search files ============================================================================ if opts.searchDirNew is not None: print("Validate all files in: %s with pipeLineSpecs: %s" % (opts.searchDirNew , opts.pipelineSpecs) ) allFiles = searchFiles(opts.searchDirNew, opts, fileValidationSpecs,fileValidationTools,pipelineTools) for ha, f in allFiles.items(): if ha in valDataAllNew: print("""WARNING: File %s already found in validation data set from globbing expr. %s """ % (f["absPath"], opts.valFileInfoGlobNew)) else: valDataAllNew[ha] = f # =============================================================================================== # load old validation datas if opts.valFileInfoGlobOld is not None: print("Load old validation files") valDataAllOld , valFilesOld = loadValidationFiles(opts.valFileInfoGlobOld) preferGlobalPaths(valDataAllOld) # add old to new validatation infos for ha, valInfo in valDataAllOld.items(): if ha not in valDataAllNew: # this old file hash is not in our current list, so add it! # check absPath if it exists otherwise try to extent the relPath with dir of this validation file. if not os.path.exists(valInfo["absPath"]): absPath = os.path.join( os.path.dirname(valInfo["validatationInfoPath"]) , valInfo["relPath"] ) if not os.path.exists(absPath): print(valInfo["validatationInfoPath"]) raise NameError("""File path in valid. info file: %s does not exist, extended rel. path to: %s does also not exist!""" % (valInfo["absPath"],absPath)) else: print("Replacing inexisting path %s with %s", valInfo["absPath"], absPath) valInfo["absPath"] = absPath # copy element to new file info valDataAllNew[ha] = valInfo else: # we have the same hash in the new info # take our new one which is better! # delete old file if it is not linked to by new file if os.path.realpath(valDataAllNew[ha]["absPath"]) != os.path.realpath(valInfo["absPath"]): deleteFiles.append(valInfo["absPath"]) # make final list finalFiles = [ f for f in valDataAllNew.values() ] printSummary(finalFiles,pipelineTools,False) print("Make output validation file") f = open(opts.output,"w+") cF.jsonDump(finalFiles,f, sort_keys=True) f.close(); # Renew status folder, move over new xml info if opts.statusFolder is not None: print("Renew status folder:") finished = os.path.join(opts.statusFolder,"finished") recover = os.path.join(opts.statusFolder,"recover") cF.makeDirectory(finished,interact=False, defaultMakeEmpty=True) cF.makeDirectory(recover ,interact=False, defaultMakeEmpty=True) # make symlinks for all files in the appropriate folder: paths = {"recover": recover, "finished": finished} for f in finalFiles: h = f["hash"] p = os.path.relpath(f["absPath"],start=paths[f["status"]]) filename = os.path.basename(p) head,ext = os.path.splitext(filename) os.symlink(p, os.path.join( paths[f["status"]] , head+"-uuid-"+h+ext ) ); print("=================================================================") except Exception as e: print("====================================================================") print("Exception occured: " + str(e)) print("====================================================================") traceback.print_exc(file=sys.stdout) parser.print_help() return 1