def register(filename,crs): """ Adds in FILE a metadata named 'CRS_def' and with value CRS, and a metadata 'CLiMAF' with CliMAF version and ref URL Records this FILE in dict crs2filename Silently skip non-existing files """ # First read index from file if it is yet empty - No : done at startup # if len(crs2filename.keys()) == 0 : cload() # It appears that we have to let some time to the file system for updating its inode tables waited=0 while waited < 20 and not os.path.exists(filename) : time.sleep(0.1) waited += 1 #time.sleep(0.5) if os.path.exists(filename) : #while time.time() < os.path.getmtime(filename) + 0.2 : time.sleep(0.2) if re.findall(".nc$",filename) : command="ncatted -h -a CRS_def,global,o,c,\"%s\" -a CliMAF,global,o,c,\"CLImate Model Assessment Framework version %s (http://climaf.rtfd.org)\" %s"%\ (crs,version,filename) if re.findall(".png$",filename) : command="convert -set \"CRS_def\" \"%s\" -set \"CliMAF\" \"CLImate Model Assessment Framework version %s (http://climaf.rtfd.org)\" %s %s.png && mv -f %s.png %s"%\ (crs,version,filename,filename,filename,filename) clogger.debug("trying stamping by %s"%command) if ( os.system(command) == 0 ) : crs2filename[crs]=filename clogger.info("%s registered as %s"%(crs,filename)) return True else : clogger.critical("cannot stamp by %s"%command) return None else : clogger.error("file %s does not exist (for crs %s)"%(filename,crs))
def register(filename,crs): """ Adds in FILE a metadata named CRS_def and with value CRS. Records this FILE in dict crs2filename Silently skip non-existing files """ # First read index from file if it is yet empty if len(crs2filename.keys()) == 0 : cload() # It appears that we have to allow the file system some time for updating its inode tables waited=0 while waited < 10 and not os.path.exists(filename) : time.sleep(0.5) waited += 1 time.sleep(0.5) if os.path.exists(filename) : #while time.time() < os.path.getmtime(filename) + 0.2 : time.sleep(0.2) if re.findall(".nc$",filename) : command="ncatted -h -a CRS_def,global,o,c,\"%s\" %s"%(crs,filename) if re.findall(".png$",filename) : command="convert -set \"CRS_def\" \"%s\" %s %s.png && mv -f %s.png %s"%\ (crs,filename,filename,filename,filename) clogger.debug("trying stamping by %s"%command) if ( os.system(command) == 0 ) : crs2filename[crs]=filename clogger.info("%s registered as %s"%(crs,filename)) return True else : clogger.critical("cannot stamp by %s"%command) return None else : clogger.error("file %s does not exist (for crs %s)"%(filename,crs))
def generateUniqueFileName_safe(expression, operator=None, format="nc"): """ Generate a filename path from string EXPRESSION and FILEFORMAT, unique for the expression and the set of cache directories currently listed in cache.cachedirs OPERATOR may be a function that provides a prefix, using EXPRESSION This uses hashlib.sha224, which are truncated to 3 (or more) characters. More characters are used if a shorter name is already in use for another expression in one of the known cache directories Generated names drive a structure where each directory name 1 or 2 characters and file names have no more characters Exits if uniqueness is unachievable (quite unexpectable !) """ # if format is None: return "" prefix = "" if operator is not None: prefix2 = operator(expression) if prefix2 is not None: prefix = prefix2 + "/" full = hashlib.sha224(expression).hexdigest() number = fileNameLength guess = full[0:number - 1] existing = searchFile(prefix + stringToPath(guess, directoryNameLength) + "." + format) if existing: readCRS = getCRS(existing) # Update index if needed if readCRS not in crs2filename: clogger.warning( "existing data %s in file %s was not yet registered in cache index" % (readCRS, existing)) crs2filename[readCRS] = existing while (existing is not None) and (readCRS != expression): clogger.debug("must skip %s which CRS is %s" % (existing, getCRS(existing))) number += 2 if number >= len(full): clogger.critical("Critical issue in cache : " + len(full) + " digits is not enough for " + expression) exit guess = full[0:number - 1] existing = searchFile(prefix + stringToPath(guess, directoryNameLength) + "." + format) if existing: readCRS = getCRS(existing) rep = currentCache + "/" + prefix + stringToPath( full[0:number - 1], directoryNameLength) + "." + format rep = os.path.expanduser(rep) # Create the relevant directory, so that user scripts don't have to care dirn = os.path.dirname(rep) if not os.path.exists(dirn): os.makedirs(dirn) clogger.debug("returning %s" % rep) return rep
def generateUniqueFileName(expression, operator=None, format="nc"): """ Generate a filename path from string EXPRESSION and FILEFORMAT, unique for the expression and the set of cache directories currently listed in cache.cachedirs OPERATOR may be a function that provides a prefix, using EXPRESSION This uses hashlib.sha224, which are truncated to 3 (or more) characters. More characters are used if a shorter name is already in use for another expression in one of the known cache directories Generated names drive a structure where each directory name 1 or 2 characters and file names have no more characters Exits if uniqueness is unachievable (quite unexpectable !) """ # import hashlib directoryNameLength=2 # if format==None : return "" prefix="" if operator is not None : prefix2=operator(expression) if prefix2 is not None : prefix=prefix2+"/" full=hashlib.sha224(expression).hexdigest() number=4 guess=full[0 : number - 1 ] existing=searchFile(prefix+stringToPath(guess, directoryNameLength )+"."+format) if existing : readCRS=getCRS(existing) # Update index if needed if readCRS not in crs2filename : clogger.warning("existing data %s in file %s was not yet registered in cache index"%\ (readCRS,existing)) crs2filename[readCRS]=existing while ( ( existing is not None ) and ( readCRS != expression )) : clogger.debug("must skip %s which CRS is %s"%\ (existing, getCRS(existing) )) number += 2 if (number >= len(full) ) : clogger.critical("Critical issue in cache : "+len(full)+" digits is not enough for "+expression) exit guess=full[0 : number - 1 ] existing=searchFile(prefix+stringToPath(guess, directoryNameLength )+"."+format) if existing : readCRS=getCRS(existing) rep=currentCache+"/"+prefix+stringToPath(full[0 : number - 1 ], directoryNameLength )+"."+format rep=os.path.expanduser(rep) # Create the relevant directory, so that user scripts don't have to care dirn=os.path.dirname(rep) if not os.path.exists(dirn) : os.makedirs(dirn) clogger.debug("returning %s"%rep) return(rep)
def ceval_select(includer,included,userflags,format,deep,derived_list,recurse_list) : """ Extract object INCLUDED from (existing) object INCLUDER, taking into account the capability of the user process (USERFLAGS) and the required delivering FORMAT(file or object) """ if format=='file' : if userflags.canSelectTime or userflags.canSelectDomain: clogger.debug("TBD - should do smthg smart when user can select time or domain") #includer.setperiod(included.period) incperiod=timePeriod(included) clogger.debug("extract sub period %s out of %s"%(`incperiod`,includer.crs)) extract=capply('select',includer, period=`incperiod`) objfile=ceval(extract,userflags,'file',deep,derived_list,recurse_list) if objfile : crs=includer.buildcrs(period=incperiod) return(cache.rename(objfile,crs)) else : clogger.critical("Cannot evaluate "+`extract`) else : clogger.error("Can yet process only files - TBD")
def getCRS(filename) : """ Returns the CRS expression found in FILENAME's meta-data""" import subprocess if re.findall(".nc$",filename) : form='ncdump -h %s | grep -E "CRS_def *=" | '+\ 'sed -r -e "s/.*:CRS_def *= *\\\"(.*)\\\" *;$/\\1/" ' elif re.findall(".png$",filename) : form='identify -verbose %s | grep -E " *CRS_def: " | sed -r -e "s/.*CRS_def: *//"' else : clogger.critical("unknown filetype for %s"%filename) return None command=form%filename try: rep=subprocess.check_output(command, shell=True).replace('\n','') if (rep == "" ) : clogger.error("file %s is not well formed (no CRS)"%filename) if re.findall(".nc$",filename) : rep=rep.replace(r"\'",r"'") except: rep="failed" clogger.debug("CRS expression read in %s is %s"%(filename,rep)) return rep
def register(filename, crs, outfilename=None): """ Adds in FILE a metadata named 'CRS_def' and with value CRS, and a metadata 'CLiMAF' with CliMAF version and ref URL Records this FILE in dict crs2filename If OUTFILENAME is not None, FILENAME is a temporary file and it's OUTFILENAME which is recorded in dict crs2filename Silently skip non-existing files """ # First read index from file if it is yet empty - No : done at startup # if len(crs2filename.keys()) == 0 : cload() # It appears that we have to let some time to the file system for updating its inode tables global dropped_crs if not stamping: clogger.debug('No stamping') crs2filename[crs] = filename return True waited = 0 while waited < 50 and not os.path.exists(filename): time.sleep(0.1) waited += 1 # time.sleep(0.5) if os.path.exists(filename): # while time.time() < os.path.getmtime(filename) + 0.2 : time.sleep(0.2) if re.findall(".nc$", filename): command = "ncatted -h -a CRS_def,global,o,c,\"%s\" -a CliMAF,global,o,c,\"CLImate Model Assessment " \ "Framework version %s (http://climaf.rtfd.org)\" %s" % (crs, version, filename) if re.findall(".png$", filename): crs2 = crs.replace("%", "\%") command = "convert -set \"CRS_def\" \"%s\" -set \"CliMAF\" \"CLImate Model Assessment Framework version " \ "%s (http://climaf.rtfd.org)\" %s %s.png && mv -f %s.png %s" % \ (crs2, version, filename, filename, filename, filename) if re.findall(".pdf$", filename): tmpfile = str(uuid.uuid4()) command = "pdftk %s dump_data output %s && echo -e \"InfoBegin\nInfoKey: Keywords\nInfoValue: %s\" >> %s " \ "&& pdftk %s update_info %s output %s.pdf && mv -f %s.pdf %s && rm -f %s" % \ (filename, tmpfile, crs, tmpfile, filename, tmpfile, filename, filename, filename, tmpfile) if re.findall(".eps$", filename): command = 'exiv2 -M"add Xmp.dc.CliMAF CLImate Model Assessment Framework version %s ' \ '(http://climaf.rtfd.org)" -M"add Xmp.dc.CRS_def %s" %s' % \ (version, crs, filename) clogger.debug("trying stamping by %s" % command) if os.system(command) == 0: if outfilename: cmd = 'mv -f %s %s ' % (filename, outfilename) if os.system(cmd) == 0: clogger.info("move %s as %s " % (filename, outfilename)) clogger.info("%s registered as %s" % (crs, outfilename)) crs2filename[crs] = outfilename if crs in dropped_crs: dropped_crs.remove(crs) return True else: clogger.critical("cannot move by" % cmd) exit() return None else: clogger.info("%s registered as %s" % (crs, filename)) crs2filename[crs] = filename if crs in dropped_crs: dropped_crs.remove(crs) return True else: clogger.critical("cannot stamp by %s" % command) exit() return None else: clogger.error("file %s does not exist (for crs %s)" % (filename, crs))