def register(filename,crs):
"""
Adds in FILE a metadata named 'CRS_def' and with value CRS, and a
metadata 'CLiMAF' with CliMAF version and ref URL
Records this FILE in dict crs2filename
Silently skip non-existing files
"""
# First read index from file if it is yet empty - No : done at startup
# if len(crs2filename.keys()) == 0 : cload()
# It appears that we have to let some time to the file system for updating its inode tables
waited=0
while waited < 20 and not os.path.exists(filename) :
time.sleep(0.1)
waited += 1
#time.sleep(0.5)
if os.path.exists(filename) :
#while time.time() < os.path.getmtime(filename) + 0.2 : time.sleep(0.2)
if re.findall(".nc$",filename) :
command="ncatted -h -a CRS_def,global,o,c,\"%s\" -a CliMAF,global,o,c,\"CLImate Model Assessment Framework version %s (http://climaf.rtfd.org)\" %s"%\
(crs,version,filename)
if re.findall(".png$",filename) :
command="convert -set \"CRS_def\" \"%s\" -set \"CliMAF\" \"CLImate Model Assessment Framework version %s (http://climaf.rtfd.org)\" %s %s.png && mv -f %s.png %s"%\
(crs,version,filename,filename,filename,filename)
clogger.debug("trying stamping by %s"%command)
if ( os.system(command) == 0 ) :
crs2filename[crs]=filename
clogger.info("%s registered as %s"%(crs,filename))
return True
else :
clogger.critical("cannot stamp by %s"%command)
return None
else :
clogger.error("file %s does not exist (for crs %s)"%(filename,crs))
def register(filename,crs):
""" Adds in FILE a metadata named CRS_def and with value CRS.
Records this FILE in dict crs2filename
Silently skip non-existing files
"""
# First read index from file if it is yet empty
if len(crs2filename.keys()) == 0 : cload()
# It appears that we have to allow the file system some time for updating its inode tables
waited=0
while waited < 10 and not os.path.exists(filename) :
time.sleep(0.5)
waited += 1
time.sleep(0.5)
if os.path.exists(filename) :
#while time.time() < os.path.getmtime(filename) + 0.2 : time.sleep(0.2)
if re.findall(".nc$",filename) :
command="ncatted -h -a CRS_def,global,o,c,\"%s\" %s"%(crs,filename)
if re.findall(".png$",filename) :
command="convert -set \"CRS_def\" \"%s\" %s %s.png && mv -f %s.png %s"%\
(crs,filename,filename,filename,filename)
clogger.debug("trying stamping by %s"%command)
if ( os.system(command) == 0 ) :
crs2filename[crs]=filename
clogger.info("%s registered as %s"%(crs,filename))
return True
else :
clogger.critical("cannot stamp by %s"%command)
return None
else :
clogger.error("file %s does not exist (for crs %s)"%(filename,crs))
def generateUniqueFileName_safe(expression, operator=None, format="nc"):
""" Generate a filename path from string EXPRESSION and FILEFORMAT, unique for the
expression and the set of cache directories currently listed in cache.cachedirs
OPERATOR may be a function that provides a prefix, using EXPRESSION
This uses hashlib.sha224, which are truncated to 3 (or more) characters.
More characters are used if a shorter name is already in use for another
expression in one of the known cache directories
Generated names drive a structure where each directory name 1 or 2
characters and file names have no more characters
Exits if uniqueness is unachievable (quite unexpectable !) """
#
if format is None:
return ""
prefix = ""
if operator is not None:
prefix2 = operator(expression)
if prefix2 is not None:
prefix = prefix2 + "/"
full = hashlib.sha224(expression).hexdigest()
number = fileNameLength
guess = full[0:number - 1]
existing = searchFile(prefix + stringToPath(guess, directoryNameLength) +
"." + format)
if existing:
readCRS = getCRS(existing)
# Update index if needed
if readCRS not in crs2filename:
clogger.warning(
"existing data %s in file %s was not yet registered in cache index"
% (readCRS, existing))
crs2filename[readCRS] = existing
while (existing is not None) and (readCRS != expression):
clogger.debug("must skip %s which CRS is %s" %
(existing, getCRS(existing)))
number += 2
if number >= len(full):
clogger.critical("Critical issue in cache : " + len(full) +
" digits is not enough for " + expression)
exit
guess = full[0:number - 1]
existing = searchFile(prefix +
stringToPath(guess, directoryNameLength) + "." +
format)
if existing:
readCRS = getCRS(existing)
rep = currentCache + "/" + prefix + stringToPath(
full[0:number - 1], directoryNameLength) + "." + format
rep = os.path.expanduser(rep)
# Create the relevant directory, so that user scripts don't have to care
dirn = os.path.dirname(rep)
if not os.path.exists(dirn):
os.makedirs(dirn)
clogger.debug("returning %s" % rep)
return rep
def generateUniqueFileName(expression, operator=None, format="nc"):
""" Generate a filename path from string EXPRESSION and FILEFORMAT, unique for the
expression and the set of cache directories currently listed in cache.cachedirs
OPERATOR may be a function that provides a prefix, using EXPRESSION
This uses hashlib.sha224, which are truncated to 3 (or more) characters.
More characters are used if a shorter name is already in use for another
expression in one of the known cache directories
Generated names drive a structure where each directory name 1 or 2
characters and file names have no more characters
Exits if uniqueness is unachievable (quite unexpectable !) """
#
import hashlib
directoryNameLength=2
#
if format==None : return ""
prefix=""
if operator is not None :
prefix2=operator(expression)
if prefix2 is not None : prefix=prefix2+"/"
full=hashlib.sha224(expression).hexdigest()
number=4
guess=full[0 : number - 1 ]
existing=searchFile(prefix+stringToPath(guess, directoryNameLength )+"."+format)
if existing :
readCRS=getCRS(existing)
# Update index if needed
if readCRS not in crs2filename :
clogger.warning("existing data %s in file %s was not yet registered in cache index"%\
(readCRS,existing))
crs2filename[readCRS]=existing
while ( ( existing is not None ) and ( readCRS != expression )) :
clogger.debug("must skip %s which CRS is %s"%\
(existing, getCRS(existing) ))
number += 2
if (number >= len(full) ) :
clogger.critical("Critical issue in cache : "+len(full)+" digits is not enough for "+expression)
exit
guess=full[0 : number - 1 ]
existing=searchFile(prefix+stringToPath(guess, directoryNameLength )+"."+format)
if existing : readCRS=getCRS(existing)
rep=currentCache+"/"+prefix+stringToPath(full[0 : number - 1 ], directoryNameLength )+"."+format
rep=os.path.expanduser(rep)
# Create the relevant directory, so that user scripts don't have to care
dirn=os.path.dirname(rep)
if not os.path.exists(dirn) : os.makedirs(dirn)
clogger.debug("returning %s"%rep)
return(rep)
def ceval_select(includer,included,userflags,format,deep,derived_list,recurse_list) :
""" Extract object INCLUDED from (existing) object INCLUDER,
taking into account the capability of the user process (USERFLAGS)
and the required delivering FORMAT(file or object)
"""
if format=='file' :
if userflags.canSelectTime or userflags.canSelectDomain:
clogger.debug("TBD - should do smthg smart when user can select time or domain")
#includer.setperiod(included.period)
incperiod=timePeriod(included)
clogger.debug("extract sub period %s out of %s"%(`incperiod`,includer.crs))
extract=capply('select',includer, period=`incperiod`)
objfile=ceval(extract,userflags,'file',deep,derived_list,recurse_list)
if objfile :
crs=includer.buildcrs(period=incperiod)
return(cache.rename(objfile,crs))
else :
clogger.critical("Cannot evaluate "+`extract`)
else :
clogger.error("Can yet process only files - TBD")
def getCRS(filename) :
""" Returns the CRS expression found in FILENAME's meta-data"""
import subprocess
if re.findall(".nc$",filename) :
form='ncdump -h %s | grep -E "CRS_def *=" | '+\
'sed -r -e "s/.*:CRS_def *= *\\\"(.*)\\\" *;$/\\1/" '
elif re.findall(".png$",filename) :
form='identify -verbose %s | grep -E " *CRS_def: " | sed -r -e "s/.*CRS_def: *//"'
else :
clogger.critical("unknown filetype for %s"%filename)
return None
command=form%filename
try:
rep=subprocess.check_output(command, shell=True).replace('\n','')
if (rep == "" ) :
clogger.error("file %s is not well formed (no CRS)"%filename)
if re.findall(".nc$",filename) : rep=rep.replace(r"\'",r"'")
except:
rep="failed"
clogger.debug("CRS expression read in %s is %s"%(filename,rep))
return rep
def register(filename, crs, outfilename=None):
"""
Adds in FILE a metadata named 'CRS_def' and with value CRS, and a
metadata 'CLiMAF' with CliMAF version and ref URL
Records this FILE in dict crs2filename
If OUTFILENAME is not None, FILENAME is a temporary file and
it's OUTFILENAME which is recorded in dict crs2filename
Silently skip non-existing files
"""
# First read index from file if it is yet empty - No : done at startup
# if len(crs2filename.keys()) == 0 : cload()
# It appears that we have to let some time to the file system for updating its inode tables
global dropped_crs
if not stamping:
clogger.debug('No stamping')
crs2filename[crs] = filename
return True
waited = 0
while waited < 50 and not os.path.exists(filename):
time.sleep(0.1)
waited += 1
# time.sleep(0.5)
if os.path.exists(filename):
# while time.time() < os.path.getmtime(filename) + 0.2 : time.sleep(0.2)
if re.findall(".nc$", filename):
command = "ncatted -h -a CRS_def,global,o,c,\"%s\" -a CliMAF,global,o,c,\"CLImate Model Assessment " \
"Framework version %s (http://climaf.rtfd.org)\" %s" % (crs, version, filename)
if re.findall(".png$", filename):
crs2 = crs.replace("%", "\%")
command = "convert -set \"CRS_def\" \"%s\" -set \"CliMAF\" \"CLImate Model Assessment Framework version " \
"%s (http://climaf.rtfd.org)\" %s %s.png && mv -f %s.png %s" % \
(crs2, version, filename, filename, filename, filename)
if re.findall(".pdf$", filename):
tmpfile = str(uuid.uuid4())
command = "pdftk %s dump_data output %s && echo -e \"InfoBegin\nInfoKey: Keywords\nInfoValue: %s\" >> %s " \
"&& pdftk %s update_info %s output %s.pdf && mv -f %s.pdf %s && rm -f %s" % \
(filename, tmpfile, crs, tmpfile, filename, tmpfile, filename, filename, filename, tmpfile)
if re.findall(".eps$", filename):
command = 'exiv2 -M"add Xmp.dc.CliMAF CLImate Model Assessment Framework version %s ' \
'(http://climaf.rtfd.org)" -M"add Xmp.dc.CRS_def %s" %s' % \
(version, crs, filename)
clogger.debug("trying stamping by %s" % command)
if os.system(command) == 0:
if outfilename:
cmd = 'mv -f %s %s ' % (filename, outfilename)
if os.system(cmd) == 0:
clogger.info("move %s as %s " % (filename, outfilename))
clogger.info("%s registered as %s" % (crs, outfilename))
crs2filename[crs] = outfilename
if crs in dropped_crs:
dropped_crs.remove(crs)
return True
else:
clogger.critical("cannot move by" % cmd)
exit()
return None
else:
clogger.info("%s registered as %s" % (crs, filename))
crs2filename[crs] = filename
if crs in dropped_crs:
dropped_crs.remove(crs)
return True
else:
clogger.critical("cannot stamp by %s" % command)
exit()
return None
else:
clogger.error("file %s does not exist (for crs %s)" % (filename, crs))
