def cdump(use_macro=True):
"""
List the in-memory content of CliMAF cache index. Interpret it
using macros except if arg use_macro is False
"""
for crs in crs2filename:
if not use_macro:
# No interpretation by macros
# print "%s : %s"%(crs2filename[crs][-30:],crs)
print "%s : %s" % (crs2filename[crs], crs)
else:
# Must update for new macros
print "%s : %s" % (crs2filename[crs], crewrite(crs))
def cdump(use_macro=True):
"""
List the in-memory content of CliMAF cache index. Interpret it
using macros except if arg use_macro is False
"""
for crs in crs2filename :
if not use_macro :
# No interpretation by macros
#print "%s : %s"%(crs2filename[crs][-30:],crs)
print "%s : %s"%(crs2filename[crs],crs)
else:
# Must update for new macros
print "%s : %s"%(crs2filename[crs],crewrite(crs))
def clist(size="",
age="",
access=0,
pattern="",
not_pattern="",
usage=False,
count=False,
remove=False,
CRS=False,
special=False):
"""
Internal function used by its front-ends : :py:func:`~climaf.cache.cls`, :py:func:`~climaf.cache.crm`,
:py:func:`~climaf.cache.cdu`, :py:func:`~climaf.cache.cwc`
List the content of CliMAF cache according to some search criteria
and operate possibly an action (usage, count or remove) on this list.
Please consider the cost and benefit of first updating CliMAF cache index (by scanning
files on disk) using :py:func:`csync()`
Args:
size (string, optional): n[ckMG]
Search files using more than n units of disk space, rounding up.
The following suffixes can be used:
- "c" for bytes (default)
- "k" for Kilobytes (units of 1,024 bytes)
- "M" for Megabytes (units of 1,048,576 bytes)
- "G" for Gigabytes (units of 1,073,741,824 bytes)
age (string, optional): Number of 24h periods. Search files which
status was last changed n*24 hours ago.
Any fractional part is ignored, so to match age='+1', a file has
to have been changed at least two days ago.
Numeric arguments can be specified as:
- `+n` for greater than n
- `-n` for less than n,
- `n` for exactly n.
access (int, optional): n
Search files which were last accessed more than n*24 hours ago. Any
fractional part is ignored, so to match access='1', a file has to
have been accessed at least two days ago.
pattern (string, optional): Scan through crs and filenames looking for
the first location where the regular expression pattern produces a match.
not_pattern (string, optional): Scan through crs and filenames looking
for the location where the regular expression not_pattern does not
produce a match.
usage (bool, optional): Estimate found files space usage, for each
found file and total size. If count is True, estimate only found
files total space usage.
count (bool, optional): Return the number of found files. If CRS is True,
also return crs of found files.
remove (bool, optional): Remove the found files. This argument is exclusive.
CRS (bool, optional): if True, print also CRS expression. Useful only
if count is True.
Return:
The dictionary corresponding to the request and associated action ( or dictionary
of CliMAF cache index if no argument is provided)
Example to search files using more than 3M of disk space, which status
was last changed more than 15 days ago and containing the pattern
'1980-1981' either in crs or filename. For found files, we want to
estimate only found files total space usage::
>>> clist(size='3M', age='+15', pattern= '1980-1981', usage=True, count=True)
"""
# cache directories
rep = os.path.expanduser(
cachedirs[0]
) # TBD: le cache ne contient qu un rep pr le moment => voir pour boucler sur tous les caches
# command for research on size/age/access
command = ""
opt_find = ""
if size:
if re.search('[kMG]', size) is None:
opt_find += "-size +%sc " % size
else:
opt_find += "-size +%s " % size
if age:
opt_find += "-ctime %s " % age
if access != 0:
opt_find += "-atime +%s" % str(int(access))
var_find = False
if size or age or access != 0:
var_find = True
command = "find %s -type f \( -name '*.png' -o -name '*.nc' -o -name '*.pdf' -o -name '*.eps' \) %s -print" % \
(rep, opt_find)
clogger.debug("Find command is :" + command)
# construction of the new dictionary after research on size/age/access
new_dict = dict()
find_return = ""
list_search_files_after_find = []
find_return = os.popen(command).read()
list_search_files_after_find = find_return.split('\n')
list_search_files_after_find.pop(-1)
clogger.debug("List of search files: " +
repr(list_search_files_after_find))
# Search CRS for each found file
for filen in list_search_files_after_find:
for crs in crs2filename:
if crs2filename[crs] == filen:
new_dict[crs] = filen
if len(new_dict) != 0:
if new_dict != crs2filename:
clogger.debug("Dictionary after find for size/age/access: " +
repr(new_dict))
else:
clogger.debug(
"Size/age/access criteria do not lead to any filtering")
else:
clogger.debug("No file meet the size/age/access criteria")
else:
new_dict = crs2filename.copy()
# size of new dictionary
len_new_dict = len(new_dict)
# filter on pattern
find_pattern = False
if pattern:
list_crs_to_rm = []
for crs in new_dict:
if re.search(pattern, crewrite(crs)) or re.search(
pattern, new_dict[crs]):
clogger.debug("Pattern found in %s: %s" % (crs, new_dict[crs]))
find_pattern = True
else:
# Do not remove now from new_dict, because we loop on it
list_crs_to_rm.append(crs)
for crs in list_crs_to_rm:
del new_dict[crs]
if find_pattern:
clogger.debug("Dictionary after search for pattern: " +
repr(new_dict))
elif len_new_dict != 0:
clogger.debug("No string found for pattern => no result")
# update size new dictionary
len_new_dict = len(new_dict)
# research on not_pattern
find_not_pattern = False
if not_pattern:
list_crs_to_rm = []
for crs in new_dict:
if re.search(not_pattern, crewrite(crs)) is None and \
re.search(not_pattern, new_dict[crs]) is None:
clogger.debug("Pattern not found in %s: %s" %
(crs, new_dict[crs]))
find_not_pattern = True
else:
list_crs_to_rm.append(crs)
for crs in list_crs_to_rm:
del new_dict[crs]
if find_not_pattern:
clogger.debug("Dictionary after search for not_pattern: " +
repr(new_dict))
elif len_new_dict != 0:
clogger.debug("All strings contain not_pattern => no result")
# update size new dictionary
len_new_dict = len(new_dict)
# request on new dictionary through usage, count and remove
work_dic = new_dict if (var_find or pattern is not ""
or not_pattern is not "") else crs2filename
if usage is True and len_new_dict != 0:
# construction of a dictionary containing crs and disk-usage associated
dic_usage = dict()
tmp = ""
for crs in work_dic:
tmp += work_dic[crs] + " "
res = os.popen("du -sc %s" % tmp).read()
regex = re.compile('([0-9]+)\t')
list_size = re.findall(regex, res)
regex2 = re.compile('([0-9]+\t)')
str_path = regex2.sub('', res)
list_fig = str_path.split('\n')
list_fig.pop(-1)
for fig, size in zip(list_fig, list_size):
if fig != "total":
for crs in work_dic:
if fig == work_dic[crs]:
dic_usage[crs] = size
else:
dic_usage[fig] = size
# sort of usage dictionary and units conversion
du_list_sort = dic_usage.items()
du_list_sort.sort(key=itemgetter(1), reverse=False)
unit = ["K", "M", "G", "T"]
for n, pair in enumerate(du_list_sort):
i = 0
flt = float(pair[1])
while flt >= 1024. and i < 4:
flt /= 1024.
i += 1
du_list_sort[n] = (du_list_sort[n][0], "%6.1f%s" % (flt, unit[i]))
if count is True: # Display total volume of found files
for fig, size in du_list_sort:
if fig == "total":
print "%7s : %s" % (size, fig)
else: # retrieve disk-usage of each found file and total volume
for fig, size in du_list_sort:
print "%7s : %s" % (size, fig)
elif count is True and len_new_dict != 0:
print "Number of files found:", len(work_dic)
if CRS is True:
for crs in work_dic:
print crs
elif remove is True and len_new_dict != 0:
print "Removed files:"
list_tmp_crs = []
list_tmp_crs = new_dict.keys() if (
var_find or pattern is not ""
or not_pattern is not "") else crs2filename.keys()
for crs in list_tmp_crs:
cdrop(crs, rm=True)
return map(crewrite, list_tmp_crs)
else: # usage, count and remove are False
if var_find or pattern is not "" or not_pattern is not "":
if len(new_dict) != 0:
if new_dict != crs2filename:
print "Filtered objects :"
else:
print "Filtered objects = cache content"
return map(crewrite, new_dict.keys())
# else : print "No matching file "
else:
print "Content of CliMAF cache"
return map(crewrite, crs2filename.keys())
# TBD
if special is True:
global dic_special
dic_special = dict()
if var_find is True or pattern is not "" or not_pattern is not "":
dic_special = new_dict.copy()
else:
dic_special = crs2filename.copy()
print "List of marked figures as 'special'", dic_special.values()
return dic_special # TBD: declarer comme var globale et enlever son effacement dans creset
new_dict.clear()
def clist(size="", age="", access=0, pattern="", not_pattern="", usage=False, count=False,
remove=False, CRS=False, special=False):
"""
Internal function used by its front-ends : :py:func:`~climaf.cache.cls`, :py:func:`~climaf.cache.crm`,
:py:func:`~climaf.cache.cdu`, :py:func:`~climaf.cache.cwc`
List the content of CliMAF cache according to some search criteria
and operate possibly an action (usage, count or remove) on this list.
Please consider the cost and benefit of first updating CliMAF cache index (by scanning
files on disk) using :py:func:`csync()`
Args:
size (string, optional): n[ckMG]
Search files using more than n units of disk space, rounding up.
The following suffixes can be used:
- "c" for bytes (default)
- "k" for Kilobytes (units of 1,024 bytes)
- "M" for Megabytes (units of 1,048,576 bytes)
- "G" for Gigabytes (units of 1,073,741,824 bytes)
age (string, optional): Number of 24h periods. Search files which status was last changed n*24 hours ago.
Any fractional part is ignored, so to match age='+1', a file has to have been changed at least two days ago.
Numeric arguments can be specified as:
- `+n` for greater than n
- `-n` for less than n,
- `n` for exactly n.
access (int, optional): n
Search files which were last accessed more than n*24 hours ago. Any
fractional part is ignored, so to match access='1', a file has to
have been accessed at least two days ago.
pattern (string, optional): Scan through crs and filenames looking for
the first location where the regular expression pattern produces a match.
not_pattern (string, optional): Scan through crs and filenames looking
for the location where the regular expression not_pattern does not
produce a match.
usage (bool, optional): Estimate found files space usage, for each
found file and total size. If count is True, estimate only found
files total space usage.
count (bool, optional): Return the number of found files. If CRS is True,
also return crs of found files.
remove (bool, optional): Remove the found files. This argument is exclusive.
CRS (bool, optional): if True, print also CRS expression. Useful only
if count is True.
Return:
The dictionary corresponding to the request and associated action ( or dictionary
of CliMAF cache index if no argument is provided)
Example to search files using more than 3M of disk space, which status
was last changed more than 15 days ago and containing the pattern
'1980-1981' either in crs or filename. For found files, we want to
estimate only found files total space usage::
>>> clist(size='3M', age='+15', pattern= '1980-1981', usage=True, count=True)
"""
#cache directories
rep=os.path.expanduser(cachedirs[0]) #TBD: le cache ne contient qu un rep pr le moment => voir pour boucler sur tous les caches
#command for research on size/age/access
command=""
opt_find=""
if size :
if re.search('[kMG]', size) is None :
opt_find+="-size +%sc "%size
else:
opt_find+="-size +%s "%size
if age :
opt_find+="-ctime %s "%age
if access !=0 :
opt_find+="-atime +%s"%str(int(access))
var_find=False
if size or age or access != 0 :
var_find=True
command="find %s -type f \( -name '*.png' -o -name '*.nc' \) %s -print" %(rep, opt_find)
clogger.debug("Find command is :"+command)
#construction of the new dictionary after research on size/age/access
new_dict=dict()
find_return=""
list_search_files_after_find=[]
find_return=os.popen(command).read()
list_search_files_after_find=find_return.split('\n')
list_search_files_after_find.pop(-1)
clogger.debug("List of search files: "+`list_search_files_after_find`)
# Search CRS for each found file
for filen in list_search_files_after_find :
for crs in crs2filename:
if crs2filename[crs]==filen:
new_dict[crs]=filen
if len(new_dict) != 0 :
if new_dict != crs2filename :
clogger.debug("Dictionary after find for size/age/access: "+`new_dict`)
else :
clogger.debug("Size/age/access criteria do not lead to any filtering")
else :
clogger.debug("No file meet the size/age/access criteria")
else:
new_dict=crs2filename.copy()
#size of new dictionary
len_new_dict=len(new_dict)
#filter on pattern
find_pattern=False
if pattern :
list_crs_to_rm=[]
for crs in new_dict :
if re.search(pattern, crewrite(crs)) or re.search(pattern, new_dict[crs]):
clogger.debug("Pattern found in %s: %s"%(crs,new_dict[crs]))
find_pattern=True
else:
# Do not remove now from new_dict, because we loop on it
list_crs_to_rm.append(crs)
for crs in list_crs_to_rm :
del new_dict[crs]
if find_pattern :
clogger.debug("Dictionary after search for pattern: "+`new_dict`)
elif len_new_dict!=0 :
clogger.debug("No string found for pattern => no result")
#update size new dictionary
len_new_dict=len(new_dict)
#research on not_pattern
find_not_pattern=False
if not_pattern:
list_crs_to_rm=[]
for crs in new_dict :
if re.search(not_pattern, crewrite(crs)) is None and \
re.search(not_pattern, new_dict[crs]) is None :
clogger.debug("Pattern not found in %s: %s"%(crs, new_dict[crs]))
find_not_pattern=True
else:
list_crs_to_rm.append(crs)
for crs in list_crs_to_rm :
del new_dict[crs]
if find_not_pattern :
clogger.debug("Dictionary after search for not_pattern: "+`new_dict`)
elif len_new_dict!=0 :
clogger.debug("All strings contain not_pattern => no result")
#update size new dictionary
len_new_dict=len(new_dict)
#request on new dictionary through usage, count and remove
work_dic=new_dict if (var_find or pattern is not "" or not_pattern is not "") else crs2filename
if usage is True and len_new_dict != 0 :
#construction of a dictionary containing crs and disk-usage associated
dic_usage=dict()
tmp=""
for crs in work_dic :
tmp+=work_dic[crs]+" "
res=os.popen("du -sc %s"%tmp).read()
regex=re.compile('([0-9]+)\t')
list_size=re.findall(regex,res)
regex2=re.compile('([0-9]+\t)')
str_path=regex2.sub('',res)
list_fig=str_path.split('\n')
list_fig.pop(-1)
for fig,size in zip(list_fig,list_size):
if fig!="total":
for crs in work_dic:
if fig==work_dic[crs]:
dic_usage[crs]=size
else:
dic_usage[fig]=size
#sort of usage dictionary and units conversion
du_list_sort=dic_usage.items()
du_list_sort.sort(key=itemgetter(1),reverse=False)
unit=["K","M","G","T"]
for n,pair in enumerate(du_list_sort):
i=0
flt=float(pair[1])
while flt >= 1024. and i < 4:
flt/=1024.
i+=1
du_list_sort[n]=(du_list_sort[n][0],"%6.1f%s"%(flt,unit[i]))
if count is True : # Display total volume of found files
for fig, size in du_list_sort:
if fig=="total":
print "%7s : %s" %(size,fig)
else: #retrieve disk-usage of each found file and total volume
for fig,size in du_list_sort:
print "%7s : %s" %(size,fig)
elif count is True and len_new_dict != 0 :
print "Number of files found:", len(work_dic)
if CRS is True:
for crs in work_dic : print crs
elif remove is True and len_new_dict != 0 :
print "Removed files:"
list_tmp_crs=[]
list_tmp_crs=new_dict.keys() if (var_find or pattern is not "" or not_pattern is not "") else crs2filename.keys()
for crs in list_tmp_crs:
cdrop(crs, rm=True)
return(map(crewrite,list_tmp_crs))
else: #usage, count and remove are False
if var_find or pattern is not "" or not_pattern is not "" :
if len(new_dict) != 0 :
if new_dict != crs2filename :
print "Filtered objects :"
else :
print "Filtered objects = cache content"
return (map(crewrite,new_dict.keys()))
#else : print "No matching file "
else:
print "Content of CliMAF cache"
return (map(crewrite,crs2filename.keys()))
#TBD
if special is True :
global dic_special
dic_special=dict()
if var_find is True or pattern is not "" or not_pattern is not "" :
dic_special=new_dict.copy()
else:
dic_special=crs2filename.copy()
print "List of marked figures as 'special'", dic_special.values()
return(dic_special) #TBD: declarer comme var globale et enlever son effacement dans creset
new_dict.clear()
