def cdump(use_macro=True): """ List the in-memory content of CliMAF cache index. Interpret it using macros except if arg use_macro is False """ for crs in crs2filename: if not use_macro: # No interpretation by macros # print "%s : %s"%(crs2filename[crs][-30:],crs) print "%s : %s" % (crs2filename[crs], crs) else: # Must update for new macros print "%s : %s" % (crs2filename[crs], crewrite(crs))
def cdump(use_macro=True): """ List the in-memory content of CliMAF cache index. Interpret it using macros except if arg use_macro is False """ for crs in crs2filename : if not use_macro : # No interpretation by macros #print "%s : %s"%(crs2filename[crs][-30:],crs) print "%s : %s"%(crs2filename[crs],crs) else: # Must update for new macros print "%s : %s"%(crs2filename[crs],crewrite(crs))
def clist(size="", age="", access=0, pattern="", not_pattern="", usage=False, count=False, remove=False, CRS=False, special=False): """ Internal function used by its front-ends : :py:func:`~climaf.cache.cls`, :py:func:`~climaf.cache.crm`, :py:func:`~climaf.cache.cdu`, :py:func:`~climaf.cache.cwc` List the content of CliMAF cache according to some search criteria and operate possibly an action (usage, count or remove) on this list. Please consider the cost and benefit of first updating CliMAF cache index (by scanning files on disk) using :py:func:`csync()` Args: size (string, optional): n[ckMG] Search files using more than n units of disk space, rounding up. The following suffixes can be used: - "c" for bytes (default) - "k" for Kilobytes (units of 1,024 bytes) - "M" for Megabytes (units of 1,048,576 bytes) - "G" for Gigabytes (units of 1,073,741,824 bytes) age (string, optional): Number of 24h periods. Search files which status was last changed n*24 hours ago. Any fractional part is ignored, so to match age='+1', a file has to have been changed at least two days ago. Numeric arguments can be specified as: - `+n` for greater than n - `-n` for less than n, - `n` for exactly n. access (int, optional): n Search files which were last accessed more than n*24 hours ago. Any fractional part is ignored, so to match access='1', a file has to have been accessed at least two days ago. pattern (string, optional): Scan through crs and filenames looking for the first location where the regular expression pattern produces a match. not_pattern (string, optional): Scan through crs and filenames looking for the location where the regular expression not_pattern does not produce a match. usage (bool, optional): Estimate found files space usage, for each found file and total size. If count is True, estimate only found files total space usage. count (bool, optional): Return the number of found files. If CRS is True, also return crs of found files. remove (bool, optional): Remove the found files. This argument is exclusive. CRS (bool, optional): if True, print also CRS expression. Useful only if count is True. Return: The dictionary corresponding to the request and associated action ( or dictionary of CliMAF cache index if no argument is provided) Example to search files using more than 3M of disk space, which status was last changed more than 15 days ago and containing the pattern '1980-1981' either in crs or filename. For found files, we want to estimate only found files total space usage:: >>> clist(size='3M', age='+15', pattern= '1980-1981', usage=True, count=True) """ # cache directories rep = os.path.expanduser( cachedirs[0] ) # TBD: le cache ne contient qu un rep pr le moment => voir pour boucler sur tous les caches # command for research on size/age/access command = "" opt_find = "" if size: if re.search('[kMG]', size) is None: opt_find += "-size +%sc " % size else: opt_find += "-size +%s " % size if age: opt_find += "-ctime %s " % age if access != 0: opt_find += "-atime +%s" % str(int(access)) var_find = False if size or age or access != 0: var_find = True command = "find %s -type f \( -name '*.png' -o -name '*.nc' -o -name '*.pdf' -o -name '*.eps' \) %s -print" % \ (rep, opt_find) clogger.debug("Find command is :" + command) # construction of the new dictionary after research on size/age/access new_dict = dict() find_return = "" list_search_files_after_find = [] find_return = os.popen(command).read() list_search_files_after_find = find_return.split('\n') list_search_files_after_find.pop(-1) clogger.debug("List of search files: " + repr(list_search_files_after_find)) # Search CRS for each found file for filen in list_search_files_after_find: for crs in crs2filename: if crs2filename[crs] == filen: new_dict[crs] = filen if len(new_dict) != 0: if new_dict != crs2filename: clogger.debug("Dictionary after find for size/age/access: " + repr(new_dict)) else: clogger.debug( "Size/age/access criteria do not lead to any filtering") else: clogger.debug("No file meet the size/age/access criteria") else: new_dict = crs2filename.copy() # size of new dictionary len_new_dict = len(new_dict) # filter on pattern find_pattern = False if pattern: list_crs_to_rm = [] for crs in new_dict: if re.search(pattern, crewrite(crs)) or re.search( pattern, new_dict[crs]): clogger.debug("Pattern found in %s: %s" % (crs, new_dict[crs])) find_pattern = True else: # Do not remove now from new_dict, because we loop on it list_crs_to_rm.append(crs) for crs in list_crs_to_rm: del new_dict[crs] if find_pattern: clogger.debug("Dictionary after search for pattern: " + repr(new_dict)) elif len_new_dict != 0: clogger.debug("No string found for pattern => no result") # update size new dictionary len_new_dict = len(new_dict) # research on not_pattern find_not_pattern = False if not_pattern: list_crs_to_rm = [] for crs in new_dict: if re.search(not_pattern, crewrite(crs)) is None and \ re.search(not_pattern, new_dict[crs]) is None: clogger.debug("Pattern not found in %s: %s" % (crs, new_dict[crs])) find_not_pattern = True else: list_crs_to_rm.append(crs) for crs in list_crs_to_rm: del new_dict[crs] if find_not_pattern: clogger.debug("Dictionary after search for not_pattern: " + repr(new_dict)) elif len_new_dict != 0: clogger.debug("All strings contain not_pattern => no result") # update size new dictionary len_new_dict = len(new_dict) # request on new dictionary through usage, count and remove work_dic = new_dict if (var_find or pattern is not "" or not_pattern is not "") else crs2filename if usage is True and len_new_dict != 0: # construction of a dictionary containing crs and disk-usage associated dic_usage = dict() tmp = "" for crs in work_dic: tmp += work_dic[crs] + " " res = os.popen("du -sc %s" % tmp).read() regex = re.compile('([0-9]+)\t') list_size = re.findall(regex, res) regex2 = re.compile('([0-9]+\t)') str_path = regex2.sub('', res) list_fig = str_path.split('\n') list_fig.pop(-1) for fig, size in zip(list_fig, list_size): if fig != "total": for crs in work_dic: if fig == work_dic[crs]: dic_usage[crs] = size else: dic_usage[fig] = size # sort of usage dictionary and units conversion du_list_sort = dic_usage.items() du_list_sort.sort(key=itemgetter(1), reverse=False) unit = ["K", "M", "G", "T"] for n, pair in enumerate(du_list_sort): i = 0 flt = float(pair[1]) while flt >= 1024. and i < 4: flt /= 1024. i += 1 du_list_sort[n] = (du_list_sort[n][0], "%6.1f%s" % (flt, unit[i])) if count is True: # Display total volume of found files for fig, size in du_list_sort: if fig == "total": print "%7s : %s" % (size, fig) else: # retrieve disk-usage of each found file and total volume for fig, size in du_list_sort: print "%7s : %s" % (size, fig) elif count is True and len_new_dict != 0: print "Number of files found:", len(work_dic) if CRS is True: for crs in work_dic: print crs elif remove is True and len_new_dict != 0: print "Removed files:" list_tmp_crs = [] list_tmp_crs = new_dict.keys() if ( var_find or pattern is not "" or not_pattern is not "") else crs2filename.keys() for crs in list_tmp_crs: cdrop(crs, rm=True) return map(crewrite, list_tmp_crs) else: # usage, count and remove are False if var_find or pattern is not "" or not_pattern is not "": if len(new_dict) != 0: if new_dict != crs2filename: print "Filtered objects :" else: print "Filtered objects = cache content" return map(crewrite, new_dict.keys()) # else : print "No matching file " else: print "Content of CliMAF cache" return map(crewrite, crs2filename.keys()) # TBD if special is True: global dic_special dic_special = dict() if var_find is True or pattern is not "" or not_pattern is not "": dic_special = new_dict.copy() else: dic_special = crs2filename.copy() print "List of marked figures as 'special'", dic_special.values() return dic_special # TBD: declarer comme var globale et enlever son effacement dans creset new_dict.clear()
def clist(size="", age="", access=0, pattern="", not_pattern="", usage=False, count=False, remove=False, CRS=False, special=False): """ Internal function used by its front-ends : :py:func:`~climaf.cache.cls`, :py:func:`~climaf.cache.crm`, :py:func:`~climaf.cache.cdu`, :py:func:`~climaf.cache.cwc` List the content of CliMAF cache according to some search criteria and operate possibly an action (usage, count or remove) on this list. Please consider the cost and benefit of first updating CliMAF cache index (by scanning files on disk) using :py:func:`csync()` Args: size (string, optional): n[ckMG] Search files using more than n units of disk space, rounding up. The following suffixes can be used: - "c" for bytes (default) - "k" for Kilobytes (units of 1,024 bytes) - "M" for Megabytes (units of 1,048,576 bytes) - "G" for Gigabytes (units of 1,073,741,824 bytes) age (string, optional): Number of 24h periods. Search files which status was last changed n*24 hours ago. Any fractional part is ignored, so to match age='+1', a file has to have been changed at least two days ago. Numeric arguments can be specified as: - `+n` for greater than n - `-n` for less than n, - `n` for exactly n. access (int, optional): n Search files which were last accessed more than n*24 hours ago. Any fractional part is ignored, so to match access='1', a file has to have been accessed at least two days ago. pattern (string, optional): Scan through crs and filenames looking for the first location where the regular expression pattern produces a match. not_pattern (string, optional): Scan through crs and filenames looking for the location where the regular expression not_pattern does not produce a match. usage (bool, optional): Estimate found files space usage, for each found file and total size. If count is True, estimate only found files total space usage. count (bool, optional): Return the number of found files. If CRS is True, also return crs of found files. remove (bool, optional): Remove the found files. This argument is exclusive. CRS (bool, optional): if True, print also CRS expression. Useful only if count is True. Return: The dictionary corresponding to the request and associated action ( or dictionary of CliMAF cache index if no argument is provided) Example to search files using more than 3M of disk space, which status was last changed more than 15 days ago and containing the pattern '1980-1981' either in crs or filename. For found files, we want to estimate only found files total space usage:: >>> clist(size='3M', age='+15', pattern= '1980-1981', usage=True, count=True) """ #cache directories rep=os.path.expanduser(cachedirs[0]) #TBD: le cache ne contient qu un rep pr le moment => voir pour boucler sur tous les caches #command for research on size/age/access command="" opt_find="" if size : if re.search('[kMG]', size) is None : opt_find+="-size +%sc "%size else: opt_find+="-size +%s "%size if age : opt_find+="-ctime %s "%age if access !=0 : opt_find+="-atime +%s"%str(int(access)) var_find=False if size or age or access != 0 : var_find=True command="find %s -type f \( -name '*.png' -o -name '*.nc' \) %s -print" %(rep, opt_find) clogger.debug("Find command is :"+command) #construction of the new dictionary after research on size/age/access new_dict=dict() find_return="" list_search_files_after_find=[] find_return=os.popen(command).read() list_search_files_after_find=find_return.split('\n') list_search_files_after_find.pop(-1) clogger.debug("List of search files: "+`list_search_files_after_find`) # Search CRS for each found file for filen in list_search_files_after_find : for crs in crs2filename: if crs2filename[crs]==filen: new_dict[crs]=filen if len(new_dict) != 0 : if new_dict != crs2filename : clogger.debug("Dictionary after find for size/age/access: "+`new_dict`) else : clogger.debug("Size/age/access criteria do not lead to any filtering") else : clogger.debug("No file meet the size/age/access criteria") else: new_dict=crs2filename.copy() #size of new dictionary len_new_dict=len(new_dict) #filter on pattern find_pattern=False if pattern : list_crs_to_rm=[] for crs in new_dict : if re.search(pattern, crewrite(crs)) or re.search(pattern, new_dict[crs]): clogger.debug("Pattern found in %s: %s"%(crs,new_dict[crs])) find_pattern=True else: # Do not remove now from new_dict, because we loop on it list_crs_to_rm.append(crs) for crs in list_crs_to_rm : del new_dict[crs] if find_pattern : clogger.debug("Dictionary after search for pattern: "+`new_dict`) elif len_new_dict!=0 : clogger.debug("No string found for pattern => no result") #update size new dictionary len_new_dict=len(new_dict) #research on not_pattern find_not_pattern=False if not_pattern: list_crs_to_rm=[] for crs in new_dict : if re.search(not_pattern, crewrite(crs)) is None and \ re.search(not_pattern, new_dict[crs]) is None : clogger.debug("Pattern not found in %s: %s"%(crs, new_dict[crs])) find_not_pattern=True else: list_crs_to_rm.append(crs) for crs in list_crs_to_rm : del new_dict[crs] if find_not_pattern : clogger.debug("Dictionary after search for not_pattern: "+`new_dict`) elif len_new_dict!=0 : clogger.debug("All strings contain not_pattern => no result") #update size new dictionary len_new_dict=len(new_dict) #request on new dictionary through usage, count and remove work_dic=new_dict if (var_find or pattern is not "" or not_pattern is not "") else crs2filename if usage is True and len_new_dict != 0 : #construction of a dictionary containing crs and disk-usage associated dic_usage=dict() tmp="" for crs in work_dic : tmp+=work_dic[crs]+" " res=os.popen("du -sc %s"%tmp).read() regex=re.compile('([0-9]+)\t') list_size=re.findall(regex,res) regex2=re.compile('([0-9]+\t)') str_path=regex2.sub('',res) list_fig=str_path.split('\n') list_fig.pop(-1) for fig,size in zip(list_fig,list_size): if fig!="total": for crs in work_dic: if fig==work_dic[crs]: dic_usage[crs]=size else: dic_usage[fig]=size #sort of usage dictionary and units conversion du_list_sort=dic_usage.items() du_list_sort.sort(key=itemgetter(1),reverse=False) unit=["K","M","G","T"] for n,pair in enumerate(du_list_sort): i=0 flt=float(pair[1]) while flt >= 1024. and i < 4: flt/=1024. i+=1 du_list_sort[n]=(du_list_sort[n][0],"%6.1f%s"%(flt,unit[i])) if count is True : # Display total volume of found files for fig, size in du_list_sort: if fig=="total": print "%7s : %s" %(size,fig) else: #retrieve disk-usage of each found file and total volume for fig,size in du_list_sort: print "%7s : %s" %(size,fig) elif count is True and len_new_dict != 0 : print "Number of files found:", len(work_dic) if CRS is True: for crs in work_dic : print crs elif remove is True and len_new_dict != 0 : print "Removed files:" list_tmp_crs=[] list_tmp_crs=new_dict.keys() if (var_find or pattern is not "" or not_pattern is not "") else crs2filename.keys() for crs in list_tmp_crs: cdrop(crs, rm=True) return(map(crewrite,list_tmp_crs)) else: #usage, count and remove are False if var_find or pattern is not "" or not_pattern is not "" : if len(new_dict) != 0 : if new_dict != crs2filename : print "Filtered objects :" else : print "Filtered objects = cache content" return (map(crewrite,new_dict.keys())) #else : print "No matching file " else: print "Content of CliMAF cache" return (map(crewrite,crs2filename.keys())) #TBD if special is True : global dic_special dic_special=dict() if var_find is True or pattern is not "" or not_pattern is not "" : dic_special=new_dict.copy() else: dic_special=crs2filename.copy() print "List of marked figures as 'special'", dic_special.values() return(dic_special) #TBD: declarer comme var globale et enlever son effacement dans creset new_dict.clear()