def process_resolution(params, reslist, nvar, nfiles, currfile):
'''
Processes files for variable defined by params for all resolutions in reslist
Parameters
----------
params : list
Row in variables table corresponding to variable processed in the call of this function
reslist : list
List of resolutions the variable should be processed at
nvar : int
Number of variables (needed for progress bar)
nfiles : int
Currently estimated total number of files for all variables (needed for progress bar)
currfile : int
Currently already processed number of files
'''
#Do seasonal resolution=
if "sem" in reslist:
seasonal = True
reslist.remove("sem")
else:
seasonal = False
log = logging.getLogger("cmorlight")
# get cdf variable name
var = params[config.get_config_value('index', 'INDEX_VAR')]
varRCM = params[config.get_config_value('index', 'INDEX_RCM_NAME')]
# create path to input files from basedir,model,driving_model
in_dir = "%s/%s" % (tools.get_input_path(), params[config.get_config_value(
'index', 'INDEX_RCM_NAME')])
log.debug("Looking for input dir(1): %s" % (in_dir))
if os.path.isdir(in_dir) == False:
log.error(
"Input directory does not exist(0): %s \n \t Change base path in .ini file or create directory! "
% in_dir)
return nfiles, currfile
cores = config.get_config_value("integer", "multi", exitprog=False)
multilst = []
seaslst = []
log.info("Used dir: %s" % (in_dir))
for dirpath, dirnames, filenames in os.walk(in_dir, followlinks=True):
if not nfiles:
#estimate total
if config.get_config_value('boolean', 'limit_range'):
nfiles = nvar * (
config.get_config_value('integer', 'proc_end') -
config.get_config_value('integer', 'proc_start') + 1)
else:
nfiles = nvar * len(filenames)
if len(filenames) == 0:
log.warning("No files found! Skipping this variable...")
i = 0
for f in sorted(filenames):
if f[-3:] != ".nc":
continue
if var not in settings.var_list_fixed:
year = f.split("_")[-1][:4]
#use other logger
if cores > 1 and var not in settings.var_list_fixed:
logger = logging.getLogger("cmorlight_" + year)
logger.info(
"\n###########################################################\n# Var in work: %s / %s\n###########################################################"
% (var, varRCM))
logger.info("Start processing at: " +
str(datetime.datetime.now()))
else:
logger = logging.getLogger("cmorlight")
#if limit_range is set: skip file if it is out of range
if config.get_config_value(
'boolean',
'limit_range') and var not in settings.var_list_fixed:
if int(year) < config.get_config_value(
'integer',
'proc_start') or int(year) > config.get_config_value(
'integer', 'proc_end'):
continue
#Define first and last month of file
if config.get_config_value('integer',
"proc_start") == int(year):
firstlast = [
config.get_config_value('integer', "first_month"), 12
]
elif config.get_config_value('integer',
"proc_end") == int(year):
firstlast = [
1, config.get_config_value('integer', "last_month")
]
else:
firstlast = [1, 12]
else:
firstlast = [1, 12]
logger.info(
"\n###########################################################"
)
if f.find("%s_" % var) == 0 or f.find(
"%s.nc" % var) == 0 or f.find(
"%s_" % varRCM) == 0 or f.find(
"%s.nc" % varRCM) == 0 or f.find(
"%s_" % varRCM[:varRCM.find('p')]) == 0:
in_file = "%s/%s" % (dirpath, f)
logger.log(35, "Input from: %s" % (in_file))
if os.access(in_file, os.R_OK) == False:
logger.error("Could not read file '%s', no permission!" %
in_file)
else:
if var in settings.var_list_fixed:
tools.process_file_fix(params, in_file)
else:
if cores > 1:
multilst.append([
params, in_file, var, reslist, year, firstlast
])
seaslst.append([params, year])
else:
reslist = tools.process_file(
params, in_file, var, reslist, year, firstlast)
if seasonal:
tools.proc_seasonal(params, year)
else:
logger.warning(
"File %s does match the file name conventions for this variable. File not processed..."
)
i = i + 1
#process as many files simultaneously as there are cores specified
if i == cores and multilst != []:
log.info("Processing years %s to %s simultaneously" %
(seaslst[0][1], seaslst[-1][1]))
pool = Pool(processes=cores)
R = pool.map(process_file_unpack, multilst)
pool.terminate()
#seasonal processing:
if seasonal:
pool = Pool(processes=cores)
pool.map(proc_seasonal_unpack, seaslst)
pool.terminate()
currfile += len(multilst)
#start new
multilst = []
seaslst = []
i = 0
#change reslist
reslist = R[0]
#update currfile
if cores <= 1:
currfile += 1
#print progress bar
tools.print_progress(currfile, nfiles)
#process remaining files
if len(multilst) != 0:
log.info("Processing years %s to %s simultaneously" %
(seaslst[0][1], seaslst[-1][1]))
pool = Pool(processes=len(multilst))
R = pool.map(process_file_unpack, multilst)
pool.terminate()
#seasonal processing:
if seasonal:
pool = Pool(processes=cores)
pool.map(proc_seasonal_unpack, seaslst)
pool.terminate()
#update currfile
currfile += len(multilst)
tools.print_progress(currfile, nfiles)
log.info("Variable '%s' finished!" % (var))
return nfiles, currfile
def proc_seasonal_unpack(args):
'''Helper function for multiprocessing to unpack arguments before using pool.map (seasonal processing)'''
tools.proc_seasonal(*args)