def run_command(cmd):
"""
Run command as subprocess
:param cmd: command to be run
:type cmd: list (of str)
:rtype: str or None
"""
_log.debug('External command execution: {c}'.format(c=cmd))
try:
sproc = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = sproc.communicate()
exitcode = sproc.returncode
if exitcode == 0:
_log.debug("Execution succeeded: {o}".format(
o=stdout.replace('\r', ' ').replace('\n', ' ')))
if stderr:
_log.warning("Execution STDERR: {o}".format(
o=stderr.replace('\r', ' ').replace('\n', ' ')))
else:
msg = 'Execution failed! EXITCODE: {c} STDOUT: {o} STDERR: {e}'.format(
c=exitcode,
o=stdout.replace('\r', ' ').replace('\n', ' '),
e=stderr.replace('\r', ' ').replace('\n', ' '))
_log.critical(msg)
raise ONEFluxError(msg)
except subprocess.CalledProcessError as e:
msg = "Execution raised error: '{e}'".format(e=str(e))
_log.critical(msg)
raise ONEFluxError(msg)
return stdout
def get_created_ustar_years(mpdir, cpdir):
mpfiles = [
f for f in os.listdir(mpdir) if os.path.isfile(os.path.join(mpdir, f))
]
cpfiles = [
f for f in os.listdir(cpdir) if os.path.isfile(os.path.join(cpdir, f))
]
mpyears_all = [FOUR_DIGIT_RE.findall(f) for f in mpfiles]
mpyears_all = [i for i in mpyears_all if i]
cpyears_all = [FOUR_DIGIT_RE.findall(f) for f in cpfiles]
cpyears_all = [i for i in cpyears_all if i]
for y_list in mpyears_all + cpyears_all:
if len(y_list) != 1:
raise ONEFluxError(
"Multiple years ({y}) found in MP/CP filenames: {f}".format(
y=y_list, f=mpfiles + cpfiles))
year = int(y_list[0])
if 1900 > year > 2100:
raise ONEFluxError(
"Year ({y}) out-of-range found in MP/CP filenames: {f}".format(
y=year, f=mpfiles + cpfiles))
mpyears = [int(l[0]) for l in mpyears_all]
cpyears = [int(l[0]) for l in cpyears_all]
return mpyears, cpyears
def ipolmiss(data, variable):
"""
Interpolates missing values in data for variable
(changes data object directly)
Note: original code allowed multiple variables to be interpolated
in a single call to ipolmiss, but only used as a single variable
at a time; this implementation assumes single variable for
each call
:param data: data structure for partitioning
:type data: numpy.ndarray
:param variable: variable to be interpolated
:type variable: str
"""
_log.debug("Interpolating variable '{v}'".format(v=variable))
#order = 2 # always 2 in all calls from original code; means linear interpolation
method = 'Exact' # always 'Exact' from original code
if not isinstance(data, numpy.ndarray):
msg = "ipolmiss ({v}) data object is not ndarray: '{d}'".format(v=variable, d=str(data))
_log.critical(msg)
raise ONEFluxError(msg)
if variable not in data.dtype.names:
msg = "ipolmiss ({v}) label not in data object".format(v=variable)
_log.critical(msg)
raise ONEFluxError(msg)
mask = not_nan(data[variable])
count = numpy.sum(mask)
if (count > 1) and ((count < data.size) or (method == 'LSQ')):
idx = numpy.where(mask)[0]
julday = data['julday'] + (data['hr'] / 24.0)
duration = [i - julday[mask][0] for i in julday[mask]]
if count < 6:
_log.error("ipolmiss ({v}) too few elements: {c}".format(v=variable, c=count))
# create interpolation function
sp_interp_function = interp1d(duration, data[variable][mask], kind='linear', bounds_error=False, fill_value=numpy.NaN)
# apply interpolation function to full extent of dataset
duration_full = [i - julday[mask][0] for i in julday]
data[variable][:] = sp_interp_function(duration_full)
# set beginning/end gaps into first/last valid value
data[variable][:idx[0]] = data[variable][idx[0]]
data[variable][idx[-1] + 1:] = data[variable][idx[-1]]
def load_ustar_cut(siteid,
sitedir,
first_year,
last_year,
nee_perc_ustar_cut_template=NEE_PERC_USTAR_CUT):
nee_perc_ustar_cut = nee_perc_ustar_cut_template.format(s=siteid,
sd=sitedir,
fy=first_year,
ly=last_year)
log.debug("{s}: processing file: {f}".format(s=siteid,
f=nee_perc_ustar_cut))
nee_perc_ustar_cut_lines = load_csv_lines(filename=nee_perc_ustar_cut)
if (last_year - first_year) < 2:
if not nee_perc_ustar_cut_lines:
log.warning("{s}: too few years, {e} file not created: {f}".format(
s=siteid, e='NEE CUT USTAR percentiles', f=nee_perc_ustar_cut))
nee_perc_ustar_cut_values = {
k: '-9999'
for i, k in enumerate(PERCENTILES)
}
nee_perc_ustar_cut_values['50.00'] = nee_perc_ustar_cut_values['50']
else:
if not nee_perc_ustar_cut_lines:
raise ONEFluxError("{s}: {e} file not found: {f}".format(
s=siteid, e='NEE CUT USTAR percentiles', f=nee_perc_ustar_cut))
if (len(nee_perc_ustar_cut_lines) == 3
and not nee_perc_ustar_cut_lines[2].strip()
) or len(nee_perc_ustar_cut_lines) > 2:
raise ONEFluxError(
"{s}: NEE USTAR CUT file too many lines ({l}): {f}".format(
s=siteid,
l=len(nee_perc_ustar_cut_lines),
f=nee_perc_ustar_cut))
elif not (nee_perc_ustar_cut_lines[0][0].startswith(PERCENTILES[0])
and nee_perc_ustar_cut_lines[0][-1].strip().endswith(
PERCENTILES[-1])):
raise ONEFluxError(
"{s}: NEE USTAR CUT bad headers ({h}): {f}".format(
s=siteid,
h=nee_perc_ustar_cut_lines[0],
f=nee_perc_ustar_cut))
nee_perc_ustar_cut_values = {
k: nee_perc_ustar_cut_lines[1][i].strip()
for i, k in enumerate(PERCENTILES)
}
nee_perc_ustar_cut_values['50.00'] = nee_perc_ustar_cut_values['50']
return nee_perc_ustar_cut_values
def load_ustar_vut(siteid, sitedir, year_range, nee_perc_ustar_vut_template=NEE_PERC_USTAR_VUT):
nee_perc_ustar_vut = nee_perc_ustar_vut_template.format(s=siteid, sd=sitedir)
log.debug("{s}: processing file: {f}".format(s=siteid, f=nee_perc_ustar_vut))
nee_perc_ustar_vut_lines = load_csv_lines(filename=nee_perc_ustar_vut)
if not nee_perc_ustar_vut_lines: raise ONEFluxError("{s}: {e} file not found: {f}".format(s=siteid, e='NEE VUT USTAR percentiles', f=nee_perc_ustar_vut))
nee_perc_ustar_vut_values = {i:{} for i in year_range}
if not ((nee_perc_ustar_vut_lines[0][0].lower().startswith('timestamp') or\
nee_perc_ustar_vut_lines[0][0].lower().startswith('isodate') or\
nee_perc_ustar_vut_lines[0][0].lower().startswith('year')) and\
nee_perc_ustar_vut_lines[0][-1].endswith(PERCENTILES[-1])):
raise ONEFluxError("{s}: NEE USTAR VUT bad headers ({h}): {f}".format(s=siteid, h=nee_perc_ustar_vut_lines[0], f=nee_perc_ustar_vut))
elif (int(nee_perc_ustar_vut_lines[1][0]) != year_range[0]) or (int(nee_perc_ustar_vut_lines[-1][0]) != year_range[-1]):
raise ONEFluxError("{s}: NEE USTAR VUT incompatible year range data=({d}), info=({i})".format(s=siteid, d="{f}-{l}".format(f=nee_perc_ustar_vut_lines[1][0], l=nee_perc_ustar_vut_lines[-1][0]), i="{f}-{l}".format(f=year_range[0], l=year_range[-1])))
for y, year in enumerate(year_range):
nee_perc_ustar_vut_values[year] = {k:nee_perc_ustar_vut_lines[y + 1][i + 1].strip() for i, k in enumerate(PERCENTILES) }
nee_perc_ustar_vut_values[year]['50.00'] = nee_perc_ustar_vut_values[year]['50']
return nee_perc_ustar_vut_values
def get_empty_array_year(year=datetime.now().year, start_end=True, variable_list=['TEST', ], variable_list_dtype=None, record_interval='HH'):
"""
Allocates and returns new empty record array for given year using list of dtypes
(or variable labels as 8byte floats if no dtype list provided) for variables plus
TIMESTAMP_START and TIMESTAMP_END at beginning
:param year: year to be represented in array (current year if not provided)
:type year: int
:param start_end: if True, uses TIMESTAMP_START and TIMESTAMP_END, if not, uses only TIMESTAMP (end)
:type start_end: bool
:param variable_list: list of strings to be used as variable labels (assumed f8 type)
:type variable_list: list (of str)
:param variable_list_dtype: list of dtype tuples (label, data type) to be used as variables
:type variable_list_dtype: list (of (str, str)-tuples)
:param record_interval: resolution to be used for record ['HR' for hourly, 'HH' for half-hourly (default)]
:type record_interval: str
"""
# record_interval
if record_interval.lower() == 'hh':
step = timedelta(minutes=30)
elif record_interval.lower() == 'hr':
step = timedelta(minutes=60)
else:
msg = 'Unknown record_interval: {r}'.format(r=record_interval)
log.critical(msg)
raise ONEFluxError(msg)
# timestamp list
timestamp_list = []
current_timestamp = datetime(int(year), 1, 1, 0, 0, 0)
while current_timestamp.year < int(year) + 1:
timestamp_list.append(current_timestamp)
current_timestamp += step
timestamp_list.append(current_timestamp)
timestamp_list_begin = timestamp_list[:-1]
timestamp_list_end = timestamp_list[1:]
# array dtype
dtype = ([(var, 'f8') for var in variable_list] if variable_list_dtype is None else variable_list_dtype)
if start_end:
dtype = [('TIMESTAMP_START', 'a25'), ('TIMESTAMP_END', 'a25')] + dtype
else:
dtype = [('TIMESTAMP', 'a25'), ] + dtype
# record array
data = numpy.zeros(len(timestamp_list_begin), dtype=dtype)
data[:] = -9999.0
if start_end:
data['TIMESTAMP_START'][:] = [i.strftime('%Y%m%d%H%M') for i in timestamp_list_begin]
data['TIMESTAMP_END'][:] = [i.strftime('%Y%m%d%H%M') for i in timestamp_list_end]
else:
data['TIMESTAMP'][:] = [i.strftime('%Y%m%d%H%M') for i in timestamp_list_end]
return data
def run_pipeline(datadir, siteid, sitedir, firstyear, lastyear, version_data=VERSION_METADATA, version_proc=VERSION_PROCESSING, prod_to_compare=PROD_TO_COMPARE, perc_to_compare=PERC_TO_COMPARE, mcr_directory=None, timestamp=NOW_TS, record_interval='hh'):
sitedir_full = os.path.abspath(os.path.join(datadir, sitedir))
if not sitedir or not os.path.isdir(sitedir_full):
msg = "Site directory for {s} not found: '{d}'".format(s=siteid, d=sitedir)
log.critical(msg)
raise ONEFluxError(msg)
log.info("Started processing site dir {d}".format(d=sitedir))
try:
pipeline = Pipeline(siteid=siteid,
data_dir=sitedir_full,
data_dir_main=os.path.abspath(datadir),
site_dir=sitedir,
tool_dir=TOOL_DIRECTORY,
first_year=firstyear,
last_year=lastyear,
prod_to_compare=prod_to_compare,
perc_to_compare=perc_to_compare,
timestamp=timestamp,
record_interval=record_interval,
fluxnet2015_first_t1=firstyear,
fluxnet2015_last_t1=lastyear,
fluxnet2015_version_data=version_data,
fluxnet2015_version_processing=version_proc,
ustar_cp_mcr_dir=mcr_directory,
qc_auto_execute=True,
ustar_mp_execute=True,
ustar_cp_execute=True,
meteo_proc_execute=True,
nee_proc_execute=True,
energy_proc_execute=True,
nee_partition_nt_execute=True,
nee_partition_dt_execute=True,
prepare_ure_execute=True,
ure_execute=True,
fluxnet2015_execute=True,
fluxnet2015_site_plots=True,
simulation=False)
pipeline.run()
#csv_manifest_entries, zip_manifest_entries = pipeline.fluxnet2015.csv_manifest_entries, pipeline.fluxnet2015.zip_manifest_entries
log.info("Finished processing site dir {d}".format(d=sitedir_full))
except ONEFluxPipelineError as e:
log.critical("ONEFlux Pipeline ERRORS processing site dir {d}".format(d=sitedir_full))
log_trace(exception=e, level=logging.CRITICAL, log=log)
raise
except ONEFluxError as e:
log.critical("ONEFlux ERRORS processing site dir {d}".format(d=sitedir_full))
log_trace(exception=e, level=logging.CRITICAL, log=log)
raise
except Exception as e:
log.critical("UNKNOWN ERRORS processing site dir {d}".format(d=sitedir_full))
log_trace(exception=e, level=logging.CRITICAL, log=log)
raise
def compress_file(filename):
"""
Compresses file with gzip
:param filename: path to file to be compressed
:type filename: str
:rtype: str or None
"""
if not os.path.isfile(filename):
msg = "File not found or cannot be accessed: '{f}'".format(f=filename)
_log.critical(msg)
raise ONEFluxError(msg)
try:
r = subprocess.call(['gzip', '-f', filename])
_log.debug("Compressing file '{d}'. Result: {o}".format(
d=filename, o=('success' if r == 0 else "fail ({r})".format(r=r))))
except subprocess.CalledProcessError, e:
msg = "Problems compressing file '{d}'. Error: '{e}'".format(
d=filename, e=str(e))
_log.critical(msg)
raise ONEFluxError(msg)
def check_create_directory(directory):
"""
Checks if directory exists and creates if not
:param directory: path to be tested/created
:type directory: str
"""
if not os.path.isdir(directory):
if os.path.exists(directory):
msg = "Directory check: not a directory '{p}'".format(p=directory)
_log.critical(msg)
raise ONEFluxError(msg)
else:
os.makedirs(directory)
_log.info("Created directory '{p}'".format(p=directory))
return
def get_headers(filename):
"""
Parse headers from FPFileV2 format and returns list
of string with header labels.
Must have at least two columns.
:param filename: name of the FPFileV2 to be loaded
:type filename: str
:rtype: list
"""
with open(filename, 'r') as f:
line = f.readline()
headers = line.strip().split(',')
if len(headers) < 2:
raise ONEFluxError("Headers too short: '{h}'".format(h=line))
headers = [i.strip() for i in headers]
return headers
def pct(array, percent):
"""
Calculates "percent" percentile of array -- not really a percentile,
but similar intention. Following implementation in original code.
:param array: 1-d array to be used in calculation
:type array: numpy.ndarray
:param percent: target percent value for percentile
:type percent: float
"""
nonnan_mask = not_nan(array)
if numpy.sum(nonnan_mask) > 1:
nonnan_array = array[nonnan_mask]
else:
msg = "No non-NA value in percentile calculation"
_log.critical(msg)
raise ONEFluxError(msg)
# indices of ascending ranking of entries in array
rank_idx_array = rankdata(nonnan_array, method='ordinal')
critical_rank = len(nonnan_array) * percent / 100.
over_critical_rank_mask = (rank_idx_array > critical_rank)
# if no index over critical rank, return max values
if numpy.sum(over_critical_rank_mask) == 0.0:
return numpy.max(nonnan_array)
### smallest rank that is greater than critical rank (or SM-RK-GT-CR)
critical_rank_idx = numpy.where(rank_idx_array == numpy.min(rank_idx_array[over_critical_rank_mask]))
### rank immediately before (SM-RK-GT-CR)
critical_rank_idx_previous = numpy.where(rank_idx_array == (numpy.min(rank_idx_array[over_critical_rank_mask]) - 1))
if critical_rank.is_integer() and (numpy.sum(critical_rank_idx_previous) != 0):
return numpy.average([nonnan_array[critical_rank_idx[0]], nonnan_array[critical_rank_idx_previous[0]]])
else:
return nonnan_array[critical_rank_idx[0]][0]
py_remove_old=False):
"""
Runs nighttime partitioning
:param sitedir: absolute path to data directory
:type sitedir: str
:param siteid: site id in CC-SSS format
:type siteid: str
:param sitedir: relative path to data directory for site data
:type sitedir: str
:param years_to_compare: list of years to be compared
:type years_to_compare: list
:param nt_dir: template name for directory with nighttime partitioning data
:type nt_dir: str
:param filename_template: template of filename for nighttime partitioning data
(product, percentile, siteid, year, format, extension)
:type filename_template: str
:param prod_to_compare: list of products to compare ('c', 'y') - CUT/VUT
:type prod_to_compare: list
:param perc_to_compare: list of percentiles to compare ('1.25', '3.75', ..., '50', ..., '96.25', '98.75')
:type perc_to_compare: list
:param py_remove_old: if True, removes old python partitioning results (after backup), file has to be missing for run
:type py_remove_old: bool
"""
remove_previous_run(datadir=datadir, siteid=siteid, sitedir=sitedir, python=py_remove_old, prod_to_compare=prod_to_compare, perc_to_compare=perc_to_compare, years_to_compare=years_to_compare)
run_python(datadir=datadir, siteid=siteid, sitedir=sitedir, prod_to_compare=prod_to_compare, perc_to_compare=perc_to_compare, years_to_compare=years_to_compare)
if __name__ == '__main__':
raise ONEFluxError('Not executable')
msg += ", site-id ({i})".format(i=args.siteid)
msg += ", site-dir ({d})".format(d=args.sitedir)
msg += ", first-year ({y})".format(y=firstyear)
msg += ", last-year ({y})".format(y=lastyear)
msg += ", perc ({i})".format(i=perc)
msg += ", prod ({i})".format(i=prod)
msg += ", log-file ({f})".format(f=args.logfile)
msg += ", force-py ({i})".format(i=args.forcepy)
log.debug(msg)
# start execution
try:
# check arguments
print(os.path.join(args.datadir, args.sitedir))
if not os.path.isdir(os.path.join(args.datadir, args.sitedir)):
raise ONEFluxError("Site dir not found: {d}".format(d=args.sitedir))
# run command
log.info("Starting execution: {c}".format(c=args.command))
if args.command == 'all':
run_pipeline(datadir=args.datadir, siteid=args.siteid, sitedir=args.sitedir, firstyear=firstyear, lastyear=lastyear,
prod_to_compare=prod, perc_to_compare=perc, mcr_directory=args.mcr_directory, timestamp=args.timestamp,
record_interval=args.recint)
elif args.command == 'partition_nt':
run_partition_nt(datadir=args.datadir, siteid=args.siteid, sitedir=args.sitedir, years_to_compare=range(firstyear, lastyear + 1),
py_remove_old=args.forcepy, prod_to_compare=prod, perc_to_compare=perc)
elif args.command == 'partition_dt':
run_partition_dt(datadir=args.datadir, siteid=args.siteid, sitedir=args.sitedir, years_to_compare=range(firstyear, lastyear + 1),
py_remove_old=args.forcepy, prod_to_compare=prod, perc_to_compare=perc)
else:
raise ONEFluxError("Unknown command: {c}".format(c=args.command))
'GPP_DT_CUT_USTAR50',
'GPP_DT_CUT_MEAN',
'GPP_DT_CUT_SE',
] + \
['GPP_DT_CUT_{n}'.format(n=i) for i in ['05', '16', '25', '50', '75', '84', '95']] + \
[
### PARTITIONING SUNDOWN
'RECO_SR',
'RECO_SR_N'
]
for i, e in enumerate(VARIABLE_LIST_FULL):
if VARIABLE_LIST_FULL.count(e) != 1:
raise ONEFluxError(
"Duplicated variable VARIABLE_LIST_FULL[{i}]: {v}".format(i=i,
v=e))
VARIABLE_LIST_SUB = [
### TIMEKEEPING
'TIMESTAMP',
'TIMESTAMP_START',
'TIMESTAMP_END',
### MICROMETEOROLOGICAL
'TA_F',
'TA_F_QC',
'SW_IN_POT',
'SW_IN_F',
'SW_IN_F_QC',
'LW_IN_F',
def partitioning_nt(datadir, siteid, sitedir, prod_to_compare, perc_to_compare, years_to_compare):
"""
NT partitioning wrapper function.
Handles all "versions" (percentiles, CUT/VUT, years, etc)
:param datadir: main data directory (full path)
:type datadir: str
:param siteid: site flux id to be processed - in format CC-SSS
:type siteid: str
:param sitedir: data directory for site (relative path to datadir)
:type sitedir: str
:param prod_to_compare: list of products to compare - ['c', 'y']
:type prod_to_compare: list (of str)
:param perc_to_compare: list of percentiles to compare - ['1.25', '3.75', ..., '96.25', '98.75']
:type perc_to_compare: list (of str)
:param years_to_compare: list of years to compare - [1996, 1997, ... , 2014]
:type years_to_compare: list (of int)
"""
_log.info("Started NT partitioning of {s}".format(s=siteid))
sitedir_full = os.path.join(datadir, sitedir)
qc_auto_dir = os.path.join(sitedir_full, QC_AUTO_DIR)
meteo_proc_dir = os.path.join(sitedir_full, METEO_PROC_DIR)
nee_proc_dir = os.path.join(sitedir_full, NEE_PROC_DIR)
nt_output_dir = os.path.join(sitedir_full, NT_OUTPUT_DIR)
# reformat percentiles to compare into data column labels
percentiles_data_columns = [i.replace('.', HEADER_SEPARATOR) for i in perc_to_compare]
# check and create output dir if needed
if os.path.isdir(sitedir_full) and not os.path.isdir(nt_output_dir):
check_create_directory(directory=nt_output_dir)
# load meteo proc results
meteo_proc_f = os.path.join(meteo_proc_dir, '{s}_meteo_hh.csv'.format(s=siteid))
if not os.path.isfile(meteo_proc_f):
msg = "Meteo proc file not found '{f}'".format(f=meteo_proc_f)
_log.critical(msg)
raise ONEFluxError(msg)
_log.info("Will now load meteo file '{f}'".format(f=meteo_proc_f))
whole_dataset_meteo, headers_meteo, timestamp_list_meteo, year_list_meteo = load_output(meteo_proc_f)
# iterate through UStar threshold types
for ustar_type in prod_to_compare:
_log.info("Started processing UStar threshold type '{u}'".format(u=ustar_type))
# load nee proc results (percentiles file)
nee_proc_percentiles_f = os.path.join(nee_proc_dir, '{s}_NEE_percentiles_{u}_hh.csv'.format(s=siteid, u=ustar_type))
if not os.path.isfile(nee_proc_percentiles_f):
msg = "NEE proc file not found '{f}', trying '{n}'".format(f=nee_proc_percentiles_f, n='{f}')
nee_proc_percentiles_f = os.path.join(nee_proc_dir, '{s}_NEE_percentiles_{u}.csv'.format(s=siteid, u=ustar_type))
msg = msg.format(f=nee_proc_percentiles_f)
_log.info(msg)
if not os.path.isfile(nee_proc_percentiles_f):
if ustar_type == 'y':
msg = "NEE proc file not found '{f}'".format(f=nee_proc_percentiles_f)
_log.critical(msg)
raise ONEFluxError(msg)
elif ustar_type == 'c':
msg = "NEE proc file not found '{f}', skipping (CUT not computed?)".format(f=nee_proc_percentiles_f)
_log.warning(msg)
continue
else:
msg = "Invalid USTAR type '{u}'".format(u=ustar_type)
raise ONEFluxError(msg)
_log.info("Will now load nee percentiles file '{f}'".format(f=nee_proc_percentiles_f))
whole_dataset_nee, headers_nee, timestamp_list_nee, year_list_nee = load_output(nee_proc_percentiles_f)
# iterate through each year
for iteration, year in enumerate(year_list_nee):
if year not in years_to_compare:
continue
_log.info("Started processing year '{y}'".format(y=year))
qc_auto_nee_f = os.path.join(qc_auto_dir, '{s}_qca_nee_{y}.csv'.format(s=siteid, y=year))
if not os.path.isfile(qc_auto_nee_f):
msg = "QC auto file not found '{f}'".format(f=qc_auto_nee_f)
_log.error(msg)
continue
latitude = get_latitude(filename=qc_auto_nee_f)
# iterate through UStar threshold values
for percentile in percentiles_data_columns:
_log.info("Started processing percentile '{p}'".format(p=percentile))
percentile_print = percentile.replace(HEADER_SEPARATOR, '.')
output_filename = os.path.join(nt_output_dir, "nee_{t}_{p}_{s}_{y}{extra}.csv".format(t=ustar_type, p=percentile_print, s=siteid, y=year, extra=EXTRA_FILENAME))
temp_output_filename = os.path.join(nt_output_dir, "nee_{t}_{p}_{s}_{y}{extra}.csv".format(t=ustar_type, p=percentile_print, s=siteid, y=year, extra='{extra}'))
if os.path.isfile(output_filename):
_log.info("Output file found, skipping: '{f}'".format(f=output_filename))
continue
else:
_log.debug("Output file missing, will be processed: '{f}'".format(f=output_filename))
# create masks for current year for both nee and meteo
year_mask_nee = (whole_dataset_nee['year'] == year)
year_mask_meteo = (whole_dataset_meteo['year'] == year)
# account for first entry being from previous year
if iteration == 0:
_log.debug("First site-year available ({y}), removing first midnight entry from meteo only".format(y=year))
first_meteo = numpy.where(year_mask_meteo == 1)[0][0]
first_nee = None
year_mask_meteo[first_meteo] = 0
else:
_log.debug("Regular site-year ({y}), removing first midnight entry from meteo and nee".format(y=year))
first_meteo = numpy.where(year_mask_meteo == 1)[0][0]
first_nee = numpy.where(year_mask_nee == 1)[0][0]
year_mask_meteo[first_meteo] = 0
year_mask_nee[first_nee] = 0
# account for last entry being from next year
_log.debug("Site-year ({y}), adding first midnight entry from next year for meteo and nee".format(y=year))
last_meteo = numpy.where(year_mask_meteo == 1)[0][-1] + 1
last_nee = numpy.where(year_mask_nee == 1)[0][-1] + 1
year_mask_meteo[last_meteo] = 1
year_mask_nee[last_nee] = 1
_log.debug("Site-year {y}: first NEE '{tn}' and first meteo '{tm}'".format(y=year, tn=whole_dataset_nee[year_mask_nee][0]['timestamp_end'], tm=whole_dataset_meteo[year_mask_meteo][0]['timestamp_end']))
_log.debug("Site-year {y}: last NEE '{tn}' and last meteo '{tm}'".format(y=year, tn=whole_dataset_nee[year_mask_nee][-1]['timestamp_end'], tm=whole_dataset_meteo[year_mask_meteo][-1]['timestamp_end']))
if numpy.sum(year_mask_nee) != numpy.sum(year_mask_meteo):
msg = "Incompatible array sizes (nee={n}, meteo={m}) for year '{y}' while processing '{f}'".format(y=year, f=output_filename, n=numpy.sum(year_mask_nee), m=numpy.sum(year_mask_meteo))
_log.error(msg)
raise ONEFluxError(msg)
working_year_data = create_data_structures(ustar_type=ustar_type, whole_dataset_nee=whole_dataset_nee, whole_dataset_meteo=whole_dataset_meteo,
percentile=percentile, year_mask_nee=year_mask_nee, year_mask_meteo=year_mask_meteo, latitude=latitude, part_type=NT_STR)
# corresponds to partitnioning_nt.pro, line: compu, set, "QCNEE=0" # NOTE: removes all information of missing data records!
compu(data=working_year_data, func=compu_qcnee_filter, columns=['qcnee']) # equivalent to: working_year_data['qcnee'][:] = 0
# get latitude from data structure
lat = var(working_year_data, 'lat')
# call flux_partition
result_year_data = flux_partition(data=working_year_data, lat=lat[0], tempvar='tair', temp_output_filename=temp_output_filename)
# save output data file
_log.debug("Saving output file '{f}".format(f=output_filename))
numpy.savetxt(fname=output_filename, X=result_year_data, delimiter=',', fmt='%s', header=','.join(result_year_data.dtype.names), comments='')
_log.debug("Saved output file '{f}".format(f=output_filename))
_log.info("Finished processing percentile '{p}'".format(p=percentile))
# sys.exit('EXIT') # TODO: testing only, remove
_log.info("Finished processing year '{y}'".format(y=year))
_log.info("Finished processing UStar threshold type '{u}'".format(u=ustar_type))
_log.info("Finished NT partitioning of {s}".format(s=siteid))
'doright4',
'doright5',
]
### parse site information
SITES_D = {}
SITES_FOLDERS_D = {}
for e in SITES_TIERS_FOLDERS:
db, siteid, fy, ly, tier, version, sitedir = e.strip().split()
if tier.strip().upper() not in ['TIER1', 'TIER2']:
msg = "Unknown TIER '{t}' for site {s} ({f}-{l})".format(t=tier,
s=siteid,
f=fy,
l=ly)
log.error(msg)
raise ONEFluxError(msg)
# sites-tiers
if siteid in SITES_D:
SITES_D[siteid][tier].append((fy, ly))
SITES_D[siteid]['FY'] = min(SITES_D[siteid]['FY'], fy)
SITES_D[siteid]['LY'] = max(SITES_D[siteid]['LY'], ly)
if SITES_D[siteid]['version'] != version:
log.error(
"Versions differ for different tiers for site {s} ({v1} <> {v2})"
.format(s=siteid, v1=SITES_D[siteid]['version'], v2=version))
else:
SITES_D[siteid] = {}
SITES_D[siteid]['TIER1'] = []
SITES_D[siteid]['TIER2'] = []
SITES_D[siteid]['version'] = version
msg += ", site-dir ({d})".format(d=args["sitedir"])
msg += ", first-year ({y})".format(y=firstyear)
msg += ", last-year ({y})".format(y=lastyear)
msg += ", perc ({i})".format(i=perc)
msg += ", prod ({i})".format(i=prod)
msg += ", log-file ({f})".format(f=args["logfile"])
msg += ", force-py ({i})".format(i=args["forcepy"])
log.debug(msg)
# start execution
try:
# check arguments
# PRI 2020/10/23 - changed use of args to dictionary syntax
print os.path.join(args["datadir"], args["sitedir"])
if not os.path.isdir(os.path.join(args["datadir"], args["sitedir"])):
raise ONEFluxError(
"Site dir not found: {d}".format(d=args["sitedir"]))
# run command
# PRI 2020/10/23 - changed use of args to dictionary syntax
log.info("Starting execution: {c}".format(c=args["command"]))
if args["command"] == 'all':
# PRI 2020/10/22
# dictionary of logicals to control which pipeline steps will be executed
pipeline_steps = {
"qc_auto_execute": True,
"ustar_mp_execute": True,
"ustar_cp_execute": False,
"meteo_proc_execute": True,
"nee_proc_execute": True,
"energy_proc_execute": True,
"nee_partition_nt_execute": True,
def generate_nee(siteid, sitedir, first_year, last_year, version_data, version_processing, pipeline=None):
log.debug("{s}: starting generation of AUXNEE file".format(s=siteid))
nee_info_template = (NEE_INFO if pipeline is None else pipeline.nee_info)
unc_info_template = (UNC_INFO if pipeline is None else pipeline.unc_info)
unc_info_alt_template = (UNC_INFO_ALT if pipeline is None else pipeline.unc_info_alt)
prodfile_aux_template = (PRODFILE_AUX_TEMPLATE if pipeline is None else pipeline.prodfile_aux_template)
mpdir_template = (MPDIR if pipeline is None else pipeline.mpdir)
cpdir_template = (CPDIR if pipeline is None else pipeline.cpdir)
nee_perc_ustar_cut_template = (NEE_PERC_USTAR_CUT if pipeline is None else pipeline.nee_perc_ustar_cut)
nee_perc_ustar_vut_template = (NEE_PERC_USTAR_VUT if pipeline is None else pipeline.nee_perc_ustar_vut)
### TEST year range for real range. Some sites have only energy/water fluxes for some of the first or last years, so NEE has shorter range
nee_perc_ustar_cut = nee_perc_ustar_cut_template.format(s=siteid, sd=sitedir, fy=first_year, ly=last_year)
if not os.path.isfile(nee_perc_ustar_cut):
log.warning("{s}: looking for alternate USTAR CUT percentiles file, NOT found: {f}".format(s=siteid, f=nee_perc_ustar_cut))
alt_nee_perc_ustar_cut = test_pattern(tdir=os.path.join(sitedir, NEEDIR_PATTERN), tpattern=NEE_PERC_USTAR_CUT_PATTERN.format(s=siteid, sd=sitedir, fy='????', ly='????'), label='aux_info_files', log_only=True)
if len(alt_nee_perc_ustar_cut) == 1:
log.warning("{s}: alternate USTAR CUT percentiles file FOUND: {f}".format(s=siteid, f=alt_nee_perc_ustar_cut))
alt_nee_perc_ustar_cut = alt_nee_perc_ustar_cut[0]
_, first_year, last_year, _ = alt_nee_perc_ustar_cut.split('_', 3)
first_year, last_year = int(first_year), int(last_year)
else:
log.warning("{s}: incorrect number of NEE_PERC_USTAR_CUT files found (1-2 years record?): {l}".format(s=siteid, l=alt_nee_perc_ustar_cut))
# # PREVIOU SINGLE YEAR IMPLEMENTATION
# alt_nee_perc_ustar_cut = NEE_PERC_USTAR_CUT.format(s=siteid, sd=sitedir, fy=int(first_year) + 1, ly=last_year)
# if os.path.isfile(alt_nee_perc_ustar_cut):
# log.warning("{s}: alternate USTAR CUT percentiles file FOUND: {f}".format(s=siteid, f=alt_nee_perc_ustar_cut))
# first_year += 1
# else:
# log.error("{s}: alternate USTAR CUT percentiles file NOT found: {f}".format(s=siteid, f=alt_nee_perc_ustar_cut))
year_range = range(int(first_year), int(last_year) + 1)
### process NEE USTAR CUT
nee_perc_ustar_cut_values = load_ustar_cut(siteid=siteid, sitedir=sitedir, first_year=first_year, last_year=last_year, nee_perc_ustar_cut_template=nee_perc_ustar_cut_template)
### process NEE USTAR VUT
nee_perc_ustar_vut_values = load_ustar_vut(siteid=siteid, sitedir=sitedir, year_range=year_range, nee_perc_ustar_vut_template=nee_perc_ustar_vut_template)
# process NEE, RECO, and GPP info
u50_ustar_perc = dict({'CUT':nee_perc_ustar_cut_values['50.00']}.items() + {year:threshold['50.00'] for year, threshold in nee_perc_ustar_vut_values.items()}.items())
nee_ref_ustar_perc = {i:{} for i in RESOLUTION_LIST}
unc_ref_ustar_perc = {i:{j:{} for j in RESOLUTION_LIST} for i in ['RECO_NT', 'GPP_NT', 'RECO_DT', 'GPP_DT']}
ustar_not_working = {'files_mp':set(), 'files_cp':set(), 'info_mp':set(), 'info_cp':set()}
lines = []
for res in RESOLUTION_LIST:
# process NEE
nee_info = nee_info_template.format(s=siteid, sd=sitedir, r=res)
method_line_num = None
if not os.path.isfile(nee_info):
raise ONEFluxError("NEE info file not found: {f}".format(f=nee_info))
with open(nee_info, 'r') as f:
lines = f.readlines()
for line_num, line in enumerate(reversed(lines)):
# NEE REF VUT
if line.strip().lower().startswith('nee_ref_y'):
year_extra = None
unsplit_year = line.strip().lower().split('on year')
if len(unsplit_year) == 2:
year = int(unsplit_year[1].strip().split()[0].strip())
elif len(unsplit_year) == 1 and len(year_range) == 1:
year = first_year
elif len(unsplit_year) == 1 and len(year_range) == 2:
year = first_year
year_extra = last_year
else:
raise ONEFluxError("{s}: Unknown NEE VUT REF percentile/threshold entry in line: '{l}'".format(s=siteid, l=line.strip()))
threshold = line.strip().lower().split('ustar percentile')[1].strip().split()[0].strip()
ustar = nee_perc_ustar_vut_values[year][threshold]
if year in nee_ref_ustar_perc[res]:
raise ONEFluxError("{s} duplicated entry for NEE REF VUT USTAR: {f}".format(s=siteid, f=nee_info))
else:
nee_ref_ustar_perc[res][year] = (threshold, ustar)
if year_extra:
if year_extra in nee_ref_ustar_perc[res]:
raise ONEFluxError("{s} duplicated entry for NEE REF VUT USTAR: {f}".format(s=siteid, f=nee_info))
else:
nee_ref_ustar_perc[res][year_extra] = (threshold, ustar)
year_extra = None
# print 'VUT', res, year, threshold, ustar
# NEE REF CUT
elif line.strip().lower().startswith('nee_ref_c'):
threshold = line.strip().lower().split('ustar percentile')[1].strip().split()[0].strip()
ustar = nee_perc_ustar_cut_values[threshold]
if 'CUT' in nee_ref_ustar_perc[res]:
raise ONEFluxError("{s} duplicated entry for NEE REF CUT USTAR: {f}".format(s=siteid, f=nee_info))
else:
nee_ref_ustar_perc[res]['CUT'] = (threshold, ustar)
# print 'CUT', res, threshold, ustar
# USTAR-method-not-working entries start
elif ('year' in line.strip().lower()) and ('method not applied' in line.strip().lower()):
if method_line_num is not None:
raise ONEFluxError('Two lines (#{l1} and #{l2}) starting with info for USTAR method not working: {f}'.format(l1=method_line_num, l2=len(lines) - line_num - 1, f=nee_info))
method_line_num = len(lines) - line_num - 1
# USTAR-method-not-working entries detect
if method_line_num:
lnum = method_line_num
while lnum < len(lines):
if is_int(lines[lnum].strip()[:4]):
if lines[lnum].strip().lower().endswith('mp'):
ustar_not_working['info_mp'].add(int(lines[lnum].strip()[:4]))
# print lines[lnum].strip()
elif lines[lnum].strip().lower().endswith('cp'):
ustar_not_working['info_cp'].add(int(lines[lnum].strip()[:4]))
# print lines[lnum].strip()
lnum += 1
else:
log.warning('USTAR-method-not-working entries not found at {r} for: {f}'.format(r=res.upper(), f=nee_info))
# USTAR-method-not-working files detect
mpyears, cpyears = get_created_ustar_years(mpdir=mpdir_template.format(sd=sitedir), cpdir=cpdir_template.format(sd=sitedir))
ustar_not_working['files_mp'] = set(year_range) - set(mpyears)
ustar_not_working['files_cp'] = set(year_range) - set(cpyears)
# print 'MP: ', mpyears
# print 'CP: ', cpyears
# print
# process RECO, GPP
for method, variable in [('NT', 'RECO'), ('NT', 'GPP'), ('DT', 'RECO'), ('DT', 'GPP')]:
key = variable + '_' + method
unc_info = unc_info_template.format(s=siteid, sd=sitedir, m=method, v=variable, r=res)
if not os.path.isfile(unc_info):
unc_info = unc_info_alt_template.format(s=siteid, sd=sitedir, m=method, v=variable, r=res)
if not os.path.isfile(unc_info):
raise ONEFluxError("UNC info file not found: {f}".format(f=unc_info))
log.debug("{s}: processing file: {f}".format(s=siteid, f=unc_info))
with open(unc_info, 'r') as f:
lines = f.readlines()
for line in reversed(lines):
# RECO/GPP REF VUT
if line.strip().lower().startswith('{v}_ref_y'.format(v=variable).lower()):
year_extra = None
unsplit_year = line.strip().lower().split('on year')
if len(unsplit_year) == 2:
year = int(unsplit_year[1].strip().split()[0].strip())
elif len(unsplit_year) == 1 and len(year_range) == 1:
year = first_year
elif len(unsplit_year) == 1 and len(year_range) == 2:
year = first_year
year_extra = last_year
else:
raise ONEFluxError("{s}: Unknown RECO/GPP VUT REF percentile/threshold entry in line: '{l}'".format(s=siteid, l=line.strip()))
threshold = line.strip().lower().split('ustar percentile')[1].strip().split()[0].strip()
ustar = nee_perc_ustar_vut_values[year][threshold]
if year in unc_ref_ustar_perc[key][res]:
raise ONEFluxError("{s} duplicated entry for {v} REF VUT USTAR: {f}".format(s=siteid, f=nee_info, v=variable))
else:
unc_ref_ustar_perc[key][res][year] = (threshold, ustar)
if year_extra:
if year_extra in unc_ref_ustar_perc[key][res]:
raise ONEFluxError("{s} duplicated entry for RECO/GPP REF VUT USTAR: {f}".format(s=siteid, f=nee_info))
else:
unc_ref_ustar_perc[key][res][year_extra] = (threshold, ustar)
year_extra = None
# print variable, method, 'VUT', res, year, threshold, ustar
# RECO/GPP REF CUT
elif line.strip().lower().startswith('{v}_ref_c'.format(v=variable).lower()):
threshold = line.strip().lower().split('ustar percentile')[1].strip().split()[0].strip()
ustar = nee_perc_ustar_cut_values[threshold]
if 'CUT' in unc_ref_ustar_perc[key][res]:
raise ONEFluxError("{s} duplicated entry for {v} REF CUT USTAR: {f}".format(s=siteid, f=nee_info, v=variable))
else:
unc_ref_ustar_perc[key][res]['CUT'] = (threshold, ustar)
# print variable, method, 'CUT', res, threshold, ustar
output_lines = [','.join(AUX_HEADER) + '\n']
# output USTAR not working
entry_number = 1
for year in year_range:
if year in ustar_not_working['files_mp']:
nline = "{i},USTAR_MP_METHOD,SUCCESS_RUN,0,{y}\n".format(i=entry_number, y=year)
if year not in ustar_not_working['info_mp']:
log.warning("{s}: USTAR_MP, year {y} not found in files, but success in info".format(s=siteid, y=year))
else:
nline = "{i},USTAR_MP_METHOD,SUCCESS_RUN,1,{y}\n".format(i=entry_number, y=year)
if year in ustar_not_working['info_mp']:
log.error("{s}: USTAR_MP, year {y} failed in info, but found in files".format(s=siteid, y=year))
output_lines.append(nline)
entry_number += 1
for year in year_range:
if year in ustar_not_working['files_cp']:
nline = "{i},USTAR_CP_METHOD,SUCCESS_RUN,0,{y}\n".format(i=entry_number, y=year)
if year not in ustar_not_working['info_cp']:
log.warning("{s}: USTAR_CP, year {y} not found in files, but success in info".format(s=siteid, y=year))
else:
nline = "{i},USTAR_CP_METHOD,SUCCESS_RUN,1,{y}\n".format(i=entry_number, y=year)
if year in ustar_not_working['info_cp']:
log.error("{s}: USTAR_CP, year {y} failed in info, but found in files".format(s=siteid, y=year))
output_lines.append(nline)
# output USTAR_THRESHOLD_50
entry_number += 1
nline = "{i},NEE_CUT_USTAR50,USTAR_THRESHOLD,{v},-9999\n".format(i=entry_number, v=u50_ustar_perc.get('CUT', -9999))
output_lines.append(nline)
entry_number += 1
for year in year_range:
nline = "{i},NEE_VUT_USTAR50,USTAR_THRESHOLD,{v},{y}\n".format(i=entry_number, v=u50_ustar_perc.get(year, -9999), y=year)
output_lines.append(nline)
# output NEE REF
for res in RESOLUTION_LIST:
entry_number += 1
perc, thres = nee_ref_ustar_perc[res].get('CUT', (-9999, -9999))
nline = "{i},NEE_CUT_REF,{r}_USTAR_PERCENTILE,{v},-9999\n".format(i=entry_number, r=res.upper(), v=perc)
output_lines.append(nline)
nline = "{i},NEE_CUT_REF,{r}_USTAR_THRESHOLD,{v},-9999\n".format(i=entry_number, r=res.upper(), v=thres)
output_lines.append(nline)
entry_number += 1
for year in year_range:
perc, thres = nee_ref_ustar_perc[res].get(year, (-9999, -9999))
nline = "{i},NEE_VUT_REF,{r}_USTAR_PERCENTILE,{v},{y}\n".format(i=entry_number, r=res.upper(), v=perc, y=year)
output_lines.append(nline)
nline = "{i},NEE_VUT_REF,{r}_USTAR_THRESHOLD,{v},{y}\n".format(i=entry_number, r=res.upper(), v=thres, y=year)
output_lines.append(nline)
# output UNC REF
for prod in ['RECO_NT', 'GPP_NT', 'RECO_DT', 'GPP_DT']:
for res in RESOLUTION_LIST:
entry_number += 1
perc, thres = unc_ref_ustar_perc[prod][res].get('CUT', (-9999, -9999))
nline = "{i},{p}_CUT_REF,{r}_USTAR_PERCENTILE,{v},-9999\n".format(i=entry_number, p=prod, r=res.upper(), v=perc)
output_lines.append(nline)
nline = "{i},{p}_CUT_REF,{r}_USTAR_THRESHOLD,{v},-9999\n".format(i=entry_number, p=prod, r=res.upper(), v=thres)
output_lines.append(nline)
entry_number += 1
for year in year_range:
perc, thres = unc_ref_ustar_perc[prod][res].get(year, (-9999, -9999))
nline = "{i},{p}_VUT_REF,{r}_USTAR_PERCENTILE,{v},{y}\n".format(i=entry_number, p=prod, r=res.upper(), v=perc, y=year)
output_lines.append(nline)
nline = "{i},{p}_VUT_REF,{r}_USTAR_THRESHOLD,{v},{y}\n".format(i=entry_number, p=prod, r=res.upper(), v=thres, y=year)
output_lines.append(nline)
# output CUT thresholds
for perc in PERCENTILES_SORTED:
entry_number += 1
thres = nee_perc_ustar_cut_values[perc]
nline = "{i},USTAR_CUT,USTAR_PERCENTILE,{v},-9999\n".format(i=entry_number, v=perc)
output_lines.append(nline)
nline = "{i},USTAR_CUT,USTAR_THRESHOLD,{v},-9999\n".format(i=entry_number, v=thres)
output_lines.append(nline)
# output VUT thresholds
for year in year_range:
for perc in PERCENTILES_SORTED:
entry_number += 1
thres = nee_perc_ustar_vut_values[year][perc]
nline = "{i},USTAR_VUT,USTAR_PERCENTILE,{v},{y}\n".format(i=entry_number, v=perc, y=year)
output_lines.append(nline)
nline = "{i},USTAR_VUT,USTAR_THRESHOLD,{v},{y}\n".format(i=entry_number, v=thres, y=year)
output_lines.append(nline)
output_filename = prodfile_aux_template.format(s=siteid, sd=sitedir, aux='AUXNEE', fy=first_year, ly=last_year, vd=version_data, vp=version_processing)
log.info("{s}: writing auxiliary NEE file: {f}".format(s=siteid, f=output_filename))
with open(output_filename, 'w') as f:
f.writelines(output_lines)
log.debug("{s}: finished generation of AUXNEE file: {f}".format(s=siteid, f=output_filename))
return output_filename
def generate_meteo(siteid, sitedir, first_year, last_year, version_data, version_processing, pipeline=None):
log.debug("{s}: starting generation of AUXMETEO file".format(s=siteid))
meteo_info = (METEO_INFO if pipeline is None else pipeline.meteo_info)
prodfile_aux_template = (PRODFILE_AUX_TEMPLATE if pipeline is None else pipeline.prodfile_aux_template)
filename = meteo_info.format(s=siteid, sd=sitedir, r='hh')
if not os.path.isfile(filename):
raise ONEFluxError("{s}: meteo info file not found: {f}".format(s=siteid, f=filename))
H_BEGIN, H_END = "var", "corr"
VAR_D = {
'Ta': 'TA',
'TA': 'TA',
'Pa': 'PA',
'PA': 'PA',
'VPD': 'VPD',
'WS': 'WS',
'Precip': 'P',
'P': 'P',
'Rg': 'SW_IN',
'SW_IN': 'SW_IN',
'LWin': 'LW_IN',
'LW_IN': 'LW_IN',
'LWin_calc': 'LW_IN_JSB',
'LW_IN_calc': 'LW_IN_JSB',
}
lines = []
with open(filename, 'r') as f:
lines = f.readlines()
c_var, c_slope, c_intercept, c_rmse, c_corr = 0, 1, 2, 3, 4
first_line = None
for i, line in enumerate(lines):
l = line.strip().lower()
if l.startswith(H_BEGIN) and l.endswith(H_END):
first_line = i
break
if first_line is None:
raise ONEFluxError("{s}: first line of meteo info file not found: {f}".format(s=siteid, f=filename))
if 'unit' in lines[first_line].lower():
log.info("{s}: handling old format meteo info file: {f}".format(s=siteid, f=filename))
c_slope, c_intercept, c_rmse, c_corr = 3, 4, 5, 6
vars_l = ['TA', 'PA', 'VPD', 'WS', 'P', 'SW_IN', 'LW_IN', 'LW_IN_JSB']
#pars_l = ['ERA_SLOPE', 'ERA_INTERCEPT', 'ERA_RMSE', 'ERA_CORRELATION']
values = {i:None for i in vars_l}
for line in lines[first_line + 1:first_line + 9]:
l = line.strip().split(',')
values[VAR_D[l[c_var]]] = [(float(l[c_slope].strip()) if (l[c_slope].strip() and l[c_slope].strip() != '-') else -9999),
(float(l[c_intercept].strip()) if (l[c_intercept].strip() and l[c_intercept].strip() != '-') else -9999),
(float(l[c_rmse].strip()) if (l[c_rmse].strip() and l[c_rmse].strip() != '-') else -9999),
(float(l[c_corr].strip()) if (l[c_corr].strip() and l[c_corr].strip() != '-') else -9999),
]
output_lines = [','.join(AUX_HEADER) + '\n']
for i, var in enumerate(vars_l, start=1):
if values[var] is None:
raise ONEFluxError("{s}: ERA variable '{v}' not found in: {f}".format(s=siteid, v=var, f=filename))
slope = ("{v:.2f}".format(v=values[var][0]) if values[var][0] != -9999 else '-9999')
intercept = ("{v:.2f}".format(v=values[var][1]) if values[var][1] != -9999 else '-9999')
rmse = ("{v:.2f}".format(v=values[var][2]) if values[var][2] != -9999 else '-9999')
corr = ("{v:.2f}".format(v=values[var][3]) if values[var][3] != -9999 else '-9999')
output_lines.append("{i},{v},{p},{val},{t}\n".format(i=i, v=var, p='ERA_SLOPE', val=slope, t='-9999'))
output_lines.append("{i},{v},{p},{val},{t}\n".format(i=i, v=var, p='ERA_INTERCEPT', val=intercept, t='-9999'))
output_lines.append("{i},{v},{p},{val},{t}\n".format(i=i, v=var, p='ERA_RMSE', val=rmse, t='-9999'))
output_lines.append("{i},{v},{p},{val},{t}\n".format(i=i, v=var, p='ERA_CORRELATION', val=corr, t='-9999'))
output_filename = prodfile_aux_template.format(s=siteid, sd=sitedir, aux='AUXMETEO', fy=first_year, ly=last_year, vd=version_data, vp=version_processing)
log.info("{s}: writing auxiliary METEO file: {f}".format(s=siteid, f=output_filename))
with open(output_filename, 'w') as f:
f.writelines(output_lines)
log.debug("{s}: finished generation of AUXMETEO file: {f}".format(s=siteid, f=output_filename))
return output_filename
def nlinlts1(data, func=lloyd_taylor, depvar='neenight', indepvar='tair', npara=2, xguess=[2.0, 200.0], trim_perc=BR_PERC):
"""
Main non-linear least-squares driver function
:param data: data structure for partitioning
:type data: numpy.ndarray
:param func: function to be optimized
:type func: function
:param depvar: dependent variable (computed by function)
:type depvar: str
:param indepvar: independent variable (parameter to function)
:type indepvar: str
:param npara: number of parameters to be optimized
:type npara: int
:param xguess: list with initial/starting guesses for variables to be optimized
:type xguess: list
:param trim_perc: precentage to trim from residual values
:type trim_perc: float
"""
if len(xguess) != npara:
msg = "Incompatible number of parameters '{n}' and length of initial guess '{i}'".format(n=npara, i=len(xguess))
_log.critical(msg)
raise ONEFluxError(msg)
status = 0 # status of execution; 0 optimization executed successfully, -1 problem with execution of optimization
first_ts, last_ts = get_first_last_ts(data=data)
# _log.debug("Starting optimization step for period '{ts1}' - '{ts2}'".format(ts1=first_ts.strftime('%Y-%m-%d %H:%M'), ts2=last_ts.strftime('%Y-%m-%d %H:%M')))
# check number of entries for independent variable
nonnan_indep_mask = not_nan(data[indepvar])
if numpy.sum(nonnan_indep_mask) < (npara * 3):
_log.warning("Not enough data points (independent variable filtered) for optimization: {n}".format(n=numpy.sum(nonnan_indep_mask)))
status = -1
return status, -9999.0, -9999.0, -9999.0, -9999.0, None, None, None, None, None, None, None
# check number of entries for dependent AND independent variable
nonnan_dep_mask = not_nan(data[depvar])
nonnan_combined_mask = nonnan_indep_mask & nonnan_dep_mask
if numpy.sum(nonnan_combined_mask) < (npara * 3):
_log.warning("Not enough data points (dependent and independent variable filtered) for optimization: {n}".format(n=numpy.sum(nonnan_combined_mask)))
status = -1
return status, -9999.0, -9999.0, -9999.0, -9999.0, None, None, None, None, None, None, None
# "clean" dependent variable so not to use NAs from independent variable
clean_dep = data[depvar].copy()
clean_dep[~nonnan_indep_mask] = NAN
# print
# print 'NEE:'
# print clean_dep
# print
# print 'TA:'
# print data[indepvar]
# print
# # TODO: remove
# define inner function to be used for optimization
def trimmed_residuals(par, nee=clean_dep, temp=data[indepvar], trim_perc=trim_perc):
"""
(inner) Function to be evaluated at each iteration,
taking care of handling NAs and trimming residuals.
Inner function used so extra arguments not needed in
call to scipy leastsq function
:param nee: array with (non-cleaned) nee values (dependent variable)
:type nee: numpy.ndarray
:param temp: array with (cleaned, no NAs) temperature (independent variable)
:type temp: numpy.ndarray
:param rref: reference respiration (1st parameter to be optimized)
:type rref: float
:param e0: temperature sensitivity (2nd parameter to be optimized)
:type e0: float
:param trim_perc: percentiled to be trimmed off
:type trim_perc: float
"""
rref, e0 = par
prediction = lloyd_taylor(ta=temp, rref=rref, e0=e0)
residuals = nee - prediction
nonnan_nee_mask = not_nan(nee)
residuals[~nonnan_nee_mask] = 0.0
# NOTE: compareIndex and compindex not used in NT partitioning code
if trim_perc == 0.0:
return residuals
absolute_residuals = numpy.abs(residuals)
pct_calc = pct(absolute_residuals, 100.0 - trim_perc)
trim_mask = (absolute_residuals > pct_calc)
residuals[trim_mask] = 0.0
# print 'rref/e0/res:', rref, e0, numpy.sum(residuals ** 2)
return residuals
parameters, std_devs, ls_status, ls_msg, residuals, covariance_matrix = least_squares(func=trimmed_residuals,
initial_guess=xguess,
entries=len(clean_dep),
iterations=1000 * (len(clean_dep) + 1),
return_residuals_cov_mat=True)
est_rref, est_e0 = parameters
est_rref_std, est_e0_std = std_devs
tvalue, pvalue = ttest_ind(clean_dep, lloyd_taylor(ta=data[indepvar], rref=est_rref, e0=est_e0))
fvalue, f_pvalue = f_oneway(clean_dep, lloyd_taylor(ta=data[indepvar], rref=est_rref, e0=est_e0))
nee_std, ta_std = numpy.nanstd(clean_dep), numpy.nanstd(data[indepvar])
# print "rref:", est_rref
# print "rref_se:", est_rref_std
# print "e0:", est_e0
# print "e0_se:", est_e0_std
# print "p-value:", pvalue, tvalue
# print "p-value (f):", f_pvalue, fvalue
# print
# # TODO: remove
# _log.debug("Finished optimization step for period '{ts1}' - '{ts2}' - parameters rref: {r} ({rs}), e0: {e} ({es})*********".format(ts1=first_ts.strftime('%Y-%m-%d %H:%M'),
# ts2=last_ts.strftime('%Y-%m-%d %H:%M'),
# r=est_rref, e=est_e0,
# rs=est_rref_std, es=est_e0_std))
# NOTE: calculation of residuals in the original code used only for graph, so not added here
# add zeros to residuals, if lenght of residuals less than maximum number of entries for window
# done to match original code, purpose unclear
if len(residuals) < 48 * WINDOW_SIZE:
new_residuals = numpy.zeros(48 * WINDOW_SIZE, dtype=FLOAT_PREC)
new_residuals[:len(residuals)] = residuals
else:
new_residuals = residuals
# NOTE: if changing return values, also update case for "not enough data points"
return status, est_rref, est_e0, est_rref_std, est_e0_std, new_residuals, covariance_matrix, ls_status, ls_msg, pvalue, nee_std, ta_std