def l1qc_process(cf, ds1):
# get the netCDF attributes from the control file
qcts.do_attributes(cf, ds1)
# round the Python datetime to the nearest second
qcutils.round_datetime(ds1, mode="nearest_second")
#check for gaps in the Python datetime series and fix if present
fixtimestepmethod = qcutils.get_keyvaluefromcf(cf, ["options"],
"FixTimeStepMethod",
default="round")
if qcutils.CheckTimeStep(ds1):
qcutils.FixTimeStep(ds1, fixtimestepmethod=fixtimestepmethod)
# recalculate the Excel datetime
qcutils.get_xldatefromdatetime(ds1)
# get the Year, Month, Day etc from the Python datetime
qcutils.get_ymdhmsfromdatetime(ds1)
# write the processing level to a global attribute
ds1.globalattributes['nc_level'] = str("L1")
# get the start and end date from the datetime series unless they were
# given in the control file
if 'start_date' not in ds1.globalattributes.keys():
ds1.globalattributes['start_date'] = str(
ds1.series['DateTime']['Data'][0])
if 'end_date' not in ds1.globalattributes.keys():
ds1.globalattributes['end_date'] = str(
ds1.series['DateTime']['Data'][-1])
# calculate variances from standard deviations and vice versa
qcts.CalculateStandardDeviations(cf, ds1)
# create new variables using user defined functions
qcts.DoFunctions(cf, ds1)
# create a series of synthetic downwelling shortwave radiation
qcts.get_synthetic_fsd(ds1)
def xl2nc(cf,InLevel):
# get the data series from the Excel file
ds = xl_read_series(cf)
if len(ds.series.keys())==0: return 1
# get the netCDF attributes from the control file
qcts.do_attributes(cf,ds)
# get a series of Python datetime objects from the Excel datetime
qcutils.get_datetimefromxldate(ds)
#check for gaps in the Excel datetime series
has_gaps = qcutils.CheckTimeStep(ds,fix='gaps')
# write the processing level to a global attribute
ds.globalattributes['nc_level'] = str(InLevel)
# get the start and end date from the datetime series unless they were
# given in the control file
if 'start_date' not in ds.globalattributes.keys():
ds.globalattributes['start_date'] = str(ds.series['DateTime']['Data'][0])
if 'end_date' not in ds.globalattributes.keys():
ds.globalattributes['end_date'] = str(ds.series['DateTime']['Data'][-1])
# get the year, month, day, hour, minute and second from the xl date/time
qcutils.get_ymdhmsfromxldate(ds)
# do any functions to create new series
qcts.do_functions(cf,ds)
# write the data to the netCDF file
outfilename = get_outfilename_from_cf(cf)
ncFile = nc_open_write(outfilename)
nc_write_series(ncFile,ds)
return 0
def l1qc(cf):
# get the data series from the Excel file
in_filename = qcio.get_infilenamefromcf(cf)
if not qcutils.file_exists(in_filename, mode="quiet"):
msg = " Input file " + in_filename + " not found ..."
logger.error(msg)
ds1 = qcio.DataStructure()
ds1.returncodes = {"value": 1, "message": msg}
return ds1
file_name, file_extension = os.path.splitext(in_filename)
if "csv" in file_extension.lower():
ds1 = qcio.csv_read_series(cf)
if ds1.returncodes["value"] != 0:
return ds1
# get a series of Excel datetime from the Python datetime objects
qcutils.get_xldatefromdatetime(ds1)
else:
ds1 = qcio.xl_read_series(cf)
if ds1.returncodes["value"] != 0:
return ds1
# get a series of Python datetime objects from the Excel datetime
qcutils.get_datetimefromxldate(ds1)
# get the netCDF attributes from the control file
qcts.do_attributes(cf, ds1)
# round the Python datetime to the nearest second
qcutils.round_datetime(ds1, mode="nearest_second")
#check for gaps in the Python datetime series and fix if present
fixtimestepmethod = qcutils.get_keyvaluefromcf(cf, ["options"],
"FixTimeStepMethod",
default="round")
if qcutils.CheckTimeStep(ds1):
qcutils.FixTimeStep(ds1, fixtimestepmethod=fixtimestepmethod)
# recalculate the Excel datetime
qcutils.get_xldatefromdatetime(ds1)
# get the Year, Month, Day etc from the Python datetime
qcutils.get_ymdhmsfromdatetime(ds1)
# write the processing level to a global attribute
ds1.globalattributes['nc_level'] = str("L1")
# get the start and end date from the datetime series unless they were
# given in the control file
if 'start_date' not in ds1.globalattributes.keys():
ds1.globalattributes['start_date'] = str(
ds1.series['DateTime']['Data'][0])
if 'end_date' not in ds1.globalattributes.keys():
ds1.globalattributes['end_date'] = str(
ds1.series['DateTime']['Data'][-1])
# calculate variances from standard deviations and vice versa
qcts.CalculateStandardDeviations(cf, ds1)
# create new variables using user defined functions
qcts.DoFunctions(cf, ds1)
# create a series of synthetic downwelling shortwave radiation
qcts.get_synthetic_fsd(ds1)
# check missing data and QC flags are consistent
qcutils.CheckQCFlags(ds1)
return ds1
logging.info('Creating the data structures')
ds_60minutes = qcio.DataStructure()
# get a sorted list of files that match the mask in the control file
file_list = sorted(glob.glob(info["in_filename"]))
# read the netcdf files
logging.info('Reading the netCDF files for '+info["site_name"])
f = access_read_mfiles2(file_list,var_list=var_list)
# get the data from the netCDF files and write it to the 60 minute data structure
logging.info('Getting the ACCESS data')
get_accessdata(cf,ds_60minutes,f,info)
# set some global attributes
logging.info('Setting global attributes')
set_globalattributes(ds_60minutes,info)
# check for time gaps in the file
logging.info("Checking for time gaps")
if qcutils.CheckTimeStep(ds_60minutes):
qcutils.FixTimeStep(ds_60minutes)
# get the datetime in some different formats
logging.info('Getting xlDateTime and YMDHMS')
qcutils.get_xldatefromdatetime(ds_60minutes)
qcutils.get_ymdhmsfromdatetime(ds_60minutes)
#f.close()
# get derived quantities and adjust units
logging.info("Changing units and getting derived quantities")
# air temperature from K to C
changeunits_airtemperature(ds_60minutes)
# soil temperature from K to C
changeunits_soiltemperature(ds_60minutes)
# pressure from Pa to kPa
changeunits_pressure(ds_60minutes)
# wind speed from components
import numpy
import os
import sys
# check the scripts directory is present
if not os.path.exists("../scripts/"):
print("erai2nc: the scripts directory is missing")
sys.exit()
# since the scripts directory is there, try importing the modules
sys.path.append('../scripts')
import qcio
import qcutils
aws_name = qcio.get_filename_dialog(path="/mnt/OzFlux/Sites")
ds_aws_30minute = qcio.nc_read_series(aws_name)
has_gaps = qcutils.CheckTimeStep(ds_aws_30minute)
if has_gaps:
print("Problems found with time step")
qcutils.FixTimeStep(ds_aws_30minute)
qcutils.get_ymdhmsfromdatetime(ds_aws_30minute)
dt_aws_30minute = ds_aws_30minute.series["DateTime"]["Data"]
ddt = [
dt_aws_30minute[i + 1] - dt_aws_30minute[i]
for i in range(0,
len(dt_aws_30minute) - 1)
]
print("Minimum time step is", min(ddt), " Maximum time step is", max(ddt))
dt_aws_30minute = ds_aws_30minute.series["DateTime"]["Data"]
start_date = dt_aws_30minute[0]
end_date = dt_aws_30minute[-1]
qcutils.CreateSeries(ds,'Ws',data_dict[bom_id][:,10],Flag=flag,Attr=attr)
attr=qcutils.MakeAttributeDictionary(long_name='Wind direction',units='degT',
bom_id=str(bom_id),bom_name=bom_sites_info[site_name][str(bom_id)]["site_name"],
bom_dist=bom_sites_info[site_name][str(bom_id)]["distance"])
qcutils.CreateSeries(ds,'Wd',data_dict[bom_id][:,11],Flag=flag,Attr=attr)
attr=qcutils.MakeAttributeDictionary(long_name='Wind gust',units='m/s',
bom_id=str(bom_id),bom_name=bom_sites_info[site_name][str(bom_id)]["site_name"],
bom_dist=bom_sites_info[site_name][str(bom_id)]["distance"])
qcutils.CreateSeries(ds,'Wg',data_dict[bom_id][:,12],Flag=flag,Attr=attr)
data_dict[bom_id][:,13] = data_dict[bom_id][:,13]/float(10)
attr=qcutils.MakeAttributeDictionary(long_name='Air Pressure',units='kPa',
bom_id=str(bom_id),bom_name=bom_sites_info[site_name][str(bom_id)]["site_name"],
bom_dist=bom_sites_info[site_name][str(bom_id)]["distance"])
qcutils.CreateSeries(ds,'ps',data_dict[bom_id][:,13],Flag=flag,Attr=attr)
# fix any time stamp issues
if qcutils.CheckTimeStep(ds):
qcutils.FixTimeStep(ds)
# update the Year, Month, Day etc from the Python datetime
qcutils.get_ymdhmsfromdatetime(ds)
# now interpolate
for label in ["Precip","Ta","Td","RH","Ws","Wd","Wg","ps"]:
qcts.InterpolateOverMissing(ds,series=label,maxlen=2)
# put this stations data into the data structure dictionary
ds_dict[bom_id] = ds
# get the earliest start datetime and the latest end datetime
log.info("Finding the start and end dates")
bom_id_list = ds_dict.keys()
ds0 = ds_dict[bom_id_list[0]]
ldt = ds0.series["DateTime"]["Data"]
#print bom_id_list[0],":",ldt[0],ldt[-1]
# get the site time zone
site_timezone = cf["Sites"][site]["site_timezone"]
# read the BIOS file
bios_ncfile = netCDF4.Dataset(infilename)
time = bios_ncfile.variables["time"][:]
nRecs = len(time)
# set some global attributes
ts = ds_30.globalattributes["time_step"] = 30
ds_30.globalattributes["time_zone"] = site_timezone
ds_30.globalattributes["nc_nrecs"] = nRecs
ds_30.globalattributes["xl_datemode"] = str(0)
ds_30.globalattributes["site_name"] = cf["Sites"][site]["site_name"]
time_units = getattr(bios_ncfile.variables["time"],"units")
qcutils.get_datetimefromnctime(ds_30,time,time_units)
qcutils.round_datetime(ds_30,mode="nearest_timestep")
if qcutils.CheckTimeStep(ds_30): qcutils.FixTimeStep(ds_30)
ldt_30 = ds_30.series["DateTime"]["Data"]
si = qcutils.GetDateIndex(ldt_30,start_date,default=0,ts=ts,match="startnexthour")
ei = qcutils.GetDateIndex(ldt_30,end_date,default=len(ldt_30),ts=ts,match="endprevioushour")
ds_30.series["DateTime"]["Data"] = ds_30.series["DateTime"]["Data"][si:ei+1]
ds_30.series["DateTime"]["Flag"] = ds_30.series["DateTime"]["Flag"][si:ei+1]
ldt_30 = ds_30.series["DateTime"]["Data"]
nRecs = ds_30.globalattributes["nc_nrecs"] = len(ldt_30)
flag = numpy.zeros(nRecs)
qcutils.get_ymdhmsfromdatetime(ds_30)
xl_date_loc = qcutils.get_xldatefromdatetime(ds_30)
attr = qcutils.MakeAttributeDictionary(long_name="Date/time (local) in Excel format",units="days since 1899-12-31 00:00:00")
qcutils.CreateSeries(ds_30,"xlDateTime",xl_date_loc,flag,attr)
# get the data
for label in var_list:
bios_name = cf["Variables"][label]["bios_name"]
