def l1qc_process(cf, ds1): # get the netCDF attributes from the control file qcts.do_attributes(cf, ds1) # round the Python datetime to the nearest second qcutils.round_datetime(ds1, mode="nearest_second") #check for gaps in the Python datetime series and fix if present fixtimestepmethod = qcutils.get_keyvaluefromcf(cf, ["options"], "FixTimeStepMethod", default="round") if qcutils.CheckTimeStep(ds1): qcutils.FixTimeStep(ds1, fixtimestepmethod=fixtimestepmethod) # recalculate the Excel datetime qcutils.get_xldatefromdatetime(ds1) # get the Year, Month, Day etc from the Python datetime qcutils.get_ymdhmsfromdatetime(ds1) # write the processing level to a global attribute ds1.globalattributes['nc_level'] = str("L1") # get the start and end date from the datetime series unless they were # given in the control file if 'start_date' not in ds1.globalattributes.keys(): ds1.globalattributes['start_date'] = str( ds1.series['DateTime']['Data'][0]) if 'end_date' not in ds1.globalattributes.keys(): ds1.globalattributes['end_date'] = str( ds1.series['DateTime']['Data'][-1]) # calculate variances from standard deviations and vice versa qcts.CalculateStandardDeviations(cf, ds1) # create new variables using user defined functions qcts.DoFunctions(cf, ds1) # create a series of synthetic downwelling shortwave radiation qcts.get_synthetic_fsd(ds1)
def xl2nc(cf,InLevel): # get the data series from the Excel file ds = xl_read_series(cf) if len(ds.series.keys())==0: return 1 # get the netCDF attributes from the control file qcts.do_attributes(cf,ds) # get a series of Python datetime objects from the Excel datetime qcutils.get_datetimefromxldate(ds) #check for gaps in the Excel datetime series has_gaps = qcutils.CheckTimeStep(ds,fix='gaps') # write the processing level to a global attribute ds.globalattributes['nc_level'] = str(InLevel) # get the start and end date from the datetime series unless they were # given in the control file if 'start_date' not in ds.globalattributes.keys(): ds.globalattributes['start_date'] = str(ds.series['DateTime']['Data'][0]) if 'end_date' not in ds.globalattributes.keys(): ds.globalattributes['end_date'] = str(ds.series['DateTime']['Data'][-1]) # get the year, month, day, hour, minute and second from the xl date/time qcutils.get_ymdhmsfromxldate(ds) # do any functions to create new series qcts.do_functions(cf,ds) # write the data to the netCDF file outfilename = get_outfilename_from_cf(cf) ncFile = nc_open_write(outfilename) nc_write_series(ncFile,ds) return 0
def l1qc(cf): # get the data series from the Excel file in_filename = qcio.get_infilenamefromcf(cf) if not qcutils.file_exists(in_filename, mode="quiet"): msg = " Input file " + in_filename + " not found ..." logger.error(msg) ds1 = qcio.DataStructure() ds1.returncodes = {"value": 1, "message": msg} return ds1 file_name, file_extension = os.path.splitext(in_filename) if "csv" in file_extension.lower(): ds1 = qcio.csv_read_series(cf) if ds1.returncodes["value"] != 0: return ds1 # get a series of Excel datetime from the Python datetime objects qcutils.get_xldatefromdatetime(ds1) else: ds1 = qcio.xl_read_series(cf) if ds1.returncodes["value"] != 0: return ds1 # get a series of Python datetime objects from the Excel datetime qcutils.get_datetimefromxldate(ds1) # get the netCDF attributes from the control file qcts.do_attributes(cf, ds1) # round the Python datetime to the nearest second qcutils.round_datetime(ds1, mode="nearest_second") #check for gaps in the Python datetime series and fix if present fixtimestepmethod = qcutils.get_keyvaluefromcf(cf, ["options"], "FixTimeStepMethod", default="round") if qcutils.CheckTimeStep(ds1): qcutils.FixTimeStep(ds1, fixtimestepmethod=fixtimestepmethod) # recalculate the Excel datetime qcutils.get_xldatefromdatetime(ds1) # get the Year, Month, Day etc from the Python datetime qcutils.get_ymdhmsfromdatetime(ds1) # write the processing level to a global attribute ds1.globalattributes['nc_level'] = str("L1") # get the start and end date from the datetime series unless they were # given in the control file if 'start_date' not in ds1.globalattributes.keys(): ds1.globalattributes['start_date'] = str( ds1.series['DateTime']['Data'][0]) if 'end_date' not in ds1.globalattributes.keys(): ds1.globalattributes['end_date'] = str( ds1.series['DateTime']['Data'][-1]) # calculate variances from standard deviations and vice versa qcts.CalculateStandardDeviations(cf, ds1) # create new variables using user defined functions qcts.DoFunctions(cf, ds1) # create a series of synthetic downwelling shortwave radiation qcts.get_synthetic_fsd(ds1) # check missing data and QC flags are consistent qcutils.CheckQCFlags(ds1) return ds1
logging.info('Creating the data structures') ds_60minutes = qcio.DataStructure() # get a sorted list of files that match the mask in the control file file_list = sorted(glob.glob(info["in_filename"])) # read the netcdf files logging.info('Reading the netCDF files for '+info["site_name"]) f = access_read_mfiles2(file_list,var_list=var_list) # get the data from the netCDF files and write it to the 60 minute data structure logging.info('Getting the ACCESS data') get_accessdata(cf,ds_60minutes,f,info) # set some global attributes logging.info('Setting global attributes') set_globalattributes(ds_60minutes,info) # check for time gaps in the file logging.info("Checking for time gaps") if qcutils.CheckTimeStep(ds_60minutes): qcutils.FixTimeStep(ds_60minutes) # get the datetime in some different formats logging.info('Getting xlDateTime and YMDHMS') qcutils.get_xldatefromdatetime(ds_60minutes) qcutils.get_ymdhmsfromdatetime(ds_60minutes) #f.close() # get derived quantities and adjust units logging.info("Changing units and getting derived quantities") # air temperature from K to C changeunits_airtemperature(ds_60minutes) # soil temperature from K to C changeunits_soiltemperature(ds_60minutes) # pressure from Pa to kPa changeunits_pressure(ds_60minutes) # wind speed from components
import numpy import os import sys # check the scripts directory is present if not os.path.exists("../scripts/"): print("erai2nc: the scripts directory is missing") sys.exit() # since the scripts directory is there, try importing the modules sys.path.append('../scripts') import qcio import qcutils aws_name = qcio.get_filename_dialog(path="/mnt/OzFlux/Sites") ds_aws_30minute = qcio.nc_read_series(aws_name) has_gaps = qcutils.CheckTimeStep(ds_aws_30minute) if has_gaps: print("Problems found with time step") qcutils.FixTimeStep(ds_aws_30minute) qcutils.get_ymdhmsfromdatetime(ds_aws_30minute) dt_aws_30minute = ds_aws_30minute.series["DateTime"]["Data"] ddt = [ dt_aws_30minute[i + 1] - dt_aws_30minute[i] for i in range(0, len(dt_aws_30minute) - 1) ] print("Minimum time step is", min(ddt), " Maximum time step is", max(ddt)) dt_aws_30minute = ds_aws_30minute.series["DateTime"]["Data"] start_date = dt_aws_30minute[0] end_date = dt_aws_30minute[-1]
qcutils.CreateSeries(ds,'Ws',data_dict[bom_id][:,10],Flag=flag,Attr=attr) attr=qcutils.MakeAttributeDictionary(long_name='Wind direction',units='degT', bom_id=str(bom_id),bom_name=bom_sites_info[site_name][str(bom_id)]["site_name"], bom_dist=bom_sites_info[site_name][str(bom_id)]["distance"]) qcutils.CreateSeries(ds,'Wd',data_dict[bom_id][:,11],Flag=flag,Attr=attr) attr=qcutils.MakeAttributeDictionary(long_name='Wind gust',units='m/s', bom_id=str(bom_id),bom_name=bom_sites_info[site_name][str(bom_id)]["site_name"], bom_dist=bom_sites_info[site_name][str(bom_id)]["distance"]) qcutils.CreateSeries(ds,'Wg',data_dict[bom_id][:,12],Flag=flag,Attr=attr) data_dict[bom_id][:,13] = data_dict[bom_id][:,13]/float(10) attr=qcutils.MakeAttributeDictionary(long_name='Air Pressure',units='kPa', bom_id=str(bom_id),bom_name=bom_sites_info[site_name][str(bom_id)]["site_name"], bom_dist=bom_sites_info[site_name][str(bom_id)]["distance"]) qcutils.CreateSeries(ds,'ps',data_dict[bom_id][:,13],Flag=flag,Attr=attr) # fix any time stamp issues if qcutils.CheckTimeStep(ds): qcutils.FixTimeStep(ds) # update the Year, Month, Day etc from the Python datetime qcutils.get_ymdhmsfromdatetime(ds) # now interpolate for label in ["Precip","Ta","Td","RH","Ws","Wd","Wg","ps"]: qcts.InterpolateOverMissing(ds,series=label,maxlen=2) # put this stations data into the data structure dictionary ds_dict[bom_id] = ds # get the earliest start datetime and the latest end datetime log.info("Finding the start and end dates") bom_id_list = ds_dict.keys() ds0 = ds_dict[bom_id_list[0]] ldt = ds0.series["DateTime"]["Data"] #print bom_id_list[0],":",ldt[0],ldt[-1]
# get the site time zone site_timezone = cf["Sites"][site]["site_timezone"] # read the BIOS file bios_ncfile = netCDF4.Dataset(infilename) time = bios_ncfile.variables["time"][:] nRecs = len(time) # set some global attributes ts = ds_30.globalattributes["time_step"] = 30 ds_30.globalattributes["time_zone"] = site_timezone ds_30.globalattributes["nc_nrecs"] = nRecs ds_30.globalattributes["xl_datemode"] = str(0) ds_30.globalattributes["site_name"] = cf["Sites"][site]["site_name"] time_units = getattr(bios_ncfile.variables["time"],"units") qcutils.get_datetimefromnctime(ds_30,time,time_units) qcutils.round_datetime(ds_30,mode="nearest_timestep") if qcutils.CheckTimeStep(ds_30): qcutils.FixTimeStep(ds_30) ldt_30 = ds_30.series["DateTime"]["Data"] si = qcutils.GetDateIndex(ldt_30,start_date,default=0,ts=ts,match="startnexthour") ei = qcutils.GetDateIndex(ldt_30,end_date,default=len(ldt_30),ts=ts,match="endprevioushour") ds_30.series["DateTime"]["Data"] = ds_30.series["DateTime"]["Data"][si:ei+1] ds_30.series["DateTime"]["Flag"] = ds_30.series["DateTime"]["Flag"][si:ei+1] ldt_30 = ds_30.series["DateTime"]["Data"] nRecs = ds_30.globalattributes["nc_nrecs"] = len(ldt_30) flag = numpy.zeros(nRecs) qcutils.get_ymdhmsfromdatetime(ds_30) xl_date_loc = qcutils.get_xldatefromdatetime(ds_30) attr = qcutils.MakeAttributeDictionary(long_name="Date/time (local) in Excel format",units="days since 1899-12-31 00:00:00") qcutils.CreateSeries(ds_30,"xlDateTime",xl_date_loc,flag,attr) # get the data for label in var_list: bios_name = cf["Variables"][label]["bios_name"]