def create_ncdf(fname, dimensions, variables, metafname=None, global_attributes=None, path='', mode='w', clobber=False, format='NETCDF4_CLASSIC'): ''' function creates an netCDF file (appends attributes, dimensions, variables) NO DATA IS STORED within created variables input: fname [str] - filename of netcdf file to be created path [str] - path of resulting netcdf file. Default is '' dimensions [dict] - dictionary with information about dimensions, where keys - dimension names, values - dimension sizes. Note: values could be any integer >0, None, or '*' variables [dict] - dictionary with strored variable info. This dictionary has a format as below... {"var_name": ["dtype", [dimension1, dimension2,...], {"attribute1_name": attribute1_val, "attribute2_name": attribute2_val, ...}], ...} global_attributes [dict] - dictionary with global attributes. Default is "None" other - inherited from netCDF4.Dataset() ''' from netCDF4 import Dataset import read_cdl #initialize root_grp = Dataset(path+fname, mode=mode, clobber=clobber, format=format) # make global attributes append_netcdf_glb_attrs(root_grp, global_attributes) #create dimensions dimensions, udimension_arrays = read_cdl.check_undefined_dimensions(dimensions, metafname=metafname) create_netcdf_dimensions(root_grp, dimensions) # create variables create_netcdf_variable(root_grp, variables) # fill variables of undefined dimensions fill_undefined_dimension_variables(root_grp, udimension_arrays) # set DATA #Closing ncdf root_grp.close() print 'File created succesfully: %s' % (path+fname)
def make_cdl(cdlFname, metafname=None, outpath='', log=False): ''' create a copy of existing CDL format file, where missing data is substituted with "*" sign and modify it file step by step: _________________________________ 1) reading CDL format file, extracting info from sections: "dimensions:", "variables:", !NOT IMPLEMENTED ! "data" 2) reading observation data if nessesary (dimensions are unknown dim="*" ) ''' sname = 'make_cdl():' import shutil import read_cdl import os import read_metafile import funcs import re # first make a copy to work with path , fname = os.path.split(os.path.abspath(cdlFname)) pathmeta , fnamemeta = os.path.split(os.path.abspath(metafname)) name, extension = fname.split('.') #newFilename = outpath+name+'_modified.'+extension tempFilename = cdlFname+'.tmp' newFileName = os.path.join(outpath, "_"+fname) #shutil.copyfile(cdlFname, cdlFname+'.bak') # proceed f = read_cdl.read_file(cdlFname, comments='//') parts, nparts, fname = read_cdl.get_parts(f) dims = read_cdl.process_dimensions(parts[0], log=log) Vars = read_cdl.process_variables(parts[1], log=log) read_cdl.process_variables_attributes(parts[1], Vars, log=log) read_cdl.check_variables_and_dimensions(dims, Vars) dimensions, udimension_arrays = read_cdl.check_undefined_dimensions(dims, metafname=metafname, log=log) Vars = read_cdl.extend_dimensions_with_sizes(Vars, dimensions) replace_undefined_dimensions(cdlFname, newFileName, dimensions, log=log) #replace_undefined_dimensions(cdlFname, newFilename, dims) metaData = read_metafile.read_metadata(metafname, log=log) # undefined_vars = read_cdl.get_undefVars() with open(newFileName, 'r') as fIn: if log: print sname, "creating temp file {0}".format(tempFilename) with open(tempFilename, 'w+') as fOut: for i, lineRaw in enumerate(fIn): #cycling through lines in old file... lineRaw = lineRaw.decode('utf8') if len(lineRaw.split('//')) > 1: # ignore comments line = lineRaw.split("//")[0] else: line = lineRaw if re.match('.*?=\s*\*.*', line): # if * is present after sign = (undefined variable is here, cause there are already no undefined dimensions at this step) uVarName = re.match('\s*?(.*?)\s*=.*', line).group(1).strip() if log: print sname, "line {0} >>> Undefined variable '{1}'' found in CDL ".format(i+1, uVarName) if uVarName in Vars.keys(): fOut.write('\t{0} =\n'.format(uVarName)) # first write variable name... # creating array with data... if log: print sname, 'Creating array based on: "{0}" = {1}'.format(uVarName, Vars[uVarName]) useCols = create_useCols_from_meta(metaData, Vars, uVarName, log=log) if log: print sname, 'variable {0} >>> useCols = {1}'.format (uVarName, useCols) obs = funcs.loadASCII(os.path.join(pathmeta, metaData['dataFname']), dtype=Vars[uVarName][0], delimiter=metaData['delimiter'], skiprows=metaData['skiprows'], usecols=useCols, log=log) #for k ,v in Vars.iteritems(): print k, '>>>', v if "_FillValue" in Vars[uVarName][2].keys(): fv = Vars[uVarName][2]["_FillValue"][0] else: fv = -999 data = funcs.create_array_from_data(Vars[uVarName][3], Vars[uVarName][1], uVarName, obs, fill_value=fv, log=log) # now write it to cdl in column order.... data = data.flatten(order="C") # flatten array in column order as in C-language if log: print sname, 'writing to files data-section values of variable <{0}>... be patient - this may take few minutes'.format(uVarName) for i, val in enumerate(data): # write to file... string by string if i != (data.size - 1): # if not last element fOut.write('\t\t{0} ,\n'.format(val)) else: #if last element fOut.write('\t\t{0} ;\n'.format(val)) else: err_msg = 'CDL file contains undefined variable {0}, which is missing in dictionary "Vars"'.format(uVarName) raise ValueError(err_msg+'\n\n') else: if log : print sname, 'line {0} >>> no changes'.format(i+1) fOut.write(lineRaw) # if line without * => rewrite it fOut.close() fIn.close() try: os.remove(newFileName) # if file exist - remove itrename() doesn't overwrite on Windows except: pass # if file do not exist, do nothing os.rename(tempFilename, newFileName) if log: print sname, "renaming temp file into >>> {0}".format(newFileName)