Ejemplo n.º 1
0
def make_netcdf(cdlFname, metafname=None, outpath=''):
    '''
    creates netcdf file step by step:
    _________________________________

    1) reading CDL format file, extracting info from sections:
        "dimensions:",
        "variables:",
        !NOT IMPLEMENTED !    "data"

    2) reading observation data if nessesary (dimensions are unknown dim="*" )
    
    3) creates netcdf file

    !NOT IMPLEMENTED ! 4) read metadata file supplied with data, extracts information about how to process data

    !NOT IMPLEMENTED ! 5) read data and fill it into netcdf
    '''
    import read_cdl

    f = read_cdl.read_file(cdlFname, comments='//')
    parts, nparts, fname = read_cdl.get_parts(f)
    dims = read_cdl.process_dimensions(parts[0])
    glb_attrs = read_cdl.process_global_attributes(parts[1])
    vars = read_cdl.process_variables(parts[1])
    read_cdl.check_variables_and_dimensions(dims, vars)
    read_cdl.process_variables_attributes(parts[1], vars)
    create_ncdf(fname, dims, vars, metafname=metafname, global_attributes=glb_attrs, path=outpath, clobber=True)
Ejemplo n.º 2
0
def make_cdl(cdlFname, metafname=None, outpath='', log=False):
    '''
    create a copy of existing CDL format file, where missing data is substituted with "*" sign
    and modify it file step by step:
    _________________________________

    1) reading CDL format file, extracting info from sections:
        "dimensions:",
        "variables:",
        !NOT IMPLEMENTED !    "data"

    2) reading observation data if nessesary (dimensions are unknown dim="*" )
    

    '''
    sname = 'make_cdl():'
    import shutil
    import read_cdl
    import os
    import read_metafile
    import funcs
    import re

    # first make a copy to work with
    path , fname = os.path.split(os.path.abspath(cdlFname))
    pathmeta , fnamemeta = os.path.split(os.path.abspath(metafname))
    name, extension = fname.split('.')
    #newFilename  = outpath+name+'_modified.'+extension
    tempFilename = cdlFname+'.tmp'
    newFileName = os.path.join(outpath, "_"+fname)
    #shutil.copyfile(cdlFname, cdlFname+'.bak')

    # proceed
    f = read_cdl.read_file(cdlFname, comments='//')
    parts, nparts, fname = read_cdl.get_parts(f)
    dims = read_cdl.process_dimensions(parts[0], log=log)
    Vars = read_cdl.process_variables(parts[1], log=log)
    read_cdl.process_variables_attributes(parts[1], Vars, log=log)
    read_cdl.check_variables_and_dimensions(dims, Vars)
    dimensions, udimension_arrays = read_cdl.check_undefined_dimensions(dims, metafname=metafname, log=log)
    Vars = read_cdl.extend_dimensions_with_sizes(Vars, dimensions)
    replace_undefined_dimensions(cdlFname, newFileName, dimensions, log=log)
    #replace_undefined_dimensions(cdlFname, newFilename, dims)
    metaData = read_metafile.read_metadata(metafname, log=log)
    # undefined_vars = read_cdl.get_undefVars()


    with open(newFileName, 'r') as fIn:
        if log: print sname, "creating temp file {0}".format(tempFilename)
        with open(tempFilename, 'w+') as fOut:
            for i, lineRaw in enumerate(fIn):  #cycling through lines in old file...
                lineRaw = lineRaw.decode('utf8')

                if len(lineRaw.split('//')) > 1:  # ignore comments
                    line = lineRaw.split("//")[0]
                else:
                    line = lineRaw

                if re.match('.*?=\s*\*.*', line):  # if * is present after sign = (undefined variable is here, cause there are already no undefined dimensions at this step)
                    uVarName = re.match('\s*?(.*?)\s*=.*', line).group(1).strip()
                    if log: print sname, "line {0} >>> Undefined variable '{1}'' found in CDL ".format(i+1, uVarName)
                    if uVarName in Vars.keys():
                        fOut.write('\t{0} =\n'.format(uVarName))  # first write variable name...

                        # creating array with data...
                        if log: print sname, 'Creating array based on: "{0}" = {1}'.format(uVarName, Vars[uVarName])
                        useCols = create_useCols_from_meta(metaData, Vars, uVarName, log=log)

                        if log: print sname, 'variable {0} >>> useCols = {1}'.format (uVarName, useCols)
                        obs = funcs.loadASCII(os.path.join(pathmeta, metaData['dataFname']), dtype=Vars[uVarName][0], delimiter=metaData['delimiter'],
                                              skiprows=metaData['skiprows'], usecols=useCols, log=log)
                        #for k ,v in Vars.iteritems(): print k, '>>>', v
                        if "_FillValue" in Vars[uVarName][2].keys():
                            fv = Vars[uVarName][2]["_FillValue"][0]
                        else:
                            fv = -999

                        data = funcs.create_array_from_data(Vars[uVarName][3], Vars[uVarName][1], uVarName, obs, fill_value=fv, log=log)

                        
                        # now write it to cdl in column order....
                        data = data.flatten(order="C")  # flatten array in column order as in C-language
                        if log: print sname, 'writing to files data-section values of variable <{0}>... be patient - this may take few minutes'.format(uVarName)
                        
                        for i, val in enumerate(data):  # write to file... string by string
                            if i != (data.size - 1):  # if not last element
                                fOut.write('\t\t{0} ,\n'.format(val))
                            else:  #if last element
                                fOut.write('\t\t{0} ;\n'.format(val))


                    else:
                        err_msg = 'CDL file contains undefined variable {0}, which is missing in dictionary "Vars"'.format(uVarName)
                        raise ValueError(err_msg+'\n\n')
                else:
                    if log : print sname, 'line {0} >>> no changes'.format(i+1)
                    fOut.write(lineRaw)  # if line without * => rewrite it
            fOut.close()
        fIn.close()

    try: os.remove(newFileName)  # if file exist - remove itrename() doesn't overwrite on Windows
    except: pass  # if file do not exist, do nothing
    os.rename(tempFilename, newFileName)
    if log: print sname, "renaming temp file into >>> {0}".format(newFileName)