def test_grid_same_as_pytdlpack(request): sampledata = request.config.rootdir / 'sampledata' ds = xr.open_dataset(sampledata / 'gfspkd47.2017020100.sq', engine='tdlpack', filters=dict(cccfff=1000)) da = ds['001_000'].sel(uuuu=1000).squeeze() import pytdlpack with pytdlpack.open(sampledata / 'gfspkd47.2017020100.sq') as f: rec = f.read() rec.unpack(data=True) np.testing.assert_array_equal(da.data, rec.data.transpose())
id = [4210008,10,24,0] grid_data = np.random.rand(nx,ny)*75.0 grid_data.fill(np.nan) # ---------------------------------------------------------------------------------------- # Grid Specs: CONUS Lambert-Conformal 2.5km 2345x1597 # ---------------------------------------------------------------------------------------- griddef = pytdlpack.create_grid_definition(proj=3,nx=nx,ny=ny,latll=19.2290, lonll=233.7234,orientlon=265.,stdlat=25.,meshlength=2.539703) # ---------------------------------------------------------------------------------------- # Create TDLPACK data record and pack # ---------------------------------------------------------------------------------------- rec = pytdlpack.TdlpackRecord(date=date,id=id,lead=24,plain="GFS WIND SPEED", data=grid_data,missing_value=9999.0,grid=griddef) rec.pack(dec_scale=3) # ---------------------------------------------------------------------------------------- # Open new sequential file and write the records # ---------------------------------------------------------------------------------------- f = pytdlpack.open('new_grid.sq',mode='w',format='sequential') f.write(rec) f.close() # ---------------------------------------------------------------------------------------- # Open new random-access file and write the records # ---------------------------------------------------------------------------------------- fra = pytdlpack.open('new_grid.ra',mode='w',format='random-access',ra_template='large') fra.write(rec) fra.close()
def to_tdlpack(self, file, mode='w-', compute: bool = True, var_constants=None, min_unique=1000): ''' mode : {"w", "w-"}, optional, default: "w-" Persistence mode: "w" means create (overwrite if exists); "w-" means create (fail if exists); ''' have_chunks = any(v.chunks for v in self._obj.variables.values()) # ensuring has x/y or station dims and that any chunks do not span those dims if 'station' in self._obj.dims: station = True if have_chunks: self._obj = self._obj.chunk({'station': -1}) elif 'x' in self._obj.dims and 'y' in self._obj.dims: station = False if have_chunks: self._obj = self._obj.chunk({'x': -1, 'y': -1}) else: raise ValueError( "data does not have 'x' and 'y' or 'station' dims for writing to tdlp grid or station formats" ) possible_multi_var_keys = [ 'ccc', 'fff', 'b', 'dd', 'v', 'llll', 'uuuu', 't', 'o', 'i', 's', 'g' ] multi_var_keys = [ k for k in possible_multi_var_keys if not self._iscoord(k) ] meta_dicts = list() for var in self._obj.data_vars: da = self._obj[var] meta_dicts.append( {key: da.encoding[f'tdlp_{key}'] for key in multi_var_keys}) df = pd.DataFrame(meta_dicts).nunique() meta_varying_by_var = df.index[df > 1] meta = RequiredTdlpMeta() keys = list(meta.__dataclass_fields__.keys()) coord_meta = list() const_meta = list() tdlpid = TdlpId() for key in keys: if f'tdlp_{key}' in da.encoding: meta[key] = da.encoding[f'tdlp_{key}'] tdlpid[key] = meta[key] const_meta.append(key) continue found = False for coord_name in self._obj.coords: coord = self._obj[coord_name] if 'tdlp_name' in coord.attrs: if coord.attrs['tdlp_name'] == key: found = True coord_meta.append(key) meta[key] = coord break if not found: raise ValueError( f'to_tdlpack requres metadata for {key} be in encoding or coordinate' ) filepath = Path(file) if mode == 'w-': if filepath.exists(): raise ValueError( f"{file} already exists and will not be overwritten; mode: 'w' can overwrite existing files" ) elif mode == 'w': if filepath.is_dir(): raise ValueError(f"cannot clobber directory {file}") open(filepath, 'w').close() store = filepath.parent / f'.{filepath.name}' if store.is_dir(): logger.warning(f'removing existing hidden directory {store}') shutil.rmtree(store) store.mkdir(parents=True) prodicized = product(*[meta[k] for k in coord_meta]) f = pytdlpack.open(store / filepath.name, mode='w', format='sequential') if station: template_rec = pytdlpack.TdlpackRecord(date=0, id=[0, 0, 0, 0], data=np.array([0])) stations = pytdlpack.TdlpackStationRecord( list(self._obj.station.data)) stations.pack() f.write(stations) else: # the grid doesn't matter ( can tweak/clean later) template_rec = pytdlpack.TdlpackRecord( date=0, id=[0, 0, 0, 0], grid=pytdlpack.grids['nbmak'], data=np.array([0])) template_rec.is2 = da.encoding[ 'tdlp_is2'] # this loads the grid metadata template_rec.primary_missing_value = 9999.0 for t in prodicized: for var in self._obj.data_vars: # select slice of array for tdlpack record loc = {k: v for (k, v) in zip(coord_meta, t)} da = self._obj[var].loc[loc].squeeze() # put extra metadata that varies by variable in loc for updating tdlpid for m in meta_varying_by_var: loc[m] = da.encoding[f'tdlp_{m}'] tdlpid.update(**loc) # shape data array appropriately for station or grid formatted tdlpack record if station: data = da.data else: data = da.data.transpose() # build out a tdlpack DataRecord with appropriate metadata idlist = [ tdlpid.word1, tdlpid.word2, tdlpid.word3, tdlpid.word4 ] if var_constants is None: plain = 'NO VAR MATCH' # let dec_scale allow for min_unique values in the space between the max and min datamax = np.nanmax(data) datamin = np.nanmin(data) if datamax == datamin or np.isnan(datamax): dec_scale = 9 # data is a constant or all missing and will compress well else: log10range = np.log10( np.nanmax(data) - np.nanmin(data)) range_place = np.floor(log10range) dec_scale = int( np.ceil(np.log10(min_unique)) - range_place) else: plain = var_constants.loc[tdlpid.cccfff]['plain'] dec_scale = var_constants.loc[tdlpid.cccfff]['iscale'] date = da.date.data.squeeze()[()] rec = make_record(template_rec, idlist, data, 'PLAIN TEXT', date) rec.pack(dec_scale=dec_scale) logger.debug( f'writing {date}, {idlist} with dec_scale: {dec_scale}') f.write(rec) f.close() shutil.move(store / filepath.name, file) shutil.rmtree(store)
# ---------------------------------------------------------------------------------------- # Import Modules # ---------------------------------------------------------------------------------------- import copy import filecmp import numpy as np import setuptools import sys platform = setuptools.distutils.util.get_platform() build_path = './build/lib.'+platform+'-'+str(sys.version_info.major)+'.'+str(sys.version_info.minor) sys.path.insert(0,build_path) import pytdlpack file_input = "sampledata/gfspkd47.2017020100.sq" file_output = "test.sq" f = pytdlpack.open(file_input) fout = pytdlpack.open(file_output,mode="w") recs = f.read(all=True) for r in recs: r.unpack() newrec = copy.deepcopy(r) newrec.pack() fout.write(newrec) f.close() fout.close() filecompare = filecmp.cmp(file_input,file_output) print("filecmp result:",filecompare)
#!/usr/bin/env python3 # ---------------------------------------------------------------------------------------- # Import Modules # ---------------------------------------------------------------------------------------- import numpy as np import setuptools import sys platform = setuptools.distutils.util.get_platform() build_path = './build/lib.'+platform+'-'+str(sys.version_info.major)+'.'+str(sys.version_info.minor) sys.path.insert(0,build_path) import pytdlpack f = pytdlpack.open('sampledata/blend.analysisgrconst.co.ra') recs = f.read(all=True) for r in recs: r.unpack(data=True) f.close()
#!/usr/bin/env python # ---------------------------------------------------------------------------------------- # Import modules # ---------------------------------------------------------------------------------------- import pytdlpack import numpy as np # ---------------------------------------------------------------------------------------- # Open new TDLPACK file. Here we use pytdlpack.open() which is a function that returns # an instance (i.e. an object) of class pytdlpack.TdlpackFile. # ---------------------------------------------------------------------------------------- f = pytdlpack.open("station.sq",mode="w") print "TDLPACK FILE INFO" print f # ---------------------------------------------------------------------------------------- # Define a station list and create an instance of TdlpackStationRecord # ---------------------------------------------------------------------------------------- ccall = ('KBWI','KPHL','KIAD','KLNS','KACY') # Parentheses indictates a tuple (can also be a list). stationrec = pytdlpack.TdlpackStationRecord(ccall=ccall) # All that is needed a station call letter list/tuple. stationrec.pack() # pack() is a method (i.e. a function that acts on a class instance). f.write(stationrec) # write() is a method of class TdlpackFile. # ---------------------------------------------------------------------------------------- # Define TDLPACK Identification Sections # # NOTE: The pytdlpack module contains constansts to use! -- Like ND7 # ---------------------------------------------------------------------------------------- is1 = np.zeros((pytdlpack.ND7),dtype=np.int32) is1[0] = 0
date = 2019052900 id = [4210008, 10, 24, 0] station_name = ('KACY', 'KBWI', 'KDCA', 'KIAD', 'KPHL') station_data = [10.3, 12.4, 15.6, 8.6, 9999.0] # ---------------------------------------------------------------------------------------- # Create station record and pack # ---------------------------------------------------------------------------------------- sta = pytdlpack.TdlpackStationRecord(station_name) sta.pack() # ---------------------------------------------------------------------------------------- # Create TDLPACK data record and pack # ---------------------------------------------------------------------------------------- rec = pytdlpack.TdlpackRecord(date=date, id=id, lead=24, plain="GFS WIND SPEED", data=station_data, missing_value=9999.0) rec.pack(dec_scale=1) # ---------------------------------------------------------------------------------------- # Open new sequential file and write the records # ---------------------------------------------------------------------------------------- f = pytdlpack.open('new_station.sq', mode='w', format='sequential') f.write(sta) f.write(rec) f.write(pytdlpack.TdlpackTrailerRecord()) f.close()