def read_processed_hsrl_data(filename): """read a netcdf file created by write_netcdf. Data will be restored to python structures""" from pycdf import CDF, CDFError nc = CDF(filename) nc_vars = nc.variables().keys() for name in nc_vars: print name, ' \t\t ', nc.var(name).dimensions() tmp = nc.var(name).get()
def make_netcdf_template(instrument, in_dir, netcdf_filename, outfilename): """reads a netcdf file and creates a outfilename.json with a template for reading this netcdf file--this file will must be edited to provide name translations and variable type information""" from pycdf import CDF, CDFError if netcdf_filename.find('.bz2') >= 0: index = netcdf_filename.find('zraw/') index2 = netcdf_filename.find('_') fileid = open(in_dir + netcdf_filename, 'r') file = fileid.read() fileid.close() temp = bz2.decompress(file) # write temp file to memory rather than disk netcdf_filename = '/dev/shm/' + instrument + '_temp_file.nc' fileid = open(netcdf_filename, 'w') fileid.write(temp) nc = CDF(netcdf_filename) nc_vars = nc.variables().keys() dims = nc.dimensions() print dims f = open(outfilename, 'w') header = ( '#\n' + '#edit this file and change extention to *.json\n' + '#entry format:"\n' + '#"desired_var_name":["name_in_netcdf","select_flag"]\n' + '#var_type == "t" ,time var or time by other than z\n' + '#select_flag when set = 0 and read_mode=="min",variable skipped\n' + '#remove these comment lines before use\n' + '#\n' + '{"config":\n' + ' {"#read_mode":"all / select / min",\n' + ' "read_mode": "select"},\n' + '"selected_vars":\n' + ' {\n') f.write(header) print header first = True for name in nc_vars: var = nc.var(name) #print help(nc) if first: print ' "' + name + '":["' + name + '", 1]' f.write(' "' + name + '":["' + name + '",1]') else: print ' "' + name + '":["' + name + '",1]' f.write(',\n "' + name + '":["' + name + '",1]') first = False f.write(' }\n') f.write('}\n')
def read_processed_hsrl_data(filename, var_name=None, list_vars=None): """read_process_hsrl_data(netcdf_file,variable,list_vars = False) read a netcdf variable from a file created by write_netcdf if list_variables = True, provide a list of file variables""" from pycdf import CDF nc = CDF(filename) if not list_vars == None: nc_vars = nc.variables().keys() for name in nc_vars: print ' %30s %40s' % (name.ljust(30), nc.var(name).dimensions()) return if not var_name == None: if var_name == 'time' or var_name == 'times': # inits a datetime object from the base time base_time = nc.var('base_time').get() time_offset = nc.var('time').get() #base = datetime.datetime.fromtimestamp(base_time,tz=datetime.tzinfo("UTC")) base = datetime.datetime(1970, 1, 1, 0, 0, 0) + datetime.timedelta( seconds=int(base_time)) # adds a time offset (seconds) to the base time to make a datetime object var = np.array([(base + datetime.timedelta(seconds=x)) for x in time_offset]) else: var = nc.var(var_name).get() pass return var
def save_netcdf(self, flo, **kwargs): """Save the current dataset to the given file as a netCDF dataset to be used with Alex Graves nnl_ndim program in task="sequence classification" mode.""" # make sure classes are defined properly assert len(self['class']) == len(self['target']) if self.nClasses > 10: raise from pycdf import CDF, NC # need to regenerate the file name filename = flo.name flo.close() # Create the file. Raise the automode flag, so that # we do not need to worry about setting the define/data mode. d = CDF(filename, NC.WRITE | NC.CREATE | NC.TRUNC) d.automode() # Create 2 global attributes, one holding a string, # and the other one 2 floats. d.title = 'Sequential data exported from PyBrain (www.pybrain.org)' # create the dimensions dimsize = { 'numTimesteps': len(self), 'inputPattSize': self.indim, 'numLabels': self.nClasses, 'numSeqs': self.getNumSequences(), 'maxLabelLength': 2 } dims = {} for name, sz in dimsize.iteritems(): dims[name] = d.def_dim(name, sz) # Create a netCDF record variables inputs = d.def_var('inputs', NC.FLOAT, (dims['numTimesteps'], dims['inputPattSize'])) targetStrings = d.def_var('targetStrings', NC.CHAR, (dims['numSeqs'], dims['maxLabelLength'])) seqLengths = d.def_var('seqLengths', NC.INT, (dims['numSeqs'])) labels = d.def_var('labels', NC.CHAR, (dims['numLabels'], dims['maxLabelLength'])) # Switch to data mode (automatic) # assign float and integer arrays directly inputs.put(self['input'].astype(single)) # strings must be written as scalars (sucks!) for i in range(dimsize['numSeqs']): targetStrings.put_1(i, str(self.getSequenceClass(i))) for i in range(self.nClasses): labels.put_1(i, str(i)) # need colon syntax for assigning list seqLengths[:] = [ self.getSequenceLength(i) for i in range(self.getNumSequences()) ] # Close file print("wrote netCDF file " + filename) d.close()