def read_processed_hsrl_data(filename):
"""read a netcdf file created by write_netcdf. Data will be
restored to python structures"""
from pycdf import CDF, CDFError
nc = CDF(filename)
nc_vars = nc.variables().keys()
for name in nc_vars:
print name, ' \t\t ', nc.var(name).dimensions()
tmp = nc.var(name).get()
def make_netcdf_template(instrument, in_dir, netcdf_filename, outfilename):
"""reads a netcdf file and creates a outfilename.json
with a template for reading this netcdf file--this file will
must be edited to provide name translations
and variable type information"""
from pycdf import CDF, CDFError
if netcdf_filename.find('.bz2') >= 0:
index = netcdf_filename.find('zraw/')
index2 = netcdf_filename.find('_')
fileid = open(in_dir + netcdf_filename, 'r')
file = fileid.read()
fileid.close()
temp = bz2.decompress(file)
# write temp file to memory rather than disk
netcdf_filename = '/dev/shm/' + instrument + '_temp_file.nc'
fileid = open(netcdf_filename, 'w')
fileid.write(temp)
nc = CDF(netcdf_filename)
nc_vars = nc.variables().keys()
dims = nc.dimensions()
print dims
f = open(outfilename, 'w')
header = (
'#\n' + '#edit this file and change extention to *.json\n' +
'#entry format:"\n' +
'#"desired_var_name":["name_in_netcdf","select_flag"]\n' +
'#var_type == "t" ,time var or time by other than z\n' +
'#select_flag when set = 0 and read_mode=="min",variable skipped\n' +
'#remove these comment lines before use\n' + '#\n' + '{"config":\n' +
' {"#read_mode":"all / select / min",\n' +
' "read_mode": "select"},\n' + '"selected_vars":\n' + ' {\n')
f.write(header)
print header
first = True
for name in nc_vars:
var = nc.var(name)
#print help(nc)
if first:
print ' "' + name + '":["' + name + '", 1]'
f.write(' "' + name + '":["' + name + '",1]')
else:
print ' "' + name + '":["' + name + '",1]'
f.write(',\n "' + name + '":["' + name + '",1]')
first = False
f.write(' }\n')
f.write('}\n')
def read_processed_hsrl_data(filename, var_name=None, list_vars=None):
"""read_process_hsrl_data(netcdf_file,variable,list_vars = False)
read a netcdf variable from a file created by write_netcdf
if list_variables = True, provide a list of file variables"""
from pycdf import CDF
nc = CDF(filename)
if not list_vars == None:
nc_vars = nc.variables().keys()
for name in nc_vars:
print ' %30s %40s' % (name.ljust(30), nc.var(name).dimensions())
return
if not var_name == None:
if var_name == 'time' or var_name == 'times':
# inits a datetime object from the base time
base_time = nc.var('base_time').get()
time_offset = nc.var('time').get()
#base = datetime.datetime.fromtimestamp(base_time,tz=datetime.tzinfo("UTC"))
base = datetime.datetime(1970, 1, 1, 0, 0, 0) + datetime.timedelta(
seconds=int(base_time))
# adds a time offset (seconds) to the base time to make a datetime object
var = np.array([(base + datetime.timedelta(seconds=x))
for x in time_offset])
else:
var = nc.var(var_name).get()
pass
return var
def save_netcdf(self, flo, **kwargs):
"""Save the current dataset to the given file as a netCDF dataset to be
used with Alex Graves nnl_ndim program in
task="sequence classification" mode."""
# make sure classes are defined properly
assert len(self['class']) == len(self['target'])
if self.nClasses > 10:
raise
from pycdf import CDF, NC
# need to regenerate the file name
filename = flo.name
flo.close()
# Create the file. Raise the automode flag, so that
# we do not need to worry about setting the define/data mode.
d = CDF(filename, NC.WRITE | NC.CREATE | NC.TRUNC)
d.automode()
# Create 2 global attributes, one holding a string,
# and the other one 2 floats.
d.title = 'Sequential data exported from PyBrain (www.pybrain.org)'
# create the dimensions
dimsize = {
'numTimesteps': len(self),
'inputPattSize': self.indim,
'numLabels': self.nClasses,
'numSeqs': self.getNumSequences(),
'maxLabelLength': 2
}
dims = {}
for name, sz in dimsize.iteritems():
dims[name] = d.def_dim(name, sz)
# Create a netCDF record variables
inputs = d.def_var('inputs', NC.FLOAT,
(dims['numTimesteps'], dims['inputPattSize']))
targetStrings = d.def_var('targetStrings', NC.CHAR,
(dims['numSeqs'], dims['maxLabelLength']))
seqLengths = d.def_var('seqLengths', NC.INT, (dims['numSeqs']))
labels = d.def_var('labels', NC.CHAR,
(dims['numLabels'], dims['maxLabelLength']))
# Switch to data mode (automatic)
# assign float and integer arrays directly
inputs.put(self['input'].astype(single))
# strings must be written as scalars (sucks!)
for i in range(dimsize['numSeqs']):
targetStrings.put_1(i, str(self.getSequenceClass(i)))
for i in range(self.nClasses):
labels.put_1(i, str(i))
# need colon syntax for assigning list
seqLengths[:] = [
self.getSequenceLength(i) for i in range(self.getNumSequences())
]
# Close file
print("wrote netCDF file " + filename)
d.close()