def add_array_params(self, traj):
length = len(traj)
da_data = np.zeros(length, dtype=np.int)
traj.f_store(only_init=True)
traj.f_add_result(SharedResult, 'daarrays.a', SharedArray()).create_shared_data(obj=da_data)
traj.f_add_result(SharedResult, 'daarrays.ca', SharedCArray()).create_shared_data( obj=da_data)
traj.f_add_result(SharedResult, 'daarrays.ea', SharedEArray()).create_shared_data(shape=(0, 10),
atom=pt.FloatAtom(),
expectedrows=length)
traj.f_add_result(SharedResult, 'daarrays.vla', SharedVLArray()).create_shared_data(atom=pt.FloatAtom())
traj.f_add_result(SharedResult, 'tabs.t1', SharedTable()).create_shared_data(description={'idx': pt.IntCol(), 'run_name': pt.StringCol(30)},
expectedrows=length)
traj.f_add_result(SharedResult, 'tabs.t2', SharedTable()).create_shared_data(description={'run_name': pt.StringCol(300)})
traj.f_add_result(SharedResult, 'pandas.df', SharedPandasFrame())
traj.f_store()
with StorageContextManager(traj) as cm:
for run_name in self.traj.f_get_run_names():
row = traj.t2.row
row['run_name'] = run_name
row.append()
traj.t2.flush()
traj.t2.create_index('run_name')
def save_h5(filepath, complib, diff):
"""Read series of floating point number from filepath and save it
as hdf5. The extension is .h5 if diff is False. If diff is True
the file ends with _diff.h5
filepath -- path of the file to be read.
complib -- compression library, passed on to PyTables. Can be
zlib, lzo, gzip, bzip2.
diff -- save the original numbers or the first differences.
"""
data = numpy.loadtxt(filepath, dtype=float)
if diff:
new_file_path = '%s_diff_%s.h5' % (filepath, complib)
to_save = numpy.diff(data)
else:
new_file_path = '%s_%s.h5' % (filepath, complib)
to_save = data
fltr = tables.Filters(complevel=9, complib=complib)
new_file = tables.openFile(new_file_path, mode='w')
carray = new_file.createCArray(new_file.root,
'array',
tables.FloatAtom(),
numpy.shape(to_save),
filters=fltr)
if diff:
carray.start = float(data[0])
carray[:] = to_save
new_file.close()
return new_file_path
def write_regional_freqs(outfile, regions, reg_freqs, tree_freqs,
ninds_dict):
"""
Writes expected regional allele frequencies over all simulated trees,
as well as for each individual tree.
"""
## Filters for compression
filters = tables.Filters(complevel=5, complib='blosc')
with tables.open_file(outfile, 'w') as f:
## Write expected allele frequency for each tree for the region
for region, freqs in tree_freqs.iteritems():
ninds = ninds_dict[region]
## Suppress warning about naming convention
with warnings.catch_warnings():
warnings.simplefilter("ignore")
ca = f.create_carray(f.root, region, tables.FloatAtom(),
shape=tree_freqs[region].shape, filters=filters)
ca[:] = tree_freqs[region]
ca.attrs.ninds = ninds
## Convert array values to string for writing to character array
ninds_array = np.asarray([str(ninds_dict[r]) for r in regions])
reg_freqs = map(str, reg_freqs)
region_freqs_array = np.asarray(zip(regions, reg_freqs, ninds_array))
## Write total expected allele frequency per region
title = 'region\texpected_alleles\tnum_inds'
f.create_array(f.root, 'regional_expected_freqs', region_freqs_array,
title=title)
def subset_and_writeout(hf_in, fname, thin, maskval, binfn=lambda x:x):
print 'Subsetting for %s'%fname
res=5
hf_out = tb.openFile(os.path.join('5k-covariates',fname.replace('-','_').replace('.','_')+'.hdf5'),'w')
hf_out.createArray('/','lon',lon[lon_min_i:lon_max_i:res])
hf_out.createArray('/','lat',lat[lat_min_i:lat_max_i:res])
d = hf_in.root.data[(hf_in.root.data.shape[0]-lat_max_i*thin):\
(hf_in.root.data.shape[0]-lat_min_i*thin):\
thin,
lon_min_i*thin:\
lon_max_i*thin:\
thin]
d = map_utils.grid_convert(map_utils.grid_convert(d,'y-x+','x+y+')[::res,::res], 'x+y+','y-x+')
hf_out.createCArray('/','data',atom=tb.FloatAtom(),shape=d.shape,filters=tb.Filters(complevel=1,complib='zlib'))
hf_out.createCArray('/','mask',atom=tb.BoolAtom(),shape=d.shape,filters=tb.Filters(complevel=1,complib='zlib'))
hf_out.root.data.attrs.view = 'y-x+'
hf_out.root.data[:]=binfn(d)
hf_out.root.mask[:] = (d==maskval)+clipped_pete_mask
hf_out.close()
def generate_array(array_fname): file = tables.open_file(array_fname, mode='w') array = file.create_earray(file.root, 'data', tables.FloatAtom(itemsize=4), (0,)) for i in xrange(WRITE_CHUNKS): new_array = np.random.rand(ELEMENT_COUNT/WRITE_CHUNKS) array.append(new_array) del new_array file.close()
def test99a_nonIntEnum(self):
"""Describing an enumerated column of floats (not implemented)."""
colors = {'red': 1.0}
self.assertRaises(NotImplementedError,
self._createCol,
colors,
'red',
base=tables.FloatAtom())
def test03_carray(self):
"""Check dtype accessor for EArray objects."""
a = self.h5file.create_earray('/',
'array',
atom=tb.FloatAtom(),
shape=[0, 2])
self.assertEqual(a.dtype, a.atom.dtype)
def init_array(outfile, array_name):
""" Initialize extendable arrays to store control likelihoods """
with tables.open_file(outfile, 'w') as f:
title = 'Likelihood of tree having produced the observed allele freq'
f.create_earray(f.root,
array_name,
atom=tables.FloatAtom(),
title=title,
shape=(0, ))
def main(args):
pedfile = os.path.expanduser(args.pedfile)
climb_lik_file = os.path.expanduser(args.climb_lik_file)
control_lik_file = os.path.expanduser(args.control_lik_file)
hap_length = args.haplotype_length
print "Loading pedigree..."
P = ped.Pedigree(pedfile)
with tables.open_file(climb_lik_file, 'r') as f:
shape = f.root.liks.shape
haplotype_liks = np.zeros(shape)
for i, tree in enumerate(f.root.trees):
num_hidden_transmissions = get_num_hidden_transissions(P, tree)
scale = 1. / (len(tree) - num_hidden_transmissions)
haplotype_liks[i] = scipy.stats.erlang(2,
scale=scale).pdf(hap_length)
if i % 10000 == 0:
print "Calculating haplotype likelihoods for tree number", i, \
"of", len(haplotype_liks)
norm_hap_liks = np.log2(haplotype_liks / np.sum(haplotype_liks))
print "Writing combined likelihoods to file..."
with tables.open_file(control_lik_file, 'a') as hapfile:
tot_hap_liks = hapfile.create_carray(
hapfile.root,
'tot_hap_liks',
tables.FloatAtom(),
shape=shape,
title='Total tree likelihoods, including haplotype length')
haps = hapfile.create_carray(hapfile.root, 'haplotype_liks',
tables.FloatAtom(), shape=shape,
title='Likelihood of observed haplotype length:' + \
str(hap_length) + 'Morgans')
haps[:] = norm_hap_liks
tot_liks = hapfile.root.tot_liks[:]
t = norm_hap_liks + tot_liks
tot_hap_liks[:] = norm_hap_liks + tot_liks
def __init__(self, name, fname, spoints=64, destination=''):
dirname = os.path.join(destination, name)
if not os.path.isdir(dirname):
os.mkdir(dirname)
fname = os.path.join(dirname, fname)
f = tables.open_file(fname, 'w')
f.create_group('/', 'pos', 'positive spikes')
f.create_group('/', 'neg', 'negative spikes')
for sign in ('pos', 'neg'):
f.create_earray('/' + sign, 'spikes', tables.Float32Atom(),
(0, spoints))
f.create_earray('/' + sign, 'times', tables.FloatAtom(), (0, ))
f.create_earray('/', 'thr', tables.FloatAtom(), (0, 3))
self.f = f
print('Initialized ' + fname)
def init_output(outfile):
"""
Creates a timestamped output directory and writes header to output file.
"""
with tables.open_file(outfile, 'w') as f:
## Create extendable arrays so we can incrementally write output
f.create_earray(f.root, 'ancs', atom=tables.IntAtom(), shape=(0, ))
f.create_earray(f.root, 'liks', atom=tables.FloatAtom(), shape=(0, ))
## Trees, which are variable-length, must be added individually
f.create_vlarray(f.root, 'trees', atom=tables.IntAtom())
f.create_vlarray(f.root, 'genotypes', atom=tables.IntAtom())
def setup_h5(self,):
filename = self.filenames[0]
x = read_wav(filename)
spec_x = calc_specgram(x,22050,1024)
spec_x = make_4tensor(spec_x)
self.data_shape = spec_x.shape[1:]
self.x_earray_shape = (0,) + self.data_shape
self.chunkshape = (1,) + self.data_shape
self.h5_x = self.h5.createEArray('/','x',tables.FloatAtom(itemsize=4),self.x_earray_shape,chunkshape=self.chunkshape,expectedrows=self.num_files)
self.h5_filenames = self.h5.createEArray('/','filenames',tables.StringAtom(256),(0,),expectedrows=self.num_files)
self.h5_x.append(spec_x)
self.h5_filenames.append([filename])
def create_examples(self, timesteps, nfea, species_names):
"""Creates file to store classification examples.
"""
path = os.path.join(self.data_dir, "examples.h5")
h5file = tables.open_file(path, 'w')
for species in species_names:
h5file.create_earray('/',
species,
atom=tables.FloatAtom(),
shape=(0, timesteps, nfea),
expectedrows=100000)
h5file.close()
def create_db(filename, params, total_env_count=None, traj_per_env=None):
"""
:param filename: file name for database
:param params: dotdict describing the domain
:param total_env_count: total number of environments in the dataset (helps to preallocate space)
:param traj_per_env: number of trajectories per environment
"""
N = params.grid_n
M = params.grid_m
num_state = N * M
if total_env_count is not None and traj_per_env is not None:
total_traj_count = total_env_count * traj_per_env
else:
total_traj_count = 0
if os.path.isfile(filename):
print (filename + " already exitst, opening.")
return tables.open_file(filename, mode='a')
db = tables.open_file(filename, mode='w')
db.create_earray(db.root, 'envs', tables.IntAtom(), shape=(0, N, M), expectedrows=total_env_count)
db.create_earray(db.root, 'expRs', tables.FloatAtom(), shape=(0, ), expectedrows=total_traj_count)
db.create_earray(db.root, 'valids', tables.IntAtom(), shape=(0, ), expectedrows=total_traj_count)
db.create_earray(db.root, 'bs', tables.FloatAtom(), shape=(0, num_state), expectedrows=total_traj_count)
db.create_earray(db.root, 'steps', tables.IntAtom(),
shape=(0, 3), # state, action, observation
expectedrows=total_traj_count * 10) # rough estimate
db.create_earray(db.root, 'samples', tables.IntAtom(),
shape=(0, 6), # env_id, goal_state, step_id, traj_length, collisions, failed
expectedrows=total_traj_count)
db.create_earray(db.root, 'qmdpBeliefs', tables.FloatAtom(), shape=(0, num_state,),expectedrows=total_traj_count*10)
return db
def create_examples(self, timesteps, nfea):
"""Creates file to store spatiotemporal examples.
"""
path = os.path.join(self.data_dir, "examples.h5")
h5file = tables.open_file(path, 'w')
for side in ['head', 'tail']:
group = h5file.create_group('/', side)
h5file.create_earray(group,
'entering',
atom=tables.FloatAtom(),
shape=(0, timesteps, nfea),
expectedrows=10000)
h5file.create_earray(group,
'exiting',
atom=tables.FloatAtom(),
shape=(0, timesteps, nfea),
expectedrows=1000)
h5file.create_earray(group,
'ignore',
atom=tables.FloatAtom(),
shape=(0, timesteps, nfea),
expectedrows=10000)
h5file.close()
def initialize_database(self, **kargs):
"""
Initializes the data_file.
:param kargs: Can pass in 'n_points': Maximum number of data points for an event to be added.
"""
filters = tb.Filters(complib='blosc', complevel=4)
shape = (kargs['n_points'],)
a = tb.FloatAtom()
if not 'data' in self.root:
self.createCArray(self.root, 'data', a, shape=shape, title='Data', filters=filters)
# set the attributes
self.root.data.attrs.sample_rate = kargs['sample_rate']
def get_pics_input(folder, input_train, filename):
img_dtype = tables.FloatAtom()
data_shape = (0, 256, 256, 3)
hdf5_file = tables.open_file('{}.hdf5'.format(filename), mode='w')
storage = hdf5_file.create_earray(hdf5_file.root,
'images',
img_dtype,
shape=data_shape)
for i in range(input_train.shape[0]):
print(i)
filename = '{}/{}'.format(folder, input_train[i][0])
img = cv2.imread(filename) / 255.0
img = cv2.resize(img, (256, 256), cv2.INTER_LINEAR)
# face = extract_face(img, required_size=(256,256))
storage.append(img[None])
hdf5_file.close()
def save(fname, Y):
'''
WARNING: "spm1d.io.save" is deprecated and will be removed from future versions of spm1d.
'''
_check4pytables()
import tables
### create file (existing file will be overwritten)
fid = tables.openFile(fname, mode='w')
try:
### write data:
atom = tables.FloatAtom()
filter0 = tables.Filters(complevel=5, complib='zlib')
CA = fid.createCArray(fid.root, 'Y', atom, Y.shape, filters=filter0)
CA[:] = Y
fid.close()
except IOError:
fid.close()
raise IOError('Error saving file.')
def bin_write(onebin):
binid = bid2i[onebin]
#pdb.set_trace()
if not "/" + binid in drugbins:
ncol = tabcache[onebin][0].shape[1]
drugbins.create_earray(drugbins.root,
binid,
tables.FloatAtom(),
shape=(0, ncol),
chunkshape=(50, ncol))
#outcome_tab.create_vlarray(outcome_tab.root, binid, tables.Int32Atom())
#pdb.set_trace()
if True: #drugbins.get_node("/" + binid).shape[0] < 50000:
to_write = np.vstack(tabcache[onebin])
drugbins.get_node("/" + binid).append(to_write)
'''
with open("tmp/e" + savename + binid,'ab') as f:
np.savetxt(f, to_write)
for i in outcomecache[onebin]:
outcome_tab.get_node("/" + binid).append(list(i))
'''
#if to_write.shape[1] > 6:
# pdb.set_trace()
if to_write.shape[1] > 6:
scaler.partial_fit(to_write[:,
6:]) ###patid, drug, [4-d bin] = 6
with open(tmp + binid, 'a') as f:
f.write("\n".join(
[json.dumps(list(i))
for i in outcomecache[onebin]]) + '\n')
#elif binid not in overflowlist:
# print("exceeded 50000 for " + binid)
# overflowlist.append(binid)
tabcacheCt[onebin] = 0
tabcache[onebin] = []
outcomecache[onebin] = []
def save_data_h5(self, filename, notes=''):
"""Save all the data tables in an hdf5 file.
The structure in the hdf5 file mirrors the structure in the
model under /data element - but only two levels deep."""
config.LOGGER.debug('Start saving the data')
starttime = datetime.now()
compression_filter = tables.Filters(complevel=9,
complib='zlib',
fletcher32=True)
h5file = tables.openFile(
filename,
mode='w',
title='Traub Network: timestamp: %s' %
(config.timestamp.strftime('%Y-%m-%d %H:%M:%S')),
filters=compression_filter)
h5file.root._v_attrs.simtime = self.simtime
h5file.root._v_attrs.simdt = self.simdt
h5file.root._v_attrs.plotdt = self.plotdt
h5file.root._v_attrs['notes'] = notes
h5file.root._v_attrs.timestamp = config.timestamp.strftime(
'%Y-%m-%d %H:%M:%S')
# Save simulation configuration data. I am saving it both in
# data file as well as network file as often the data file is
# too large and may not be available if the simulation is
# cancelled midway.
runconfig = h5file.createGroup(h5file.root, 'runconfig',
'Simulation settings')
for section in config.runconfig.sections():
table_data = config.runconfig.items(section)
if table_data:
sectiontab = h5file.createTable(runconfig, section, table_data)
for child_id in self.data.children():
child = moose.Neutral(child_id)
if child.className == 'Neutral':
group = h5file.createGroup(h5file.root, child.name)
for tab_id in child.children():
tab = moose.Table(tab_id)
print 'Saving', tab.path, 'of length:', len(tab)
if tab.className == 'Table':
carray = h5file.createCArray(group,
tab.name,
tables.FloatAtom(),
shape=(len(tab), ))
carray[:] = numpy.array(tab)
elif child.className == 'Table':
tab = moose.Table(child_id)
carray = h5file.createCArray(h5file.root,
tab.name,
tables.FloatAtom(),
shape=(len(tab), ))
carray[:] = numpy.array(tab)
# NOTE: I found that saving the first value with first
# forwards diffs saves a lot of space on
# compression. But will go with direct method now to
# see if it is worth it.
else:
raise Warning(
'Element %s of unhandled type %s under data element: -- not known for saving.'
% (tab.name, tab.className))
h5file.close()
endtime = datetime.now()
delta = endtime - starttime
config.BENCHMARK_LOGGER.info(
'Saved data to %s: %g s' %
(filename,
delta.days * 86400 + delta.seconds + delta.microseconds * 1e-6))
freq_files = sorted([line.strip() for line in open(freq_file_paths)])
with tables.open_file(freq_files[0], 'r') as f:
nodes = [
n for n in f.walk_nodes()
if n._v_name != '/' and n._v_name != 'regional_expected_freqs'
]
region_names = [n._v_name for n in nodes]
region_sizes = [n._v_attrs['ninds'] for n in nodes]
with tables.open_file(freq_merge_file, 'w') as f:
node_dict = {}
for name, size in zip(region_names, region_sizes):
node_dict[name] = tables.EArray(f.root,
name,
tables.FloatAtom(),
shape=(0, ))
node_dict[name]._v_attrs['ninds'] = size
for fname in freq_files:
print("Merging", fname)
with tables.open_file(fname, 'r') as g:
for name in region_names:
print("Merging", name)
new_node = g.get_node(g.root, name)
node_dict[name].append(new_node[:])
# with tables.open_file(climb_merge_file, 'w') as climb_merge_file:
# ancs = tables.EArray(climb_merge_file.root, 'ancs', tables.IntAtom(), shape=(0,))
# genotypes = tables.VLArray(climb_merge_file.root, 'genotypes', tables.IntAtom())
# liks = tables.EArray(climb_merge_file.root, 'liks', tables.IntAtom(), shape=(0,))
# Append several rows with default values
for i in range(10):
table.row.append()
table.flush()
# create new arrays
atom1 = tables.IntAtom()
shape1 = (2, 10, 10, 1)
filters1 = tables.Filters(complevel=1)
#(2, 10, 10, 3)
array1 = fileh.create_carray(fileh.root,
'array1',
atom1,
shape1,
filters=filters1)
atom2 = tables.FloatAtom()
shape2 = (2, 10, 10, 3, 1)
filters2 = tables.Filters(complevel=1)
#(2, 10, 10, 3, 200)
array2 = fileh.create_carray(fileh.root,
'array2',
atom2,
shape2,
filters=filters2)
# Add multimensional attributes to the objects
# Integers will go in /table
table.attrs.MD1 = numpy.arange(5, dtype="int8")
table.attrs.MD2 = numpy.arange(10, dtype="int64").reshape(2, 5)
# Complex will go in /array1
climb_lik_file = '/RQusagers/dnelson/project/anc_finder/results/BALSAC/CAID_3M_all_anc_out.csv'
max_trees = 1000
print "Loading climbing likelihoods"
climb_liks = np.genfromtxt(climb_lik_file,
skip_header=True,
delimiter=',',
usecols=[1])[:max_trees]
print "Loading trees"
trees = [line.strip() for line in open(tree_anc_file, 'r')][:max_trees]
climb_outfile = os.path.expanduser('~/temp/CAID_climb_1000.h5')
with tables.open_file(climb_outfile, 'w') as f:
## Create extendable arrays so we can incrementally write output
f.create_earray(f.root, 'liks', atom=tables.FloatAtom(), shape=(0, ))
f.create_earray(f.root, 'ancs', atom=tables.FloatAtom(), shape=(0, ))
## Trees, which are variable-length, must be added individually
f.create_vlarray(f.root, 'trees', atom=tables.IntAtom())
f.create_vlarray(f.root, 'genotypes', atom=tables.IntAtom())
incremental_write(climb_liks, trees, climb_outfile)
## Store control likelihoods
# control_outfile = os.path.expanduser('~/temp/CAID_control.h5')
#
# init_array(control_outfile, array_name='control_liks')
# with tables.open_file(control_outfile, 'a') as f:
# f.root.control_liks.append([np.log2(conv_prob)])
def main():
# Input variables
# ##########################################################################
# Start and end year
start_year = 1997
end_year = 2012
# Aggregations ('A' : annual, 'M' : monthly)
agg = ['A', 'M']
# Number of nan values in daily resolution for each aggregation
nan_max = [25, 2]
# plot variogram : True or False
save_plots = True
# Input and output files
# ##########################################################################
# daily precipitation data
fn_precip = os.path.join(basepath, '03_Data', '1781_2016_daily.h5')
## fn_precip = os.path.join(basepath, '03_Data','02_data_cross_validation',
## 'set_a', '1781_2016_daily.h5')
# dem germany
fn_dem = os.path.join(basepath, '02_Import', '03_QGIS_project_germany',
'xyz_germany_utm.dat')
# smoothed elevation of gauges
fn_sme_gauges = os.path.join(basepath, '05_Smoothed_elevation',
'smoothed_elevation_gauges.h5')
# smoothed elevation grid
fn_sme_grid = os.path.join(basepath, '05_Smoothed_elevation',
'smoothed_elevation_ger.h5')
outpath = os.path.join(basepath, '04_Research', '00_Interpolation',
'01_results', '01_data', 'set_orig')
outpath_misc = os.path.join(outpath, 'misc')
if not os.path.exists(outpath_misc): os.makedirs(outpath_misc)
# output file name (with EDK interpolated precipitation values)
fn_month = os.path.join(
outpath,
r'monthly_precip_interpolated_{}_{}.h5'.format(start_year, end_year))
fn_year = os.path.join(
outpath,
r'yearly_precip_interpolated_{}_{}.h5'.format(start_year, end_year))
# Load data
# ##########################################################################
print('reading data')
# dem germany
dem = np.loadtxt(fn_dem, delimiter=';').astype(int)
## ids =['00001', '00002', '00003', '00004', '00006', '00007', '00008',
## '00009', '00010', '00012', '00013', '00014', '00015', '00016',
#### '00017', '00018', '00019', '00020', '00021', '00022', '00023',
#### '00024', '00025', '00026', '00027', '00028', '00029', '00030',
#### '00031', '00032', '00033', '00034', '00035', '00036', '00037',
#### '00038', '00039', '00040', '00041', '00042', '00043', '00044',
## '00045', '00046', '00047', '00048', '00050', '00051', '00052',
## '00053']
# get data with the SYNOPSE II functions ;-)
data_daily = z_functions.Get_Daily_Data(
fn_precip,
start_year,
end_year ##, ids = ids
)
# meta data
ts_iso = data_daily.df.index.map(
lambda x: datetime.datetime.strftime(x, '%Y-%m-%dT%H:%M:%S'))
ts_year = data_daily.df.index.year
# Preprocessing
# ##########################################################################
print 'aggregating data'
for iagg, inan_max in zip(agg, nan_max):
data_daily.get_filtered_timeseries(iagg, inan_max)
# Isoformat transformation
ts_month = data_daily.data['M'].index.map(
lambda x: datetime.datetime.strftime(x, '%Y-%m-%dT%H:%M:%S'))
ts_year = data_daily.data['A'].index.map(
lambda x: datetime.datetime.strftime(x, '%Y-%m-%dT%H:%M:%S'))
# Create output hdfs
# ##########################################################################
print 'create hdf output files'
# Monthly
# --------------------------------------------------------------------------
hf = tables.open_file(fn_month, 'w', filters=tables.Filters(complevel=6))
# Write kriged data to file
hf.create_carray(where=hf.root,
name='monthly_inter',
atom=tables.FloatAtom(dflt=np.nan),
shape=(dem.shape[0], ts_month.shape[0]),
chunkshape=(dem.shape[0], 1),
title=('Interpolated monthly rainfall values (IDW).'))
# Write metadata to hdf file
# Timestamps
hf.create_carray(where=hf.root,
name='ts_iso',
atom=tables.StringAtom(itemsize=19, dflt=''),
shape=ts_month.shape,
chunkshape=ts_month.shape,
title='Timestamps in isoformat.')
hf.root.ts_iso[:] = ts_month
# xyz values
hf.create_group(where=hf.root,
name='coord_utm',
title=('UTM-coordinates for a 5 km grid of Germany.'))
hf.create_carray(where=hf.root.coord_utm,
name='x',
atom=tables.Int64Atom(dflt=-99),
shape=(dem.shape[0], ),
chunkshape=(dem.shape[0], ),
title='UTM-coordinates in x direction.')
hf.create_carray(where=hf.root.coord_utm,
name='y',
atom=tables.Int64Atom(dflt=-99),
shape=(dem.shape[0], ),
chunkshape=(dem.shape[0], ),
title='UTM-coordinates in y direction.')
hf.root.coord_utm.x[:] = dem[:, 0]
hf.root.coord_utm.y[:] = dem[:, 1]
# Close hdf file
hf.close()
# Yearly
# --------------------------------------------------------------------------
hf = tables.open_file(fn_year, 'w', filters=tables.Filters(complevel=6))
# Write kriged data to file
hf.create_carray(where=hf.root,
name='yearly_inter',
atom=tables.FloatAtom(dflt=np.nan),
shape=(dem.shape[0], ts_year.shape[0]),
chunkshape=(dem.shape[0], 1),
title=('Interpolated yearly rainfall values (IDW).'))
# Write metadata to hdf file
# Timestamps
hf.create_carray(where=hf.root,
name='ts_iso',
atom=tables.StringAtom(itemsize=19, dflt=''),
shape=ts_year.shape,
chunkshape=ts_year.shape,
title='Timestamps in isoformat.')
hf.root.ts_iso[:] = ts_year
# xyz values
hf.create_group(where=hf.root,
name='coord_utm',
title=('UTM-coordinates for a 5 km grid of Germany.'))
hf.create_carray(where=hf.root.coord_utm,
name='x',
atom=tables.Int64Atom(dflt=-99),
shape=(dem.shape[0], ),
chunkshape=(dem.shape[0], ),
title='UTM-coordinates in x direction.')
hf.create_carray(where=hf.root.coord_utm,
name='y',
atom=tables.Int64Atom(dflt=-99),
shape=(dem.shape[0], ),
chunkshape=(dem.shape[0], ),
title='UTM-coordinates in y direction.')
hf.root.coord_utm.x[:] = dem[:, 0]
hf.root.coord_utm.y[:] = dem[:, 1]
# Close hdf file
hf.close()
# Preprocessing
# ##########################################################################
# ##########################################################################
# smoothed elevation of gauges
hdf = tables.open_file(fn_sme_gauges, 'r')
# smoothing radii
se_radii = hdf.root.sm_radii[:]
# smoothing vector
se_vector = hdf.root.sm_vec[:]
# gauges for smoothed elevation arrays
se_ids = hdf.root.id[:]
bool_ids = np.in1d(se_ids, data_daily.ids)
# smoothed elevation
se_sme = hdf.root.sme[:][bool_ids]
se_ids = hdf.root.id[:][bool_ids]
hdf.close()
# Calculate correlations with smoothed elevation
# ##########################################################################
print 'find smoothed elevation'
corr_month = np.full(
(ts_month.shape[0], se_vector.shape[0], se_radii.shape[0]),
fill_value=np.nan)
corr_year = np.full(
(ts_year.shape[0], se_vector.shape[0], se_radii.shape[0]),
fill_value=np.nan)
# monthly precipitation
# ---------------------
for ivec, vec in enumerate(se_vector):
for iradii, radii in enumerate(se_radii):
for imonth, month in enumerate(ts_month):
# finite values
fin_bool = np.isfinite(data_daily.data['M'].values[imonth])
# calculate correlation coefficient : smoothed elevation and
# precipitation
corr_month[imonth, ivec, iradii] = (np.corrcoef(
se_sme[fin_bool, ivec, iradii],
data_daily.data['M'].values[imonth, fin_bool])[0, 1])
# mean monthly correlations (mean over months)
meancorr_month = np.mean(corr_month, axis=0)
# find smoothing vector and radius for maximum correlation
idxs_maxmonth = np.where(meancorr_month == np.max(meancorr_month))
print 'monthly: {} / {} km'.format(se_vector[idxs_maxmonth[0]][0],
se_radii[idxs_maxmonth[1]][0])
# smoothed elevation gauges
smgauges_month = se_sme[:, idxs_maxmonth[0], idxs_maxmonth[1]]
# smoothed grid elevation for monthly precipitation
hdf = tables.open_file(fn_sme_grid, 'r')
smgrid_month = hdf.root.sme[:, idxs_maxmonth[0],
idxs_maxmonth[1]].flatten()
hdf.close()
# save information to file
with open(os.path.join(outpath_misc, 'monthly_se_info.txt'), 'w') as f:
f.write('direction: {}\n'.format(se_vector[idxs_maxmonth[0]][0]))
f.write('radius: {}'.format(se_radii[idxs_maxmonth[1]][0]))
# yearly precipitation
# --------------------
for ivec, vec in enumerate(se_vector):
for iradii, radii in enumerate(se_radii):
for iyear, year in enumerate(ts_year):
# finite values
fin_bool = np.isfinite(data_daily.data['A'].values[iyear])
# calculate correlation coefficient : smoothed elevation and
# precipitation
corr_year[iyear, ivec, iradii] = (np.corrcoef(
se_sme[fin_bool, ivec, iradii],
data_daily.data['A'].values[iyear, fin_bool])[0, 1])
# mean yearly correlations (mean over years)
meancorr_year = np.mean(corr_year, axis=0)
# find smoothing vector and radius for maximum correlation
idxs_maxyear = np.where(meancorr_year == np.max(meancorr_year))
print 'yearly: {} / {} km'.format(se_vector[idxs_maxyear[0]][0],
se_radii[idxs_maxyear[1]][0])
# smoothed elevation gauges
smgauges_year = se_sme[:, idxs_maxyear[0], idxs_maxyear[1]]
# smoothed grid elevation for yearly precipitation
hdf = tables.open_file(fn_sme_grid, 'r')
smgrid_year = hdf.root.sme[:, idxs_maxyear[0], idxs_maxyear[1]].flatten()
hdf.close()
# save information to file
with open(os.path.join(outpath_misc, 'yearly_se_info.txt'), 'w') as f:
f.write('direction: {}\n'.format(se_vector[idxs_maxyear[0]][0]))
f.write('radius: {}'.format(se_radii[idxs_maxyear[1]][0]))
# Variograms
# ##########################################################################
print 'estimate variograms'
# Set variogram
vario = vf.Variogram()
# width of distance classes of the variogram (5 km)
vario.setwidthclasses(5000.)
# empirical variogram array
distances = np.arange(2500., 102500., 5000.)
empvarios_month = np.full((distances.shape[0], ts_month.shape[0]),
fill_value=np.nan)
# bounds of the variogram parameters for the variogram fit
bounds = {
'spherical range': [5000., 100000.],
'spherical sill': [0., 100000.],
'exponential range': [5000., 100000.],
'exponential sill': [0., 100000.],
'gauss range': [5000., 100000.],
'gauss sill': [0., 100000.]
}
# monthly precipitation
# --------------------------------------------------------------------------
print 'monthly precip'
# calculate empirical variograms for each month
for imonth, month in enumerate(ts_month):
##print imonth
# finite values
fin_bool = np.isfinite(data_daily.data['M'].values[imonth])
# empirical variograms
vario.setxcoord(data_daily.x[fin_bool].values)
vario.setycoord(data_daily.y[fin_bool].values)
vario.setdata(data_daily.data['M'].values[imonth][fin_bool])
vario.calc_expervar()
# for which distances
bool_dist = np.in1d(vario.expervar[0], distances)
empvarios_month[bool_dist, imonth] = vario.expervar[1]
# fit theoretical variogram to every month of the year
variomod_month = pd.Series().astype(str)
for year_month in range(1, 13):
print year_month
# varios months belonging to the year_month
empvarios = empvarios_month[:, data_daily.data['M'].index.month ==
year_month]
# mean vario
meanvario = np.nanmean(empvarios, axis=1)
# fit variogram
dist_vario = np.vstack((distances, meanvario))
nb_dec, sillbound = 2, 100000.
variomod = vf.find_bestvario_expervar(
dist_vario,
bounds,
nb_dec,
sillbound,
variomods=['exponential', 'gauss', 'spherical'])
# calculate theoretical variogram values
plotdistances = np.arange(0., distances[-1] + 2500., 100.)
vario = vf.Variogram()
vario.name = variomod
vario.get_params_name()
vario.get_theovartype_name()
vario.calc_theovar(plotdistances)
variomod_month[str(year_month)] = variomod
# Control plot
if save_plots:
plt.title('monthly variogram / month: {}'.format(year_month))
plt.plot(plotdistances, vario.theovar, 'r-', lw=3)
plt.plot(distances, meanvario, 'kx')
plt.tight_layout()
plt.savefig(
os.path.join(outpath_misc,
'monthly_vario_{}.png'.format(year_month)))
plt.close()
variomod_month.to_csv(os.path.join(outpath_misc, 'monthly_vario_fit.csv'),
sep=';')
# read the data to have it in a consistent structure compared to other
# scripts
variomod_month = pd.read_csv(os.path.join(outpath_misc,
'monthly_vario_fit.csv'),
sep=';',
header=None,
index_col=0)[1]
# yearly precipitation
# --------------------------------------------------------------------------
# Set variogram
vario = vf.Variogram()
# width of distance classes of the variogram (5 km)
vario.setwidthclasses(5000.)
empvarios_year = np.full((distances.shape[0], ts_year.shape[0]),
fill_value=np.nan)
print 'yearly precip'
# calculate empirical variograms for each year
for iyear, year in enumerate(ts_year):
print iyear
# finite values
fin_bool = np.isfinite(data_daily.data['A'].values[iyear])
# empirical variograms
vario.setxcoord(data_daily.x[fin_bool].values)
vario.setycoord(data_daily.y[fin_bool].values)
vario.setdata(data_daily.data['A'].values[iyear][fin_bool])
vario.calc_expervar()
# for which distances
bool_dist = np.in1d(vario.expervar[0], distances)
empvarios_year[bool_dist, iyear] = vario.expervar[1]
# fit theoretical variogram to every month of the year
variomod_year = pd.Series().astype(str)
for iyear, year in enumerate(data_daily.data['A'].index.year):
# varios for every year
empvarios = empvarios_year[:, iyear]
# fit variogram
dist_vario = np.vstack((distances, empvarios))
nb_dec, sillbound = 2, 100000.
variomod = vf.find_bestvario_expervar(
dist_vario,
bounds,
nb_dec,
sillbound,
variomods=['exponential', 'gauss', 'spherical'])
# calculate theoretical variogram values
plotdistances = np.arange(0., distances[-1] + 2500., 100.)
vario = vf.Variogram()
vario.name = variomod
vario.get_params_name()
vario.get_theovartype_name()
vario.calc_theovar(plotdistances)
variomod_year[str(year)] = variomod
# Control plot
if save_plots:
plt.title('yearly variogram / year: {}'.format(year))
plt.plot(plotdistances, vario.theovar, 'r-', lw=3)
plt.plot(distances, empvarios, 'kx')
plt.tight_layout()
plt.savefig(
os.path.join(outpath_misc, 'yearly_vario_{}.png'.format(year)))
plt.close()
variomod_year.to_csv(os.path.join(outpath_misc, 'yearly_vario_fit.csv'),
sep=';')
# read the data to have it in a consistent structure compared to other
# scripts
variomod_year = pd.read_csv(os.path.join(outpath_misc,
'yearly_vario_fit.csv'),
sep=';',
header=None,
index_col=0)[1]
# Interpolation
# ##########################################################################
# ##########################################################################
print 'monthly interpolation'
# external drift kriging
kriging = kf.Kriging()
# loop over timesteps
for its, ts in enumerate(ts_month):
print 'its: {} / {}'.format(its + 1, ts_month.shape[0])
# Select only finite values
mask_precip = np.isfinite(data_daily.data['M'].values[its])
precip_ts = data_daily.data['M'].values[its, mask_precip]
xy = np.vstack(
(data_daily.x[mask_precip], data_daily.y[mask_precip])).T
# get variogram of the month and set the variogram for kriging
variomod = variomod_month[data_daily.data['M'].index.month[its]]
vario = vf.Variogram()
vario.name = variomod
vario.get_params_name()
vario.get_theovartype_name()
# Get external drift of gauges
ed_gauges = (smgauges_month.flatten())[mask_precip]
## # control plot
## plt.scatter(dem[:, 0], dem[:, 1], c=smgrid_month,
## lw=0, s=20, marker='o')
## plt.scatter(data_daily.x, data_daily.y, c=smgauges_month,
## lw=0, s=20, marker='s')
## plt.show()
## plt.close()
# controls
kriging.setcontrols(xy)
kriging.setcontrolvalues(precip_ts)
kriging.setcontrols_ed(ed_gauges)
# targets
kriging.settargets(dem[:, :2])
kriging.settargets_ed(smgrid_month)
kriging.krige_values(vario,
method='external drift kriging',
min_stations=10,
positive_weights='no')
# do not allow for negative values
target_values = kriging.kriged_values
# checking for error values (negative values)
target_values[target_values < 0.] = 0.
print kriging.kriging_weights[0]
print target_values[0]
# control plot
plt.scatter(dem[:, 0],
dem[:, 1],
c=target_values,
lw=0,
s=20,
marker='o')
plt.scatter(xy[:, 0], xy[:, 1], c=precip_ts, lw=0, s=20, marker='s')
# plt.show()
plt.savefig('montly_interpolation.png', dpi=600)
plt.close()
# Write interpolated data to hdf file
hf = tables.open_file(fn_month, 'r+')
hf.root.monthly_inter[:, its] = target_values
hf.close()
# ##########################################################################
print 'yearly interpolation'
# loop over timesteps
for its, ts in enumerate(ts_year):
print 'its: {} / {}'.format(its + 1, ts_year.shape[0])
# Select only finite values
mask_precip = np.isfinite(data_daily.data['A'].values[its])
precip_ts = data_daily.data['A'].values[its, mask_precip]
xy = np.vstack(
(data_daily.x[mask_precip], data_daily.y[mask_precip])).T
# get variogram of the year and set the variogram for kriging
variomod = variomod_year[data_daily.data['A'].index.year[its]]
vario = vf.Variogram()
vario.name = variomod
vario.get_params_name()
vario.get_theovartype_name()
# Get external drift of gauges
ed_gauges = (smgauges_year.flatten())[mask_precip]
# external drift kriging
kriging = kf.Kriging()
# controls
kriging.setcontrols(xy)
kriging.setcontrolvalues(precip_ts)
kriging.setcontrols_ed(ed_gauges)
# targets
kriging.settargets(dem[:, :2])
kriging.settargets_ed(smgrid_year)
kriging.krige_values(vario,
method='external drift kriging',
min_stations=10,
positive_weights='no')
# do not allow for negative values
target_values = kriging.kriged_values
# checking for error values (negative values)
target_values[target_values < 0.] = 0.
# control plot
plt.scatter(dem[:, 0],
dem[:, 1],
c=target_values,
lw=0,
s=20,
marker='o')
plt.scatter(xy[:, 0], xy[:, 1], c=precip_ts, lw=0, s=20, marker='s')
plt.savefig('yearly_interpolation.png', dpi=600)
# plt.show()
plt.close()
# Write interpolated data to hdf file
hf = tables.open_file(fn_year, 'r+')
hf.root.yearly_inter[:, its] = target_values
hf.close()
def createCarray(file, path, name, shape, atom=tables.FloatAtom()):
file.create_carray(path, name, atom, shape)
def initialize_database(self, **kargs):
"""
Initializes the EventDatabase. Adds a group 'events' with
table 'eventsTable' and matrices 'raw_data', 'levels', and 'level_lengths'.
:param kargs: Dictionary - includes:
-maxEventLength: Maximum number of datapoints for an event to be added.
"""
if 'maxEventLength' in kargs:
if kargs['maxEventLength'] > self.max_event_length:
self.max_event_length = kargs['maxEventLength']
if 'events' not in self.root:
self.createGroup(self.root, 'events', 'Events')
if not 'eventTable' in self.root.events:
self.createTable(self.root.events, 'eventTable', _Event,
'Event parameters')
self.event_row = None
filters = tb.Filters(complib='blosc', complevel=4)
shape = (0, self.max_event_length)
a = tb.FloatAtom()
b = tb.IntAtom()
if not 'raw_data' in self.root.events:
self.createEArray(self.root.events,
'raw_data',
a,
shape=shape,
title="Raw data points",
filters=filters)
if not 'levels' in self.root.events:
self.createEArray(self.root.events,
'levels',
a,
shape=shape,
title="Cusum levels",
filters=filters)
if not 'level_lengths' in self.root.events:
self.createEArray(self.root.events,
'level_lengths',
b,
shape=shape,
title="Lengths of the cusum levels",
filters=filters)
# Create/init the debug group if needed.
if 'debug' in kargs and kargs['debug']:
if not 'debug' in self.root:
self.createGroup(self.root, 'debug', 'Debug')
debug_shape = (kargs['n_channels'], kargs['n_points'])
if not 'data' in self.root.debug:
self.createCArray(self.root.debug,
'data',
a,
shape=debug_shape,
title="Raw data",
filters=filters)
if not 'baseline' in self.root.debug:
self.createCArray(self.root.debug,
'baseline',
a,
shape=debug_shape,
title="Baseline data",
filters=filters)
if not 'threshold_positive' in self.root.debug:
self.createCArray(self.root.debug,
'threshold_positive',
a,
shape=debug_shape,
title="Raw data",
filters=filters)
if not 'threshold_negative' in self.root.debug:
self.createCArray(self.root.debug,
'threshold_negative',
a,
shape=debug_shape,
title="Raw data",
filters=filters)
def test04_vlarray(self):
"""Check dtype accessor for VLArray objects."""
a = self.h5file.create_vlarray('/', 'array', tb.FloatAtom())
self.assertEqual(a.dtype, a.atom.dtype)
def scan(self):
# Output data structures
scan_parameter_values = self.scan_parameters.PlsrDAC
shape=(len(scan_parameter_values), self.max_data_index)
atom = tb.FloatAtom()
data_out = self.raw_data_file.h5_file.create_carray(self.raw_data_file.h5_file.root, name='PlsrDACwaveforms', title='Waveforms from transient PlsrDAC calibration scan', atom=atom, shape=shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
shape=(self.max_data_index,)
atom = tb.FloatAtom()
time_out = self.raw_data_file.h5_file.create_carray(self.raw_data_file.h5_file.root, name='Times', title='Time values', atom=atom, shape=shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
data_out.attrs.scan_parameter_values = scan_parameter_values
data_out.attrs.enable_double_columns = self.enable_double_columns
data_out.attrs.fit_ranges = self.fit_ranges
data_out.attrs.trigger_level_offset = self.trigger_level_offset
trigger_levels = []
progress_bar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA()], maxval=len(scan_parameter_values), term_width=80)
progress_bar.start()
for index, scan_parameter_value in enumerate(scan_parameter_values):
if self.stop_run.is_set():
break
# Update PlsrDAC parameter
self.set_scan_parameters(PlsrDAC=scan_parameter_value) # set scan parameter
self.write_global_register('PlsrDAC', scan_parameter_value) # write to FE
self.dut['Oscilloscope'].set_acquire_mode('SAMple')
self.dut['Oscilloscope'].set_acquire_stop_after("RUNSTop")
self.dut['Oscilloscope'].set_acquire_state("RUN")
time.sleep(1.5)
self.dut['Oscilloscope'].force_trigger()
self.dut['Oscilloscope'].set_acquire_state("STOP")
data = self.dut['Oscilloscope']._intf._resource.query_binary_values("DATA:SOURCE CH%d;:CURVe?" % self.channel, datatype='h', is_big_endian=True)
self.preamble = self.dut['Oscilloscope'].get_parameters(channel=self.channel)
times, voltages, time_unit, voltage_unit = interpret_oscilloscope_data(self.preamble, data)
if len(data):
trigger_level = (np.mean(voltages) - self.trigger_level_offset * 1e-3) / 2.0 + self.trigger_level_offset * 1e-3
else:
trigger_level = trigger_levels[-1]
self.dut['Oscilloscope'].set_trigger_level(trigger_level)
self.dut['Oscilloscope'].set_vertical_scale(min(self.vertical_scale, (np.mean(voltages) + 0.2 * np.mean(voltages)) / 10), channel=self.channel)
#self.dut['Oscilloscope'].set_vertical_scale(0.05, channel=self.channel)
if self.show_debug_plots:
plt.clf()
plt.grid()
plt.plot(times * 1e9, voltages * 1e3, label='PlsrDAC Pulse')
plt.axhline(y=trigger_level * 1e3, linewidth=2, linestyle="--", color='r', label='Trigger (%0.1f mV)' % (trigger_level * 1e3))
plt.xlabel('Time [ns]')
plt.ylabel('Voltage [mV]')
plt.legend(loc=0)
plt.show()
# Setup data aquisition and start scan loop
self.dut['Oscilloscope'].set_acquire_mode('AVErage') # average to get rid of noise and keeping high band width
self.dut['Oscilloscope'].set_acquire_stop_after("SEQuence")
self.dut['Oscilloscope'].set_acquire_state("RUN")
time.sleep(1.5)
super(PlsrDacTransientCalibrationAdvanced, self).scan() # analog scan loop
self.dut['Oscilloscope'].set_acquire_state("STOP")
# get final number of data points
# if not self.dut['Oscilloscope'].get_number_points():
# raise RuntimeError()
data = self.dut['Oscilloscope']._intf._resource.query_binary_values("DATA:SOURCE CH%d;:CURVe?" % self.channel, datatype='h', is_big_endian=True)
self.preamble = self.dut['Oscilloscope'].get_parameters(channel=self.channel)
times, voltages, time_unit, voltage_unit = interpret_oscilloscope_data(self.preamble, data)
data_out[index, :] = voltages[:]
trigger_level = float(self.dut['Oscilloscope'].get_trigger_level())
trigger_levels.append(trigger_level)
progress_bar.update(index)
if self.show_debug_plots:
plt.clf()
plt.ylim(0, 1500)
plt.grid()
plt.plot(times * 1e9, voltages * 1e3, label='PlsrDAC Pulse')
plt.axhline(y=trigger_level * 1e3, linewidth=2, linestyle="--", color='r', label='Trigger (%0.1f mV)' % (trigger_level * 1e3))
plt.xlabel('Time [ns]')
plt.ylabel('Voltage [mV]')
plt.legend(loc=0)
plt.show()
self.dut['Oscilloscope'].set_vertical_scale(self.vertical_scale, channel=self.channel)
time_out[:] = times
data_out.attrs.trigger_levels = trigger_levels
progress_bar.finish()
def scan(self):
# Output data structures
scan_parameter_values = self.scan_parameters.PlsrDAC
shape = (len(scan_parameter_values), self.data_index)
atom = tb.FloatAtom()
data_out = self.raw_data_file.h5_file.create_carray(self.raw_data_file.h5_file.root, name='PlsrDACwaveforms', title='Waveforms from transient PlsrDAC calibration scan', atom=atom, shape=shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
shape = (self.data_index,)
atom = tb.FloatAtom()
time_out = self.raw_data_file.h5_file.create_carray(self.raw_data_file.h5_file.root, name='Times', title='Time values', atom=atom, shape=shape, filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
data_out.attrs.scan_parameter_values = scan_parameter_values
data_out.attrs.enable_double_columns = self.enable_double_columns
data_out.attrs.fit_ranges = self.fit_ranges
data_out.attrs.trigger_level_offset = self.trigger_level_offset
trigger_levels = []
progress_bar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.AdaptiveETA()], maxval=len(scan_parameter_values), term_width=80)
progress_bar.start()
for index, scan_parameter_value in enumerate(scan_parameter_values):
if self.stop_run.is_set():
break
# Update PlsrDAC parameter
self.set_scan_parameters(PlsrDAC=scan_parameter_value) # set scan parameter
self.write_global_register('PlsrDAC', scan_parameter_value) # write to FE
self.dut['Oscilloscope'].set_trigger_mode("AUTO")
self.dut['Oscilloscope'].set_trigger_type("EDGe")
self.dut['Oscilloscope'].set_acquire_mode('SAMple')
self.dut['Oscilloscope'].set_acquire_stop_after("RUNSTop")
self.dut['Oscilloscope'].set_acquire_state("RUN")
time.sleep(1.5)
self.dut['Oscilloscope'].force_trigger()
self.dut['Oscilloscope'].set_acquire_state("STOP")
data = self.dut['Oscilloscope']._intf._resource.query_binary_values("DATA:SOURCE CH%d;:CURVe?" % self.channel, datatype='h', is_big_endian=True)
self.preamble = self.dut['Oscilloscope'].get_parameters(channel=self.channel)
times, voltages, time_unit, voltage_unit = interpret_oscilloscope_data(self.preamble, data)
if len(data):
trigger_level = (np.mean(voltages) - self.trigger_level_offset * 1e-3) / 2.0 + self.trigger_level_offset * 1e-3
else:
trigger_level = trigger_levels[-1]
self.dut['Oscilloscope'].set_trigger_level(trigger_level)
self.dut['Oscilloscope'].set_vertical_scale(min(self.vertical_scale, (np.mean(voltages) + 0.2 * np.mean(voltages)) / 10), channel=self.channel)
# self.dut['Oscilloscope'].set_vertical_scale(0.05, channel=self.channel)
# Setup data aquisition and start scan loop
self.dut['Oscilloscope'].set_trigger_mode("NORMal")
self.dut['Oscilloscope'].set_trigger_type("PULSe")
self.dut['Oscilloscope'].set_acquire_mode('AVErage') # average to get rid of noise and keeping high band width
self.dut['Oscilloscope'].set_acquire_stop_after("SEQuence")
self.dut['Oscilloscope'].set_acquire_state("RUN")
time.sleep(1.5)
super(PlsrDacTransientCalibration, self).scan() # analog scan loop
self.dut['Oscilloscope'].set_acquire_state("STOP")
if self.dut['Oscilloscope'].get_number_waveforms() == 0:
logging.warning("No acquisition taking place.")
data = self.dut['Oscilloscope']._intf._resource.query_binary_values("DATA:SOURCE CH%d;:CURVe?" % self.channel, datatype='h', is_big_endian=True)
self.preamble = self.dut['Oscilloscope'].get_parameters(channel=self.channel)
times, voltages, time_unit, voltage_unit = interpret_oscilloscope_data(self.preamble, data)
data_out[index, :] = voltages[:]
trigger_level = float(self.dut['Oscilloscope'].get_trigger_level())
trigger_levels.append(trigger_level)
progress_bar.update(index)
self.dut['Oscilloscope'].set_vertical_scale(self.vertical_scale, channel=self.channel)
time_out[:] = times
data_out.attrs.trigger_levels = trigger_levels
progress_bar.finish()
for cmph in cmph_covariates:
print 'Subsetting for %s'%cmph
lon_,lat_,data = map_utils.CRU_extract('.','%s'%cmph, zip=False)
lon_.sort()
lat_.sort()
# data = map_utils.interp_geodata(lon_, lat_, data, lon[lon_min_i:lon_max_i], lat[lon_min_i:lon_max_i])
data = map_utils.grid_convert(basemap.interp(map_utils.grid_convert(data,'y-x+','y+x+'), lon_, lat_, *np.meshgrid(lon[lon_min_i:lon_max_i],lat[lat_min_i:lat_max_i])),'y+x+','x+y+')
for res in [5]:
hf_out = tb.openFile(os.path.join('%ik-covariates'%res,cmph.lower()+'.hdf5'),'w')
hf_out.createArray('/','lon',lon[lon_min_i:lon_max_i][::res])
hf_out.createArray('/','lat',lat[lat_min_i:lat_max_i][::res])
d = map_utils.grid_convert(data[::res,::res], 'x+y+','y-x+')
hf_out.createCArray('/','data',atom=tb.FloatAtom(),shape=d.shape,filters=tb.Filters(complevel=1,complib='zlib'))
hf_out.createCArray('/','mask',atom=tb.BoolAtom(),shape=d.shape,filters=tb.Filters(complevel=1,complib='zlib'))
hf_out.root.data.attrs.view = 'y-x+'
hf_out.root.data[:]=d
hf_out.root.mask[:] = clipped_pete_mask
hf_out.close()
glob = tb.openFile('Globcover.hdf5')
for c in glob_channels:
subset_and_writeout(glob, 'globcover-channel-%i'%c, 3, glob_missing, lambda x:x==c)
glob.close()
# Reconcile the masks
print 'Finding the conservative mask'
