def _k2is_read_ranges_tile_block(
slices_arr,
fileset_arr,
slice_sig_sizes,
sig_origins,
inner_indices_start,
inner_indices_stop,
frame_indices,
sig_size,
px_to_bytes,
bpp,
frame_header_bytes,
frame_footer_bytes,
file_idxs,
slice_offset,
extra,
sig_shape,
):
result = List()
# positions in the signal dimensions:
for slice_idx in range(slices_arr.shape[0]):
# (offset, size) arrays defining what data to read (in pixels)
slice_origin = slices_arr[slice_idx][0]
slice_shape = slices_arr[slice_idx][1]
read_ranges = List()
n_blocks_y = slice_shape[0] // 930
n_blocks_x = slice_shape[1] // 16
origin_block_y = slice_origin[0] // 930
origin_block_x = slice_origin[1] // 16
# inner "depth" loop along the (flat) navigation axis of a tile:
for i, inner_frame_idx in enumerate(
range(inner_indices_start, inner_indices_stop)):
inner_frame = frame_indices[inner_frame_idx]
frame_in_file_idx = inner_frame # in k2is all files contain data from all frames
for block_y_i in range(n_blocks_y):
sector_index_y = origin_block_y + block_y_i
for block_x_i in range(n_blocks_x):
block_index_x = origin_block_x + block_x_i
sector_id = block_index_x // 16
sector_index_x = block_index_x % 16
# f = fileset_arr[sector_id]
# "linear" block index per sector:
blockidx = (15 - sector_index_x) + sector_index_y * 16
offset = (frame_in_file_idx * BLOCK_SIZE *
BLOCKS_PER_SECTOR_PER_FRAME +
blockidx * BLOCK_SIZE)
start = offset + HEADER_SIZE
stop = offset + HEADER_SIZE + DATA_SIZE
read_ranges.append((sector_id, start, stop, n_blocks_y,
n_blocks_x, block_y_i, block_x_i))
# the indices are compressed to the selected frames
compressed_slice = np.array([
[slice_offset + inner_indices_start] + [i for i in slice_origin],
[inner_indices_stop - inner_indices_start] +
[i for i in slice_shape],
])
result.append((slice_idx, compressed_slice, read_ranges))
return result
def test_basic(self):
l = List.empty_list(int32)
# len
self.assertEqual(len(l), 0)
# append
l.append(0)
# len
self.assertEqual(len(l), 1)
# setitem
l.append(0)
l.append(0)
l[0] = 10
l[1] = 11
l[2] = 12
# getitem
self.assertEqual(l[0], 10)
self.assertEqual(l[1], 11)
self.assertEqual(l[2], 12)
self.assertEqual(l[-3], 10)
self.assertEqual(l[-2], 11)
self.assertEqual(l[-1], 12)
# __iter__
# the default __iter__ from MutableSequence will raise an IndexError
# via __getitem__ and thus leak an exception, so this shouldn't
for i in l:
pass
# contains
self.assertTrue(10 in l)
self.assertFalse(0 in l)
# count
l.append(12)
self.assertEqual(l.count(0), 0)
self.assertEqual(l.count(10), 1)
self.assertEqual(l.count(12), 2)
# pop
self.assertEqual(len(l), 4)
self.assertEqual(l.pop(), 12)
self.assertEqual(len(l), 3)
self.assertEqual(l.pop(1), 11)
self.assertEqual(len(l), 2)
# extend
l.extend((100, 200, 300))
self.assertEqual(len(l), 5)
self.assertEqual(list(l), [10, 12, 100, 200, 300])
# insert
l.insert(0, 0)
self.assertEqual(list(l), [0, 10, 12, 100, 200, 300])
l.insert(3, 13)
self.assertEqual(list(l), [0, 10, 12, 13, 100, 200, 300])
l.insert(100, 400)
self.assertEqual(list(l), [0, 10, 12, 13, 100, 200, 300, 400])
# remove
l.remove(0)
l.remove(400)
l.remove(13)
self.assertEqual(list(l), [10, 12, 100, 200, 300])
# clear
l.clear()
self.assertEqual(len(l), 0)
self.assertEqual(list(l), [])
# reverse
l.extend(tuple(range(10, 20)))
l.reverse()
self.assertEqual(list(l), list(range(10, 20))[::-1])
# copy
new = l.copy()
self.assertEqual(list(new), list(range(10, 20))[::-1])
# equal
self.assertEqual(l, new)
# not equal
new[-1] = 42
self.assertNotEqual(l, new)
# index
self.assertEqual(l.index(15), 4)
def foo():
x = List()
for i in global_typed_list:
x.append(i)
def foo():
l = List(iterable=(0, 1, 2))
return l
def producer():
l = List.empty_list(int32)
l.append(23)
return l
def foo():
return List(np.array([[1, 2], [3, 4]]))
def foo():
l = List((1, 1.0))
return l
# Reindex data:
obs_i = obs_i.reset_index(drop=True)
# Check that there are ANY data for this country/season:
if any(obs_i > 0) and not obs_i.isnull().all():
# Replace leading/lagging zeros:
replaceLeadLag(obs_i)
# Get OEV:
obs_vars = calc_obsvars_ISOLATED(obs_i, oev_base, oev_denom)
# Get time start and time range:
tm_ini = clim_start_dict[season] - 2 # b/c python_init indexes at 0, while R does it at 1
tm_range1 = [i for i in range(clim_start_dict[season] - 1, clim_end_dict[season], 1)]
tm_range = List()
[tm_range.append(i) for i in tm_range1]
# Run forecasts!
for run in range(num_runs):
# print(run)
# Get initial states/parameters for each ensemble member:
param_init = pd.read_csv(os.path.join('data/python_init/initial_parms/', 'parms' + str(run) + '_INDIV.txt'),
header=None, sep='\t')
param_init = param_init.to_numpy(dtype=np.float64)
# print(param_init.shape) # (6, 300)
# Run EAKF:
res = EAKF_fn_fitOnly_ISOLATED(num_ens, tmstep, param_init, obs_i, nsn, nsn, obs_vars, tm_ini,
tm_range, N, ah_count, dt, lambda_val, wk_start)
import numpy as np
from numba import njit
from numba import int32, float32, prange
from numba.core import types
from numba import typeof
from numba.typed import List, Dict
from numba.core.errors import TypingError
from numba.tests.support import (TestCase, MemoryLeakMixin, override_config,
forbid_codegen, skip_parfors_unsupported)
from numba.core.unsafe.refcount import get_refcount
from numba.experimental import jitclass
# global typed-list for testing purposes
global_typed_list = List.empty_list(int32)
for i in (1, 2, 3):
global_typed_list.append(int32(i))
def to_tl(l):
""" Convert cpython list to typed-list. """
tl = List.empty_list(int32)
for k in l:
tl.append(k)
return tl
class TestTypedList(MemoryLeakMixin, TestCase):
def test_basic(self):
l = List.empty_list(int32)
def _get_tiles_w_copy(
self,
tiling_scheme,
open_files,
read_ranges,
read_dtype,
native_dtype,
decoder=None,
corrections=None,
):
if decoder is None:
decoder = DtypeConversionDecoder()
decode = decoder.get_decode(
native_dtype=np.dtype(native_dtype),
read_dtype=np.dtype(read_dtype),
)
r_n_d = self._r_n_d = self.get_read_and_decode(decode)
native_dtype = decoder.get_native_dtype(native_dtype, read_dtype)
mmaps = List()
for fh in open_files:
mmaps.append(np.frombuffer(fh.mmap, dtype=np.uint8))
sig_dims = tiling_scheme.shape.sig.dims
ds_shape = np.array(tiling_scheme.dataset_shape)
largest_slice = sorted(
((prod(s_.shape), s_) for _, s_ in tiling_scheme.slices),
key=lambda x: x[0],
reverse=True)[0][1]
buf_shape = (tiling_scheme.depth, ) + tuple(largest_slice.shape)
need_clear = decoder.do_clear()
with self._buffer_pool.empty(buf_shape,
dtype=read_dtype) as out_decoded:
out_decoded = out_decoded.reshape((-1, ))
slices = read_ranges[0]
# Use NumPy prod for multidimensional array and axis parameter
shape_prods = np.prod(slices[..., 1, :], axis=1, dtype=np.int64)
ranges = read_ranges[1]
scheme_indices = read_ranges[2]
for idx in range(slices.shape[0]):
origin = slices[idx, 0]
shape = slices[idx, 1]
tile_slice = Slice(origin=origin,
shape=Shape(shape, sig_dims=sig_dims))
tile_ranges = ranges[idx]
scheme_idx = scheme_indices[idx]
# if idx < slices.shape[0] - 1:
# self._prefetch_for_tile(fileset, ranges[idx + 1])
# pass
out_cut = out_decoded[:shape_prods[idx]].reshape(
(shape[0], -1))
data = r_n_d(
idx,
mmaps,
sig_dims,
tile_ranges,
out_cut,
native_dtype,
do_zero=need_clear,
origin=origin,
shape=shape,
ds_shape=ds_shape,
)
data = data.reshape(shape)
self.preprocess(data, tile_slice, corrections)
yield DataTile(
data,
tile_slice=tile_slice,
scheme_idx=scheme_idx,
)
def _get_read_ranges_inner(
start_at_frame, stop_before_frame, roi, depth,
slices_arr, fileset_arr, sig_shape,
bpp, sync_offset=0, extra=None, frame_header_bytes=0, frame_footer_bytes=0,
):
result = NumbaList()
sig_size = np.prod(np.array(sig_shape).astype(np.int64))
if roi is None:
frame_indices = np.arange(max(0, start_at_frame), stop_before_frame)
# in case of a negative sync_offset, start_at_frame can be negative
if start_at_frame < 0:
slice_offset = abs(sync_offset)
else:
slice_offset = start_at_frame - sync_offset
else:
frame_indices = _roi_to_indices(
roi, max(0, start_at_frame), stop_before_frame, sync_offset
)
# in case of a negative sync_offset, start_at_frame can be negative
if start_at_frame < 0:
slice_offset = np.count_nonzero(roi.reshape((-1,))[:abs(sync_offset)])
else:
slice_offset = np.count_nonzero(roi.reshape((-1,))[:start_at_frame - sync_offset])
num_indices = frame_indices.shape[0]
# indices into `frame_indices`:
inner_indices_start = 0
inner_indices_stop = min(depth, num_indices)
# this should be `np.prod(..., axis=-1)``, which is not supported by numba yet:
# slices that divide the signal dimensions:
slice_sig_sizes = np.array([
np.prod(slices_arr[slice_idx, 1, :].astype(np.int64))
for slice_idx in range(slices_arr.shape[0])
])
sig_origins = np.array([
_ravel_multi_index(slices_arr[slice_idx][0], sig_shape)
for slice_idx in range(slices_arr.shape[0])
])
# outer "depth" loop skipping over `depth` frames at a time:
while inner_indices_start < num_indices:
file_idxs = np.array([
_find_file_for_frame_idx(fileset_arr, frame_indices[inner_frame_idx])
for inner_frame_idx in range(inner_indices_start, inner_indices_stop)
])
for slice_idx, compressed_slice, read_ranges in read_ranges_tile_block(
slices_arr, fileset_arr, slice_sig_sizes, sig_origins,
inner_indices_start, inner_indices_stop, frame_indices, sig_size,
px_to_bytes, bpp, frame_header_bytes, frame_footer_bytes, file_idxs,
slice_offset, extra=extra, sig_shape=sig_shape,
):
result.append((compressed_slice, read_ranges, slice_idx))
inner_indices_start = inner_indices_start + depth
inner_indices_stop = min(inner_indices_stop + depth, num_indices)
result_slices = np.zeros((len(result), 2, 1 + len(sig_shape)), dtype=np.int64)
for tile_idx, res in enumerate(result):
result_slices[tile_idx] = res[0]
if len(result) == 0:
return (
result_slices,
np.zeros((len(result), depth, 3), dtype=np.int64),
np.zeros((len(result)), dtype=np.int64),
)
max_rr_per_tile = max([len(res[1])
for res in result])
slice_indices = np.zeros(len(result), dtype=np.int64)
# read_ranges_tile_block can decide how many entries there are per read range,
# so we need to generate a result array with the correct size:
rr_num_entries = max(3, len(result[0][1][0]))
result_ranges = np.zeros((len(result), max_rr_per_tile, rr_num_entries), dtype=np.int64)
for tile_idx, res in enumerate(result):
for depth_idx, read_range in enumerate(res[1]):
result_ranges[tile_idx][depth_idx] = read_range
slice_indices[tile_idx] = res[2]
return result_slices, result_ranges, slice_indices
def __init__(self, deviceXML, placement_delay_lookup_file, mdg_checkpoint):
# parse dimensions of device layout
# current implementation assumes square layout (W = H)
# which is the common case in current commercial FPGA devices
# and also assume corners of the layout are empty
# and perimeters of the layout are IO
root = ET.parse(deviceXML).getroot()
device_layout = root[2]
if device_layout[0].tag != 'fixed_layout':
print(
'GraphPlace can only work with fixed layout architecture XML file'
)
exit(0)
W, H = int(device_layout[0].attrib['width']), int(
device_layout[0].attrib['height'])
if W != H:
print('GraphPlace can only work with square layout')
exit(0)
if (os.path.exists(mdg_checkpoint)):
print('Found an existing MDG checkpoint, loading it', flush=True)
# if MDG is already generated before, it's unnecessary to generate that
# directly load it
MDG = loadMDG(mdg_checkpoint)
else: # generate a new MDG and store it to mdg_file
print('Did not find an existing MDG checkpoint, generating it',
flush=True)
# generate a delta delay lookup dictionary
delta_delay_lookup_dict = dict()
with open(placement_delay_lookup_file, 'r') as f:
next(f)
for line in f:
tokens = line.split()
delta_y = int(tokens[0])
for delta_x in range(len(tokens) - 1):
delta_delay_lookup_dict[(
delta_x,
delta_y)] = float(tokens[delta_x + 1]) * 1e9
# create metric distance graph
MDG = nx.Graph()
for (source_x, source_y) in [(x, y) for x in range(W)
for y in range(H)]:
if (source_x, source_y) not in [(0, 0), (0, H - 1), (W - 1, 0),
(W - 1, H - 1)]:
for (sink_x, sink_y) in [(x, y) for x in range(W)
for y in range(H)]:
if (sink_x, sink_y) not in [(0, 0), (0, H - 1),
(W - 1, 0),
(W - 1, H - 1)]:
delta_x, delta_y = abs(source_x -
sink_x), abs(source_y -
sink_y)
delay = delta_delay_lookup_dict[(delta_x, delta_y)]
source_vertex, sink_vertex = 'BLK_%d_%d' % (
source_x, source_y), 'BLK_%d_%d' % (sink_x,
sink_y)
MDG.add_edge(source_vertex,
sink_vertex,
weight=delay)
# Save MDG checkpoint
saveMDG(MDG, mdg_checkpoint)
# Construct two lists, one for IO, one for CLB
# Each list contains all physical locations compatible for the list type
# e.g. IO_sites list all physical locations which IO blocks in netlist can sit
# currently assuming each physical location can be only compatible with one kind of block in netlist
self.IO_sites = List()
self.CLB_sites = List()
for (idx, node) in enumerate(MDG):
x, y = int(re.findall(r'\d+',
node)[0]), int(re.findall(r'\d+', node)[1])
if x == 0 or x == W - 1 or y == 0 or y == H - 1: # an IO site because it's at perimeter
self.IO_sites.append(idx)
else: # an CLB site
self.CLB_sites.append(idx)
self.nodes = list(MDG)
self.MDM = nx.adjacency_matrix(MDG).todense()
self.W = W
self.H = H
def nlist(xs):
nl = NList()
for x in xs:
nl.append(x)
return nl
def hydr(store, siminfo, uci, ts):
''' find the state of the reach/reservoir at the end of the time interval
and the outflows during the interval
CALL: hydr(store, general, ui, ts)
store is the Pandas/PyTable open store
general is a dictionary with simulation level infor (OP_SEQUENCE for example)
ui is a dictionary with RID specific HSPF UCI like data
ts is a dictionary with RID specific timeseries'''
steps = siminfo['steps'] # number of simulation points
nexits = int(uci['PARAMETERS']['NEXITS'])
u = uci['PARAMETERS']
funct = array([u[name] for name in u.keys()
if name.startswith('FUNCT')]).astype(int)[0:nexits]
ODGTF = array([u[name] for name in u.keys()
if name.startswith('ODGTF')]).astype(int)[0:nexits]
ODFVF = array([u[name] for name in u.keys()
if name.startswith('ODFVF')]).astype(int)[0:nexits]
u = uci['STATES']
colin = array([u[name] for name in u.keys()
if name.startswith('COLIN')]).astype(float)[0:nexits]
outdg = array([u[name] for name in u.keys()
if name.startswith('OUTDG')]).astype(float)[0:nexits]
# COLIND timeseries might come in as COLIND, COLIND0, etc. otherwise UCI default
names = list(
sorted([n for n in ts if n.startswith('COLIND')], reverse=True))
df = DataFrame()
for i, c in enumerate(ODFVF):
df[i] = ts[names.pop()][0:steps] if c < 0 else full(steps, c)
COLIND = df.to_numpy()
# OUTDGT timeseries might come in as OUTDGT, OUTDGT0, etc. otherwise UCI default
names = list(sorted([n for n in ts if n.startswith('OUTDG')],
reverse=True))
df = DataFrame()
for i, c in enumerate(ODGTF):
df[i] = ts[names.pop()][0:steps] if c > 0 else zeros(steps)
OUTDGT = df.to_numpy()
# generic SAVE table doesn't know nexits for output flows and rates
if nexits > 1:
u = uci['SAVE']
for key in ('O', 'OVOL'):
for i in range(nexits):
u[f'{key}{i+1}'] = u[key]
del u[key]
# optional - defined, but can't used accidently
for name in ('SOLRAD', 'CLOUD', 'DEWTEMP', 'GATMP', 'WIND'):
if name not in ts:
ts[name] = full(steps, nan)
# optional timeseries
for name in ('IVOL', 'POTEV', 'PREC'):
if name not in ts:
ts[name] = zeros(steps)
ts['CONVF'] = initm(siminfo, uci, 'VCONFG', 'MONTHLY_CONVF', 1.0)
# extract key columns of specified FTable for faster access (1d vs. 2d)
rchtab = store[f"FTABLES/{uci['PARAMETERS']['FTBUCI']}"]
ts['volumeFT'] = rchtab['Volume'].to_numpy() * VFACT
ts['depthFT'] = rchtab['Depth'].to_numpy()
ts['sareaFT'] = rchtab['Area'].to_numpy() * AFACT
rchtab = rchtab.to_numpy()
ui = make_numba_dict(
uci) # Note: all values coverted to float automatically
ui['steps'] = steps
ui['delt'] = siminfo['delt']
ui['nexits'] = nexits
ui['errlen'] = len(ERRMSGS)
ui['nrows'] = rchtab.shape[0]
ui['nodfv'] = any(ODFVF)
# Numba can't do 'O' + str(i) stuff yet, so do it here. Also need new style lists
Olabels = List()
OVOLlabels = List()
for i in range(nexits):
Olabels.append(f'O{i+1}')
OVOLlabels.append(f'OVOL{i+1}')
###########################################################################
errors = _hydr_(ui, ts, COLIND, OUTDGT, rchtab, funct, Olabels,
OVOLlabels) # run reaches simulation code
###########################################################################
if 'O' in ts: del ts['O']
if 'OVOL' in ts: del ts['OVOL']
return errors, ERRMSGS
def foo():
return List(np.ones(3))
def foo():
l = List()
l.append(Bag(21))
l.append(Bag(22))
l.append(Bag(23))
return l
def foo():
return List(np.array(1))
def foo(string):
l = List()
l.extend(string)
return l
def foo():
l = List(23)
return l
def foo():
l = List()
l.append(1)
return l._is_mutable()
def foo():
l = List((0, 1, 2), (3, 4, 5), (6, 7, 8))
return l
def foo():
l = List()
l.append(1)
l._make_immutable()
return l[0], l._is_mutable()
def foo():
li, lf = List(), List()
li.append(int32(1))
lf.append(float32(1.0))
return li._dtype, lf._dtype
def foo():
l = List()
l.append(1)
l._make_immutable()
l.append(1)
def to_tl(l):
""" Convert cpython list to typed-list. """
tl = List.empty_list(int32)
for k in l:
tl.append(k)
return tl
def foo():
l = List("abc")
return l
def test_list_create_no_jit_using_List(self):
with override_config('DISABLE_JIT', True):
with forbid_codegen():
l = List()
self.assertEqual(type(l), list)
def generate_expected(values):
expected = List()
for i in values:
expected.append(i)
return expected
def impl():
l = List()
l.append(None)
return l
from numba import njit from numba.typed import List # Some dummy parameters for use when testing: p = 3 q = 1 N = 5 g2 = -3.7 g4 = 1 # Get the clifford algebra for the model we are examining. cliff = Clifford(p, q) cliff.introduce() # odd_products = cliff.get_odd_gamma_products() # Fixing a numba error. odd_products = List() [odd_products.append(x) for x in cliff.get_odd_gamma_products()] odd_products # odd_products = np.array(odd_products[:], order='C')[:] # odd_products # %% """ There are some import hyperparameters for the Metropolis-Hastings algorithm. Specifically, in our implementation, the variable weightA is important, which is the step size between Dirac operators. WeightA to be very small, so that the steps between Diracs is relatively small. There is a balance between having a small enough step size so that many Diracs are accepted and we "follow the valley downwards" and having a large enough step size so that we can escape local minima. This value will change for each type, it seems to be a hyperparameter of this setup, i.e. requires tuning. weightA values for the types that give acceptance_rate/num_moves ~50% (this is what I understand to be a good thing to achieve) (2,0) -> 1./np.power(cliff.matdim,3)/25 (1,3) -> 1./e-2/6 for size 10x10 - takes 30mins to do 40,000 moves.
