def _k2is_read_ranges_tile_block( slices_arr, fileset_arr, slice_sig_sizes, sig_origins, inner_indices_start, inner_indices_stop, frame_indices, sig_size, px_to_bytes, bpp, frame_header_bytes, frame_footer_bytes, file_idxs, slice_offset, extra, sig_shape, ): result = List() # positions in the signal dimensions: for slice_idx in range(slices_arr.shape[0]): # (offset, size) arrays defining what data to read (in pixels) slice_origin = slices_arr[slice_idx][0] slice_shape = slices_arr[slice_idx][1] read_ranges = List() n_blocks_y = slice_shape[0] // 930 n_blocks_x = slice_shape[1] // 16 origin_block_y = slice_origin[0] // 930 origin_block_x = slice_origin[1] // 16 # inner "depth" loop along the (flat) navigation axis of a tile: for i, inner_frame_idx in enumerate( range(inner_indices_start, inner_indices_stop)): inner_frame = frame_indices[inner_frame_idx] frame_in_file_idx = inner_frame # in k2is all files contain data from all frames for block_y_i in range(n_blocks_y): sector_index_y = origin_block_y + block_y_i for block_x_i in range(n_blocks_x): block_index_x = origin_block_x + block_x_i sector_id = block_index_x // 16 sector_index_x = block_index_x % 16 # f = fileset_arr[sector_id] # "linear" block index per sector: blockidx = (15 - sector_index_x) + sector_index_y * 16 offset = (frame_in_file_idx * BLOCK_SIZE * BLOCKS_PER_SECTOR_PER_FRAME + blockidx * BLOCK_SIZE) start = offset + HEADER_SIZE stop = offset + HEADER_SIZE + DATA_SIZE read_ranges.append((sector_id, start, stop, n_blocks_y, n_blocks_x, block_y_i, block_x_i)) # the indices are compressed to the selected frames compressed_slice = np.array([ [slice_offset + inner_indices_start] + [i for i in slice_origin], [inner_indices_stop - inner_indices_start] + [i for i in slice_shape], ]) result.append((slice_idx, compressed_slice, read_ranges)) return result
def test_basic(self): l = List.empty_list(int32) # len self.assertEqual(len(l), 0) # append l.append(0) # len self.assertEqual(len(l), 1) # setitem l.append(0) l.append(0) l[0] = 10 l[1] = 11 l[2] = 12 # getitem self.assertEqual(l[0], 10) self.assertEqual(l[1], 11) self.assertEqual(l[2], 12) self.assertEqual(l[-3], 10) self.assertEqual(l[-2], 11) self.assertEqual(l[-1], 12) # __iter__ # the default __iter__ from MutableSequence will raise an IndexError # via __getitem__ and thus leak an exception, so this shouldn't for i in l: pass # contains self.assertTrue(10 in l) self.assertFalse(0 in l) # count l.append(12) self.assertEqual(l.count(0), 0) self.assertEqual(l.count(10), 1) self.assertEqual(l.count(12), 2) # pop self.assertEqual(len(l), 4) self.assertEqual(l.pop(), 12) self.assertEqual(len(l), 3) self.assertEqual(l.pop(1), 11) self.assertEqual(len(l), 2) # extend l.extend((100, 200, 300)) self.assertEqual(len(l), 5) self.assertEqual(list(l), [10, 12, 100, 200, 300]) # insert l.insert(0, 0) self.assertEqual(list(l), [0, 10, 12, 100, 200, 300]) l.insert(3, 13) self.assertEqual(list(l), [0, 10, 12, 13, 100, 200, 300]) l.insert(100, 400) self.assertEqual(list(l), [0, 10, 12, 13, 100, 200, 300, 400]) # remove l.remove(0) l.remove(400) l.remove(13) self.assertEqual(list(l), [10, 12, 100, 200, 300]) # clear l.clear() self.assertEqual(len(l), 0) self.assertEqual(list(l), []) # reverse l.extend(tuple(range(10, 20))) l.reverse() self.assertEqual(list(l), list(range(10, 20))[::-1]) # copy new = l.copy() self.assertEqual(list(new), list(range(10, 20))[::-1]) # equal self.assertEqual(l, new) # not equal new[-1] = 42 self.assertNotEqual(l, new) # index self.assertEqual(l.index(15), 4)
def foo(): x = List() for i in global_typed_list: x.append(i)
def foo(): l = List(iterable=(0, 1, 2)) return l
def producer(): l = List.empty_list(int32) l.append(23) return l
def foo(): return List(np.array([[1, 2], [3, 4]]))
def foo(): l = List((1, 1.0)) return l
# Reindex data: obs_i = obs_i.reset_index(drop=True) # Check that there are ANY data for this country/season: if any(obs_i > 0) and not obs_i.isnull().all(): # Replace leading/lagging zeros: replaceLeadLag(obs_i) # Get OEV: obs_vars = calc_obsvars_ISOLATED(obs_i, oev_base, oev_denom) # Get time start and time range: tm_ini = clim_start_dict[season] - 2 # b/c python_init indexes at 0, while R does it at 1 tm_range1 = [i for i in range(clim_start_dict[season] - 1, clim_end_dict[season], 1)] tm_range = List() [tm_range.append(i) for i in tm_range1] # Run forecasts! for run in range(num_runs): # print(run) # Get initial states/parameters for each ensemble member: param_init = pd.read_csv(os.path.join('data/python_init/initial_parms/', 'parms' + str(run) + '_INDIV.txt'), header=None, sep='\t') param_init = param_init.to_numpy(dtype=np.float64) # print(param_init.shape) # (6, 300) # Run EAKF: res = EAKF_fn_fitOnly_ISOLATED(num_ens, tmstep, param_init, obs_i, nsn, nsn, obs_vars, tm_ini, tm_range, N, ah_count, dt, lambda_val, wk_start)
import numpy as np from numba import njit from numba import int32, float32, prange from numba.core import types from numba import typeof from numba.typed import List, Dict from numba.core.errors import TypingError from numba.tests.support import (TestCase, MemoryLeakMixin, override_config, forbid_codegen, skip_parfors_unsupported) from numba.core.unsafe.refcount import get_refcount from numba.experimental import jitclass # global typed-list for testing purposes global_typed_list = List.empty_list(int32) for i in (1, 2, 3): global_typed_list.append(int32(i)) def to_tl(l): """ Convert cpython list to typed-list. """ tl = List.empty_list(int32) for k in l: tl.append(k) return tl class TestTypedList(MemoryLeakMixin, TestCase): def test_basic(self): l = List.empty_list(int32)
def _get_tiles_w_copy( self, tiling_scheme, open_files, read_ranges, read_dtype, native_dtype, decoder=None, corrections=None, ): if decoder is None: decoder = DtypeConversionDecoder() decode = decoder.get_decode( native_dtype=np.dtype(native_dtype), read_dtype=np.dtype(read_dtype), ) r_n_d = self._r_n_d = self.get_read_and_decode(decode) native_dtype = decoder.get_native_dtype(native_dtype, read_dtype) mmaps = List() for fh in open_files: mmaps.append(np.frombuffer(fh.mmap, dtype=np.uint8)) sig_dims = tiling_scheme.shape.sig.dims ds_shape = np.array(tiling_scheme.dataset_shape) largest_slice = sorted( ((prod(s_.shape), s_) for _, s_ in tiling_scheme.slices), key=lambda x: x[0], reverse=True)[0][1] buf_shape = (tiling_scheme.depth, ) + tuple(largest_slice.shape) need_clear = decoder.do_clear() with self._buffer_pool.empty(buf_shape, dtype=read_dtype) as out_decoded: out_decoded = out_decoded.reshape((-1, )) slices = read_ranges[0] # Use NumPy prod for multidimensional array and axis parameter shape_prods = np.prod(slices[..., 1, :], axis=1, dtype=np.int64) ranges = read_ranges[1] scheme_indices = read_ranges[2] for idx in range(slices.shape[0]): origin = slices[idx, 0] shape = slices[idx, 1] tile_slice = Slice(origin=origin, shape=Shape(shape, sig_dims=sig_dims)) tile_ranges = ranges[idx] scheme_idx = scheme_indices[idx] # if idx < slices.shape[0] - 1: # self._prefetch_for_tile(fileset, ranges[idx + 1]) # pass out_cut = out_decoded[:shape_prods[idx]].reshape( (shape[0], -1)) data = r_n_d( idx, mmaps, sig_dims, tile_ranges, out_cut, native_dtype, do_zero=need_clear, origin=origin, shape=shape, ds_shape=ds_shape, ) data = data.reshape(shape) self.preprocess(data, tile_slice, corrections) yield DataTile( data, tile_slice=tile_slice, scheme_idx=scheme_idx, )
def _get_read_ranges_inner( start_at_frame, stop_before_frame, roi, depth, slices_arr, fileset_arr, sig_shape, bpp, sync_offset=0, extra=None, frame_header_bytes=0, frame_footer_bytes=0, ): result = NumbaList() sig_size = np.prod(np.array(sig_shape).astype(np.int64)) if roi is None: frame_indices = np.arange(max(0, start_at_frame), stop_before_frame) # in case of a negative sync_offset, start_at_frame can be negative if start_at_frame < 0: slice_offset = abs(sync_offset) else: slice_offset = start_at_frame - sync_offset else: frame_indices = _roi_to_indices( roi, max(0, start_at_frame), stop_before_frame, sync_offset ) # in case of a negative sync_offset, start_at_frame can be negative if start_at_frame < 0: slice_offset = np.count_nonzero(roi.reshape((-1,))[:abs(sync_offset)]) else: slice_offset = np.count_nonzero(roi.reshape((-1,))[:start_at_frame - sync_offset]) num_indices = frame_indices.shape[0] # indices into `frame_indices`: inner_indices_start = 0 inner_indices_stop = min(depth, num_indices) # this should be `np.prod(..., axis=-1)``, which is not supported by numba yet: # slices that divide the signal dimensions: slice_sig_sizes = np.array([ np.prod(slices_arr[slice_idx, 1, :].astype(np.int64)) for slice_idx in range(slices_arr.shape[0]) ]) sig_origins = np.array([ _ravel_multi_index(slices_arr[slice_idx][0], sig_shape) for slice_idx in range(slices_arr.shape[0]) ]) # outer "depth" loop skipping over `depth` frames at a time: while inner_indices_start < num_indices: file_idxs = np.array([ _find_file_for_frame_idx(fileset_arr, frame_indices[inner_frame_idx]) for inner_frame_idx in range(inner_indices_start, inner_indices_stop) ]) for slice_idx, compressed_slice, read_ranges in read_ranges_tile_block( slices_arr, fileset_arr, slice_sig_sizes, sig_origins, inner_indices_start, inner_indices_stop, frame_indices, sig_size, px_to_bytes, bpp, frame_header_bytes, frame_footer_bytes, file_idxs, slice_offset, extra=extra, sig_shape=sig_shape, ): result.append((compressed_slice, read_ranges, slice_idx)) inner_indices_start = inner_indices_start + depth inner_indices_stop = min(inner_indices_stop + depth, num_indices) result_slices = np.zeros((len(result), 2, 1 + len(sig_shape)), dtype=np.int64) for tile_idx, res in enumerate(result): result_slices[tile_idx] = res[0] if len(result) == 0: return ( result_slices, np.zeros((len(result), depth, 3), dtype=np.int64), np.zeros((len(result)), dtype=np.int64), ) max_rr_per_tile = max([len(res[1]) for res in result]) slice_indices = np.zeros(len(result), dtype=np.int64) # read_ranges_tile_block can decide how many entries there are per read range, # so we need to generate a result array with the correct size: rr_num_entries = max(3, len(result[0][1][0])) result_ranges = np.zeros((len(result), max_rr_per_tile, rr_num_entries), dtype=np.int64) for tile_idx, res in enumerate(result): for depth_idx, read_range in enumerate(res[1]): result_ranges[tile_idx][depth_idx] = read_range slice_indices[tile_idx] = res[2] return result_slices, result_ranges, slice_indices
def __init__(self, deviceXML, placement_delay_lookup_file, mdg_checkpoint): # parse dimensions of device layout # current implementation assumes square layout (W = H) # which is the common case in current commercial FPGA devices # and also assume corners of the layout are empty # and perimeters of the layout are IO root = ET.parse(deviceXML).getroot() device_layout = root[2] if device_layout[0].tag != 'fixed_layout': print( 'GraphPlace can only work with fixed layout architecture XML file' ) exit(0) W, H = int(device_layout[0].attrib['width']), int( device_layout[0].attrib['height']) if W != H: print('GraphPlace can only work with square layout') exit(0) if (os.path.exists(mdg_checkpoint)): print('Found an existing MDG checkpoint, loading it', flush=True) # if MDG is already generated before, it's unnecessary to generate that # directly load it MDG = loadMDG(mdg_checkpoint) else: # generate a new MDG and store it to mdg_file print('Did not find an existing MDG checkpoint, generating it', flush=True) # generate a delta delay lookup dictionary delta_delay_lookup_dict = dict() with open(placement_delay_lookup_file, 'r') as f: next(f) for line in f: tokens = line.split() delta_y = int(tokens[0]) for delta_x in range(len(tokens) - 1): delta_delay_lookup_dict[( delta_x, delta_y)] = float(tokens[delta_x + 1]) * 1e9 # create metric distance graph MDG = nx.Graph() for (source_x, source_y) in [(x, y) for x in range(W) for y in range(H)]: if (source_x, source_y) not in [(0, 0), (0, H - 1), (W - 1, 0), (W - 1, H - 1)]: for (sink_x, sink_y) in [(x, y) for x in range(W) for y in range(H)]: if (sink_x, sink_y) not in [(0, 0), (0, H - 1), (W - 1, 0), (W - 1, H - 1)]: delta_x, delta_y = abs(source_x - sink_x), abs(source_y - sink_y) delay = delta_delay_lookup_dict[(delta_x, delta_y)] source_vertex, sink_vertex = 'BLK_%d_%d' % ( source_x, source_y), 'BLK_%d_%d' % (sink_x, sink_y) MDG.add_edge(source_vertex, sink_vertex, weight=delay) # Save MDG checkpoint saveMDG(MDG, mdg_checkpoint) # Construct two lists, one for IO, one for CLB # Each list contains all physical locations compatible for the list type # e.g. IO_sites list all physical locations which IO blocks in netlist can sit # currently assuming each physical location can be only compatible with one kind of block in netlist self.IO_sites = List() self.CLB_sites = List() for (idx, node) in enumerate(MDG): x, y = int(re.findall(r'\d+', node)[0]), int(re.findall(r'\d+', node)[1]) if x == 0 or x == W - 1 or y == 0 or y == H - 1: # an IO site because it's at perimeter self.IO_sites.append(idx) else: # an CLB site self.CLB_sites.append(idx) self.nodes = list(MDG) self.MDM = nx.adjacency_matrix(MDG).todense() self.W = W self.H = H
def nlist(xs): nl = NList() for x in xs: nl.append(x) return nl
def hydr(store, siminfo, uci, ts): ''' find the state of the reach/reservoir at the end of the time interval and the outflows during the interval CALL: hydr(store, general, ui, ts) store is the Pandas/PyTable open store general is a dictionary with simulation level infor (OP_SEQUENCE for example) ui is a dictionary with RID specific HSPF UCI like data ts is a dictionary with RID specific timeseries''' steps = siminfo['steps'] # number of simulation points nexits = int(uci['PARAMETERS']['NEXITS']) u = uci['PARAMETERS'] funct = array([u[name] for name in u.keys() if name.startswith('FUNCT')]).astype(int)[0:nexits] ODGTF = array([u[name] for name in u.keys() if name.startswith('ODGTF')]).astype(int)[0:nexits] ODFVF = array([u[name] for name in u.keys() if name.startswith('ODFVF')]).astype(int)[0:nexits] u = uci['STATES'] colin = array([u[name] for name in u.keys() if name.startswith('COLIN')]).astype(float)[0:nexits] outdg = array([u[name] for name in u.keys() if name.startswith('OUTDG')]).astype(float)[0:nexits] # COLIND timeseries might come in as COLIND, COLIND0, etc. otherwise UCI default names = list( sorted([n for n in ts if n.startswith('COLIND')], reverse=True)) df = DataFrame() for i, c in enumerate(ODFVF): df[i] = ts[names.pop()][0:steps] if c < 0 else full(steps, c) COLIND = df.to_numpy() # OUTDGT timeseries might come in as OUTDGT, OUTDGT0, etc. otherwise UCI default names = list(sorted([n for n in ts if n.startswith('OUTDG')], reverse=True)) df = DataFrame() for i, c in enumerate(ODGTF): df[i] = ts[names.pop()][0:steps] if c > 0 else zeros(steps) OUTDGT = df.to_numpy() # generic SAVE table doesn't know nexits for output flows and rates if nexits > 1: u = uci['SAVE'] for key in ('O', 'OVOL'): for i in range(nexits): u[f'{key}{i+1}'] = u[key] del u[key] # optional - defined, but can't used accidently for name in ('SOLRAD', 'CLOUD', 'DEWTEMP', 'GATMP', 'WIND'): if name not in ts: ts[name] = full(steps, nan) # optional timeseries for name in ('IVOL', 'POTEV', 'PREC'): if name not in ts: ts[name] = zeros(steps) ts['CONVF'] = initm(siminfo, uci, 'VCONFG', 'MONTHLY_CONVF', 1.0) # extract key columns of specified FTable for faster access (1d vs. 2d) rchtab = store[f"FTABLES/{uci['PARAMETERS']['FTBUCI']}"] ts['volumeFT'] = rchtab['Volume'].to_numpy() * VFACT ts['depthFT'] = rchtab['Depth'].to_numpy() ts['sareaFT'] = rchtab['Area'].to_numpy() * AFACT rchtab = rchtab.to_numpy() ui = make_numba_dict( uci) # Note: all values coverted to float automatically ui['steps'] = steps ui['delt'] = siminfo['delt'] ui['nexits'] = nexits ui['errlen'] = len(ERRMSGS) ui['nrows'] = rchtab.shape[0] ui['nodfv'] = any(ODFVF) # Numba can't do 'O' + str(i) stuff yet, so do it here. Also need new style lists Olabels = List() OVOLlabels = List() for i in range(nexits): Olabels.append(f'O{i+1}') OVOLlabels.append(f'OVOL{i+1}') ########################################################################### errors = _hydr_(ui, ts, COLIND, OUTDGT, rchtab, funct, Olabels, OVOLlabels) # run reaches simulation code ########################################################################### if 'O' in ts: del ts['O'] if 'OVOL' in ts: del ts['OVOL'] return errors, ERRMSGS
def foo(): return List(np.ones(3))
def foo(): l = List() l.append(Bag(21)) l.append(Bag(22)) l.append(Bag(23)) return l
def foo(): return List(np.array(1))
def foo(string): l = List() l.extend(string) return l
def foo(): l = List(23) return l
def foo(): l = List() l.append(1) return l._is_mutable()
def foo(): l = List((0, 1, 2), (3, 4, 5), (6, 7, 8)) return l
def foo(): l = List() l.append(1) l._make_immutable() return l[0], l._is_mutable()
def foo(): li, lf = List(), List() li.append(int32(1)) lf.append(float32(1.0)) return li._dtype, lf._dtype
def foo(): l = List() l.append(1) l._make_immutable() l.append(1)
def to_tl(l): """ Convert cpython list to typed-list. """ tl = List.empty_list(int32) for k in l: tl.append(k) return tl
def foo(): l = List("abc") return l
def test_list_create_no_jit_using_List(self): with override_config('DISABLE_JIT', True): with forbid_codegen(): l = List() self.assertEqual(type(l), list)
def generate_expected(values): expected = List() for i in values: expected.append(i) return expected
def impl(): l = List() l.append(None) return l
from numba import njit from numba.typed import List # Some dummy parameters for use when testing: p = 3 q = 1 N = 5 g2 = -3.7 g4 = 1 # Get the clifford algebra for the model we are examining. cliff = Clifford(p, q) cliff.introduce() # odd_products = cliff.get_odd_gamma_products() # Fixing a numba error. odd_products = List() [odd_products.append(x) for x in cliff.get_odd_gamma_products()] odd_products # odd_products = np.array(odd_products[:], order='C')[:] # odd_products # %% """ There are some import hyperparameters for the Metropolis-Hastings algorithm. Specifically, in our implementation, the variable weightA is important, which is the step size between Dirac operators. WeightA to be very small, so that the steps between Diracs is relatively small. There is a balance between having a small enough step size so that many Diracs are accepted and we "follow the valley downwards" and having a large enough step size so that we can escape local minima. This value will change for each type, it seems to be a hyperparameter of this setup, i.e. requires tuning. weightA values for the types that give acceptance_rate/num_moves ~50% (this is what I understand to be a good thing to achieve) (2,0) -> 1./np.power(cliff.matdim,3)/25 (1,3) -> 1./e-2/6 for size 10x10 - takes 30mins to do 40,000 moves.