def _backward_cpu(self, x: tensor, y: tensor): dx = x.gradient npdx = x.gradient.host_data.ravel() dcol = y.gradient.host_data.ravel() _col2vol(npdx, dcol, self.batch_size, self.in_channels, self.n_output_plane, self.index_length, nbt.List(self._vol), nbt.List(self._col), nbt.List(self.kernel_size), nbt.List(self.stride), nbt.List(self.padding), nbt.List(self.dilation)) dx.host_data = npdx.reshape(x.shape) return dx
def make_empty_numba_list(): _nb_a_list = typed.List() # add variable, to tell numba the type, then remove it for the benchmark dummy = typed.List([0.01]) _nb_a_list.append(dummy) _nb_a_list.pop(0) return _nb_a_list
def get_pointcloud(path): # converting point cloud data to numba type ans = typed.List() with open(path, 'rb') as f: plydata = plyfile.PlyData.read(f) plydata = np.asarray(plydata.elements[1].data) for i in range(len(plydata)): raw = typed.List() [raw.append(types.float64(n)) for n in plydata[i]] ans.append(raw) return ans
def make_list(a_list): for i in range(10**4): new_list = typed.List(float64) for j in range(10**4): new_list.append(0.01) a_list.append(new_list) return a_list
def make_neural_network(layer_sizes, layer_activations, recurrent_layers, learning_rate=0.01, low=-2, high=2): for size in layer_sizes: assert size > 0 # Initialize typed layer sizes list. typed_layer_sizes = typed.List() for size in layer_sizes: typed_layer_sizes.append(size) # Initialie typed layer activation method strings list. prototype = types.FunctionType(types.float64[:, ::1](types.float64[:, ::1], types.boolean)) typed_layer_activations = typed.List.empty_list(prototype) for activation in layer_activations: typed_layer_activations.append(activation) # Initialize typed recurrent layers. typed_recurrent_layers = typed.List() for val in recurrent_layers: typed_recurrent_layers.append(val) # Initialize weights between every neuron in all adjacent layers. typed_weights = typed.List() for i in range(1, len(layer_sizes)): typed_weights.append( np.random.uniform(low, high, (layer_sizes[i - 1], layer_sizes[i]))) # Initialize biases for every neuron in all layers typed_biases = typed.List() for i in range(1, len(layer_sizes)): typed_biases.append(np.random.uniform(low, high, (layer_sizes[i], 1))) # Initialize empty list of output of every neuron in all layers. typed_layer_outputs = typed.List() for i in range(len(layer_sizes)): typed_layer_outputs.append(np.zeros((layer_sizes[i], 1))) typed_learning_rate = learning_rate return NeuralNetwork(typed_layer_sizes, typed_layer_activations, typed_recurrent_layers, typed_weights, typed_biases, typed_layer_outputs, typed_learning_rate)
def simulate_battle(M, terrain, max_step: int = 100, ret_frames: bool = True): """ Given a Numpy Matrix of units, simulate a fight. This uses a matrix M which is **heterogenous**; that is to say that it has named columns [pos, target, hp, dpos, team, group] which helps interpretability at the cost of some processing time. Columns in M are '("team", np.uint8), ("utype", np.uint8), ("pos", np.float32, 2), ("hp", np.float32), ("armor", np.float32), ("range", np.float32), ("speed", np.float32), ("acc", np.float32), ("dodge", np.float32), ("dmg", np.float32), ("target", np.int32), ("group", np.uint8)' Parameters -------- M : np.ndarray (units, ) A heterogenous matrix containing data values for units terrain : bsm.Terrain object Terrain object containing the bounds. max_step : int The maximum number of steps ret_frames : bool If True, save each frame, else, return the allegiance that is victorious. Returns ------- frames : np.ndarray (frame, :) Each frame is a numpy.ndmatrix. """ # define teams. teams = np.unique(M["team"]) # unpack bounds bounds = np.asarray(terrain.bounds_) Z = np.copy(terrain.Z_) # initialise enemy targets enemy_targets = typed.List([np.where((M["team"] != T))[0] for T in teams]) # initilise frames array if returning full set. if ret_frames: frames = np.zeros( (max_step + 1, M.shape[0]), dtype=np.dtype([("x", "f4"), ("y", "f4"), ("target", "u4"), ("hp", "f4"), ("armor", "f4"), ("ddx", "f4"), ("ddy", "f4"), ("team", "u1"), ("utype", "u1") ], align=True) ) t = _step_through_update(M, Z, max_step, teams, enemy_targets, bounds, frames) return frames[:t] else: t = _step_through_noframe(M, Z, max_step, teams, enemy_targets, bounds) return np.array([np.sum(np.logical_and(M["hp"] > 0., M["team"] == T)) for T in teams])
def udt(n, raise_at): lst = typed.List() lst.append(0xbe11) try: appender(lst, n, raise_at) except Exception: return lst else: return lst
def create_dict(seed=0): random.seed(seed) d = random_dict(random.randint(0, 10), random.randint(0, 10)) d[tuple((np.str_("ciao"), np.str_("ciao")))] = 9 d["numba_list"] = typed.List() d["numba_dict"] = typed.Dict() d["callable"] = create_dict d["date"] = date(1994, 12, 12) d["none"] = None return d
def udt(n, raise_at): lst = typed.List() try: for i in range(n): if i == raise_at: raise IndexError lst.append(i) except Exception: return lst else: return lst
def udt(): try: lst = typed.List() print("A") lst.append(0) print("B") lst.append("fda") # invalid type will cause typing error print("C") return lst except Exception: print("D")
def forward_cpu(self, x: tensor) -> tensor: _vol2col(x.host_data.ravel(), self.col.host_data.ravel(), self.batch_size, self.in_channels, self.n_output_plane, self.index_length, nbt.List(self._vol), nbt.List(self._col), nbt.List(self.kernel_size), nbt.List(self.stride), nbt.List(self.padding), nbt.List(self.dilation)) self.cache = [x] return self.col
def args_kws(): yield [*range(10)], 12 yield [x + x / 10.0 for x in range(10)], 19j yield [x * 1j for x in range(10)], -2 yield (1, 2, 3), 9 yield (1, 2, 3j), -0 # uints will likely end up as floats as `start` is signed, so just # test mixed signed ints yield (np.int64(32), np.int32(2), np.int8(3)), np.uint32(7) tl = typed.List(range(5)) yield tl, 100 yield np.ones((5, 5)), 10 * np.ones((5, )) yield ntpl(100, 200), -50 yield ntpl(100, 200j), 9
def args(): yield [*range(10)] yield [x + x / 10.0 for x in range(10)] yield [x * 1j for x in range(10)] yield (1, 2, 3) yield (1, 2, 3j) # uints will likely end up as floats as `start` is signed, so just # test mixed signed ints yield (np.int64(32), np.int32(2), np.int8(3)) tl = typed.List(range(5)) yield tl yield np.ones(5) yield ntpl(100, 200) yield ntpl(100, 200j)
def test_array_comp_with_iter(self): def array_comp(a): l = np.array([x * x for x in a]) return l # with list iterator l = [1, 2, 3, 4, 5] self.check(array_comp, l) # with array iterator self.check(array_comp, np.array(l)) # with tuple iterator (issue #7394) self.check(array_comp, tuple(l)) # with typed.List iterator (issue #6550) self.check(array_comp, typed.List(l))
def test_isinstance_numba_types(self): # This makes use of type aliasing between python scalars and NumPy # scalars, see also test_numba_types() pyfunc = isinstance_usecase_numba_types cfunc = jit(nopython=True)(pyfunc) inputs = ((types.int32(1), 'int32'), (types.int64(2), 'int64'), (types.float32(3.0), 'float32'), (types.float64(4.0), 'float64'), (types.complex64(5j), 'no match'), (typed.List([1, 2]), 'typed list'), (typed.Dict.empty(types.int64, types.int64), 'typed dict')) for inpt, expected in inputs: got = cfunc(inpt) self.assertEqual(expected, got)
def backward_cpu(self): x = self.cache[0] tmp = x.gradient.host_data.ravel() col = self.col.gradient.host_data.ravel() _col2vol(tmp, col, self.batch_size, self.in_channels, self.n_output_plane, self.index_length, nbt.List(self._vol), nbt.List(self._col), nbt.List(self.kernel_size), nbt.List(self.stride), nbt.List(self.padding), nbt.List(self.dilation)) x.gradient.host_data = tmp.reshape(x.shape) self.col = zeros([self.n_output_plane, self.output_length]) return x
def __init__(self, egg_no, larva_no, pupa_no, naive_no, adult_no, Temps): self.egg_numbers = typed.List([egg_no]) self.larva_numbers = typed.List([larva_no]) self.pupa_numbers = typed.List([pupa_no]) self.naive_numbers = typed.List([naive_no]) self.adult_numbers = typed.List([adult_no]) self.total_numbers = typed.List( [egg_no + larva_no + pupa_no + naive_no + adult_no]) self.Temps = typed.List(Temps) self.T = self.Temps[0] mosquitos = [Mosquito('egg', self.T, 0.0) for i in range(egg_no)] mosquitos += [Mosquito('larva', self.T, 0.0) for i in range(larva_no)] mosquitos += [Mosquito('pupa', self.T, 0.0) for i in range(pupa_no)] mosquitos += [Mosquito('naive', self.T, 0.0) for i in range(naive_no)] mosquitos += [Mosquito('adult', self.T, 0.0) for i in range(adult_no)] self.mosquitos = mosquitos
def lookup_typed(self): """ Numba typed version of `self.lookup` TODO: Unikely to be necessary in future versions of numba (0.47) Returns ------- lookup_typed : numba.typed.List[numba.typed.Dict] """ if self._lookup_typed is None: self._lookup_typed = typed.List() for i in range(self.n_illuminations): self._lookup_typed.append( typed.Dict.empty(key_type=types.unicode_type, value_type=types.intp)) for key, value in self.lookup[i].items(): self._lookup_typed[i][key] = value return self._lookup_typed
def apply(self, *charges): """ Create charge histogram and fit it with a model of the spectrum Resulting parameters values are found in `self.fit_result_values` Parameters ---------- charges : list[ndarray] A list of the charges to fit. Should have a length equal to the self.n_illuminations. """ assert len(charges) == self.n_illuminations self.charge_hist_y = [] self.charge_hist_y_typed = typed.List() for i in range(self.n_illuminations): hist, edges = np.histogram(charges[i], bins=self.n_bins, range=self.range) between = (edges[1:] + edges[:-1]) / 2 self.charge_hist_x = between.astype(np.float32) self.charge_hist_y.append(hist.astype(np.float32)) self.charge_hist_y_typed.append(hist.astype(np.float32)) self.charge_hist_edges = edges.astype(np.float32) m0 = iminuit.Minuit(self._minimize_function, **self.parameters.minuit_kwargs, print_level=0, pedantic=False, throw_nan=True, errordef=1, forced_parameters=self.parameters.parameter_names) m0.migrad() self.fit_result_values = m0.values with warnings.catch_warnings(): warnings.simplefilter('ignore', HesseFailedWarning) m0.hesse() self.fit_result_errors = m0.errors
def getMatches(self, searchString): """ Iterates through search string finding minmers in searchString and yields their list of minmer occurrences in targetString, each as a pair of (x, (y,)*N), where x is the index in searchString and y is an occurrence in targetString. For example if k = 2 and w = 4 and targetString = "GATTACATTT" and searchString = "GATTTAC" then self.minimizerMap = { "AT":(1,6), "AC":(4,) } and getMatches will yield the following sequence: (1, (1,6)), (5, (4,)) You will need to use the "yield" keyword """ # Code to complete - you are free to define additional functions # Track minmers already returned by the generator to avoid re-processing yieldedSites = typed.List.empty_list(types.int64) # list() # print("getMatches()", searchString) # print("getMatches() with string {0}".format(searchString)) for i in range(len(searchString) - self.w + 1): # Iterate each possible window of width w in the targetString minmer = "" # Store each read minmer to be iterated via the generator sites = typed.List.empty_list(types.int64) for j in range(self.w - self.k + 1): # Then iterate each possible kmer of length k in the window w candidateMinmer = searchString[i + j:i + j + self.k] if not minmer or candidateMinmer < minmer: minmer = candidateMinmer sites = typed.List([int(i + j)]) # print(typeof(self.minimizerMap.get(minmer))) # print(len(self.minimizerMap.get(minmer))) # yield (sites[0], self.minimizerMap[minmer]) # except Exception as e: # print(e) # print("getMatches()", sites) if sites[0] not in yieldedSites and minmer in self.minimizerMap: # self.minimizerMap.get(minmer): yieldedSites.append(sites[0]) yield (sites[0], self.minimizerMap[minmer])
def _to_typed_list(iterable): l = typed.List() for i in iterable: l.append(i) return l
def __init__(self, targetString, w, k, t): """ The target string is a string/array of form "[ACGT]*". Stores the lexicographically smallest k-mer in each window of length w, such that w >= k positions. This smallest k-mer is termed a minmer. If a minmer occurs in the target sequence more than t times as a minmer then it is omitted from the index, i.e. if the given minmer (kmer) is a minmer in more than t different locations in the target string. Note, a minmer may be the minmer for more than t distinct windows and not be pruned, we remove minmers only if they have more than t distinct occurrences as minmers in the sequence. """ self.targetString = targetString self.w = w self.k = k self.t = t # If a minmer occurs more than t times then its entry is removed from the index # This is a heuristic to remove repetitive minmers that would create many spurious alignments between # repeats # Hash of minmers to query locations, stored as a map whose keys # are minmers and whose values are lists of the start indexes of # occurrences of the corresponding minmer in the targetString, # sorted in ascending order of index in the targetString. # # For example if k = 2 and w = 4 and targetString = "GATTACATTT" # # GATTACATTT # GATT (AT) # ATTA (AT) # TTAC (AC) # TACA (AC) # ACAT (AC) # CATT (AT) # ATTT (AT) # # then self.minimizerMap = { "AT":(1,6), "AC":(4,) } # Declare the container attributes using Numba's pre-defined types self.minimizerMap = typed.Dict.empty(*mm_kv_types) self.minmerOccurrences = typed.Dict.empty(*mo_kv_types) # Code to complete to build index - you are free to define additional functions for i in range(len(targetString) - self.w + 1): # , self.w): # Iterate each possible window of width w in the targetString minmer = "" candidateMinmers = typed.List() # list() for j in range(self.w - self.k + 1): # Then iterate each possible kmer of length k in the window w candidateMinmer = targetString[i + j:i + j + self.k] candidateMinmers.append((candidateMinmer, i + j)) try: # Take the lexico min, if present minmerTuple = min(candidateMinmers) minmer = minmerTuple[0] site = minmerTuple[1] if 'N' not in minmer: try: # Log the minmer occurrences to consider only rare ones self.minmerOccurrences[minmer] += 1 except: # REF: https://numba.pydata.org/numba-doc/latest/user/troubleshoot.html # self.minmerOccurrences[minmer] = [np.float64(x) for x in range(0)] # list() # self.minmerOccurrences[minmer].append(site) self.minmerOccurrences[minmer] = 1 if self.minmerOccurrences[minmer] > self.t: # Exclude minmers that are present in more than t sites try: del self.minimizerMap[minmer] except: # CITATION: https://stackoverflow.com/questions/19522990/python-catch-exception-and-continue-try-block pass else: #continue # print('Excluded minmer:', minmer) pass except: # Nothing to add if the candidate list had no min # continue pass try: if 'N' not in minmer and site not in self.minimizerMap[minmer]: # Prevent duplicate sites from adjacent windows self.minimizerMap[minmer].append(site) except: if 'N' not in minmer: self.minimizerMap[minmer] = typed.List([site])
def typed_list(l): tl = typed.List() for e in l: tl.append(e) return tl
def process_html( html: str, store_url: str, html_path: str, full_html_path: str, json_path: str, title: Optional[str] = None, image: Optional[str] = None, ) -> Optional[str]: try: soup = BeautifulSoup(html, "lxml") data = {"path": html_path} if image is None: # Let's see if header has img metadata og_img = soup.find("meta", attrs={"property": "og:image"}) twitter_img = soup.find("meta", attrs={"name": "twitter:image"}) if og_img is None and twitter_img is None: # Header has no img metadata. og_img_idx = html.find("og:image") if og_img_idx != -1: idx_s = og_img_idx + 11 idx_e = html.find('"', idx_s) if idx_e == -1: idx_e = html.find("'", idx_s) image = html[og_img_idx + 11:idx_e] else: twitter_img_idx = html.find('name="twitter:image"') if twitter_img_idx != -1: idx_s = og_img_idx + 16 idx_e = html.find('"', idx_s) if idx_e == -1: idx_e = html.find("'", idx_s) image = html[og_img_idx + 11:idx_e] else: return "no_image_metadata" else: if og_img is not None: image = og_img.attrs["content"] else: image = twitter_img.attrs["content"] data["image"] = image description = soup.find("meta", attrs={"property": "og:description"}) if description is None: description = soup.find("meta", attrs={"name": "twitter:description"}) if description is None: description = soup.find("meta", attrs={"name": "description"}) data["description"] = description.attrs["content"].strip().replace( "\n", " ") found_title = soup.find("meta", attrs={ "property": "og:title" }).attrs["content"] if found_title is None: found_title = soup.find("meta", attrs={ "name": "twitter:title" }).attrs["content"] if found_title is None: found_title = soup.find("title").text if found_title is None and title is None: return "no_title_metadata" if found_title is not None: title = found_title final_html_with_body = process_tree(html) title_arr = typed.List(title.split()) extracted_text, unbranded_title = extract_all_text( final_html_with_body, title_arr) final_html = get_rid_of_the_body(final_html_with_body) if unbranded_title: data["title"] = unbranded_title else: data["title"] = title data["extracted_text"] = extracted_text data["store_url"] = store_url asyncio.run(save_json(json_path, data)) asyncio.run(save_html(html_path, final_html)) asyncio.run(save_html(full_html_path, html)) print(f"Finished processing {html_path}.") return None except Exception: print(f"Failed at processing {html_path}.") return None
def numba_arange_ranges(ranges): expanded = nba_typed.List() for t in prange(len(ranges)): e = np.arange(ranges[t]) expanded.append(e) return expanded