def _backward_cpu(self, x: tensor, y: tensor):
dx = x.gradient
npdx = x.gradient.host_data.ravel()
dcol = y.gradient.host_data.ravel()
_col2vol(npdx, dcol, self.batch_size, self.in_channels,
self.n_output_plane, self.index_length, nbt.List(self._vol), nbt.List(self._col),
nbt.List(self.kernel_size), nbt.List(self.stride), nbt.List(self.padding), nbt.List(self.dilation))
dx.host_data = npdx.reshape(x.shape)
return dx
def make_empty_numba_list():
_nb_a_list = typed.List()
# add variable, to tell numba the type, then remove it for the benchmark
dummy = typed.List([0.01])
_nb_a_list.append(dummy)
_nb_a_list.pop(0)
return _nb_a_list
def get_pointcloud(path):
# converting point cloud data to numba type
ans = typed.List()
with open(path, 'rb') as f:
plydata = plyfile.PlyData.read(f)
plydata = np.asarray(plydata.elements[1].data)
for i in range(len(plydata)):
raw = typed.List()
[raw.append(types.float64(n)) for n in plydata[i]]
ans.append(raw)
return ans
def make_list(a_list):
for i in range(10**4):
new_list = typed.List(float64)
for j in range(10**4):
new_list.append(0.01)
a_list.append(new_list)
return a_list
def make_neural_network(layer_sizes,
layer_activations,
recurrent_layers,
learning_rate=0.01,
low=-2,
high=2):
for size in layer_sizes:
assert size > 0
# Initialize typed layer sizes list.
typed_layer_sizes = typed.List()
for size in layer_sizes:
typed_layer_sizes.append(size)
# Initialie typed layer activation method strings list.
prototype = types.FunctionType(types.float64[:, ::1](types.float64[:, ::1],
types.boolean))
typed_layer_activations = typed.List.empty_list(prototype)
for activation in layer_activations:
typed_layer_activations.append(activation)
# Initialize typed recurrent layers.
typed_recurrent_layers = typed.List()
for val in recurrent_layers:
typed_recurrent_layers.append(val)
# Initialize weights between every neuron in all adjacent layers.
typed_weights = typed.List()
for i in range(1, len(layer_sizes)):
typed_weights.append(
np.random.uniform(low, high, (layer_sizes[i - 1], layer_sizes[i])))
# Initialize biases for every neuron in all layers
typed_biases = typed.List()
for i in range(1, len(layer_sizes)):
typed_biases.append(np.random.uniform(low, high, (layer_sizes[i], 1)))
# Initialize empty list of output of every neuron in all layers.
typed_layer_outputs = typed.List()
for i in range(len(layer_sizes)):
typed_layer_outputs.append(np.zeros((layer_sizes[i], 1)))
typed_learning_rate = learning_rate
return NeuralNetwork(typed_layer_sizes, typed_layer_activations,
typed_recurrent_layers, typed_weights, typed_biases,
typed_layer_outputs, typed_learning_rate)
def simulate_battle(M,
terrain,
max_step: int = 100,
ret_frames: bool = True):
"""
Given a Numpy Matrix of units, simulate a fight.
This uses a matrix M which is **heterogenous**; that is to say that
it has named columns [pos, target, hp, dpos, team, group] which
helps interpretability at the cost of some processing time.
Columns in M are '("team", np.uint8), ("utype", np.uint8), ("pos", np.float32, 2), ("hp", np.float32),
("armor", np.float32), ("range", np.float32), ("speed", np.float32), ("acc", np.float32),
("dodge", np.float32), ("dmg", np.float32), ("target", np.int32),
("group", np.uint8)'
Parameters
--------
M : np.ndarray (units, )
A heterogenous matrix containing data values for units
terrain : bsm.Terrain object
Terrain object containing the bounds.
max_step : int
The maximum number of steps
ret_frames : bool
If True, save each frame, else, return the allegiance that is victorious.
Returns
-------
frames : np.ndarray (frame, :)
Each frame is a numpy.ndmatrix.
"""
# define teams.
teams = np.unique(M["team"])
# unpack bounds
bounds = np.asarray(terrain.bounds_)
Z = np.copy(terrain.Z_)
# initialise enemy targets
enemy_targets = typed.List([np.where((M["team"] != T))[0] for T in teams])
# initilise frames array if returning full set.
if ret_frames:
frames = np.zeros(
(max_step + 1, M.shape[0]),
dtype=np.dtype([("x", "f4"), ("y", "f4"), ("target", "u4"),
("hp", "f4"), ("armor", "f4"), ("ddx", "f4"), ("ddy", "f4"),
("team", "u1"), ("utype", "u1")
], align=True)
)
t = _step_through_update(M, Z, max_step,
teams, enemy_targets, bounds,
frames)
return frames[:t]
else:
t = _step_through_noframe(M, Z, max_step, teams, enemy_targets,
bounds)
return np.array([np.sum(np.logical_and(M["hp"] > 0., M["team"] == T)) for T in teams])
def udt(n, raise_at):
lst = typed.List()
lst.append(0xbe11)
try:
appender(lst, n, raise_at)
except Exception:
return lst
else:
return lst
def create_dict(seed=0):
random.seed(seed)
d = random_dict(random.randint(0, 10), random.randint(0, 10))
d[tuple((np.str_("ciao"), np.str_("ciao")))] = 9
d["numba_list"] = typed.List()
d["numba_dict"] = typed.Dict()
d["callable"] = create_dict
d["date"] = date(1994, 12, 12)
d["none"] = None
return d
def udt(n, raise_at):
lst = typed.List()
try:
for i in range(n):
if i == raise_at:
raise IndexError
lst.append(i)
except Exception:
return lst
else:
return lst
def udt():
try:
lst = typed.List()
print("A")
lst.append(0)
print("B")
lst.append("fda") # invalid type will cause typing error
print("C")
return lst
except Exception:
print("D")
def forward_cpu(self, x: tensor) -> tensor:
_vol2col(x.host_data.ravel(), self.col.host_data.ravel(),
self.batch_size, self.in_channels, self.n_output_plane,
self.index_length, nbt.List(self._vol), nbt.List(self._col),
nbt.List(self.kernel_size), nbt.List(self.stride),
nbt.List(self.padding), nbt.List(self.dilation))
self.cache = [x]
return self.col
def args_kws():
yield [*range(10)], 12
yield [x + x / 10.0 for x in range(10)], 19j
yield [x * 1j for x in range(10)], -2
yield (1, 2, 3), 9
yield (1, 2, 3j), -0
# uints will likely end up as floats as `start` is signed, so just
# test mixed signed ints
yield (np.int64(32), np.int32(2), np.int8(3)), np.uint32(7)
tl = typed.List(range(5))
yield tl, 100
yield np.ones((5, 5)), 10 * np.ones((5, ))
yield ntpl(100, 200), -50
yield ntpl(100, 200j), 9
def args():
yield [*range(10)]
yield [x + x / 10.0 for x in range(10)]
yield [x * 1j for x in range(10)]
yield (1, 2, 3)
yield (1, 2, 3j)
# uints will likely end up as floats as `start` is signed, so just
# test mixed signed ints
yield (np.int64(32), np.int32(2), np.int8(3))
tl = typed.List(range(5))
yield tl
yield np.ones(5)
yield ntpl(100, 200)
yield ntpl(100, 200j)
def test_array_comp_with_iter(self):
def array_comp(a):
l = np.array([x * x for x in a])
return l
# with list iterator
l = [1, 2, 3, 4, 5]
self.check(array_comp, l)
# with array iterator
self.check(array_comp, np.array(l))
# with tuple iterator (issue #7394)
self.check(array_comp, tuple(l))
# with typed.List iterator (issue #6550)
self.check(array_comp, typed.List(l))
def test_isinstance_numba_types(self):
# This makes use of type aliasing between python scalars and NumPy
# scalars, see also test_numba_types()
pyfunc = isinstance_usecase_numba_types
cfunc = jit(nopython=True)(pyfunc)
inputs = ((types.int32(1), 'int32'), (types.int64(2), 'int64'),
(types.float32(3.0), 'float32'), (types.float64(4.0),
'float64'),
(types.complex64(5j), 'no match'), (typed.List([1, 2]),
'typed list'),
(typed.Dict.empty(types.int64, types.int64), 'typed dict'))
for inpt, expected in inputs:
got = cfunc(inpt)
self.assertEqual(expected, got)
def backward_cpu(self):
x = self.cache[0]
tmp = x.gradient.host_data.ravel()
col = self.col.gradient.host_data.ravel()
_col2vol(tmp, col, self.batch_size, self.in_channels,
self.n_output_plane, self.index_length, nbt.List(self._vol),
nbt.List(self._col), nbt.List(self.kernel_size),
nbt.List(self.stride), nbt.List(self.padding),
nbt.List(self.dilation))
x.gradient.host_data = tmp.reshape(x.shape)
self.col = zeros([self.n_output_plane, self.output_length])
return x
def __init__(self, egg_no, larva_no, pupa_no, naive_no, adult_no, Temps):
self.egg_numbers = typed.List([egg_no])
self.larva_numbers = typed.List([larva_no])
self.pupa_numbers = typed.List([pupa_no])
self.naive_numbers = typed.List([naive_no])
self.adult_numbers = typed.List([adult_no])
self.total_numbers = typed.List(
[egg_no + larva_no + pupa_no + naive_no + adult_no])
self.Temps = typed.List(Temps)
self.T = self.Temps[0]
mosquitos = [Mosquito('egg', self.T, 0.0) for i in range(egg_no)]
mosquitos += [Mosquito('larva', self.T, 0.0) for i in range(larva_no)]
mosquitos += [Mosquito('pupa', self.T, 0.0) for i in range(pupa_no)]
mosquitos += [Mosquito('naive', self.T, 0.0) for i in range(naive_no)]
mosquitos += [Mosquito('adult', self.T, 0.0) for i in range(adult_no)]
self.mosquitos = mosquitos
def lookup_typed(self):
"""
Numba typed version of `self.lookup`
TODO: Unikely to be necessary in future versions of numba (0.47)
Returns
-------
lookup_typed : numba.typed.List[numba.typed.Dict]
"""
if self._lookup_typed is None:
self._lookup_typed = typed.List()
for i in range(self.n_illuminations):
self._lookup_typed.append(
typed.Dict.empty(key_type=types.unicode_type,
value_type=types.intp))
for key, value in self.lookup[i].items():
self._lookup_typed[i][key] = value
return self._lookup_typed
def apply(self, *charges):
"""
Create charge histogram and fit it with a model of the spectrum
Resulting parameters values are found in `self.fit_result_values`
Parameters
----------
charges : list[ndarray]
A list of the charges to fit. Should have a length equal to the
self.n_illuminations.
"""
assert len(charges) == self.n_illuminations
self.charge_hist_y = []
self.charge_hist_y_typed = typed.List()
for i in range(self.n_illuminations):
hist, edges = np.histogram(charges[i],
bins=self.n_bins,
range=self.range)
between = (edges[1:] + edges[:-1]) / 2
self.charge_hist_x = between.astype(np.float32)
self.charge_hist_y.append(hist.astype(np.float32))
self.charge_hist_y_typed.append(hist.astype(np.float32))
self.charge_hist_edges = edges.astype(np.float32)
m0 = iminuit.Minuit(self._minimize_function,
**self.parameters.minuit_kwargs,
print_level=0,
pedantic=False,
throw_nan=True,
errordef=1,
forced_parameters=self.parameters.parameter_names)
m0.migrad()
self.fit_result_values = m0.values
with warnings.catch_warnings():
warnings.simplefilter('ignore', HesseFailedWarning)
m0.hesse()
self.fit_result_errors = m0.errors
def getMatches(self, searchString):
""" Iterates through search string finding minmers in searchString and
yields their list of minmer occurrences in targetString, each as a pair of (x, (y,)*N),
where x is the index in searchString and y is an occurrence in targetString.
For example if k = 2 and w = 4 and targetString = "GATTACATTT" and searchString = "GATTTAC"
then self.minimizerMap = { "AT":(1,6), "AC":(4,) }
and getMatches will yield the following sequence:
(1, (1,6)), (5, (4,))
You will need to use the "yield" keyword
"""
# Code to complete - you are free to define additional functions
# Track minmers already returned by the generator to avoid re-processing
yieldedSites = typed.List.empty_list(types.int64) # list()
# print("getMatches()", searchString)
# print("getMatches() with string {0}".format(searchString))
for i in range(len(searchString) - self.w + 1):
# Iterate each possible window of width w in the targetString
minmer = ""
# Store each read minmer to be iterated via the generator
sites = typed.List.empty_list(types.int64)
for j in range(self.w - self.k + 1):
# Then iterate each possible kmer of length k in the window w
candidateMinmer = searchString[i + j:i + j + self.k]
if not minmer or candidateMinmer < minmer:
minmer = candidateMinmer
sites = typed.List([int(i + j)])
# print(typeof(self.minimizerMap.get(minmer)))
# print(len(self.minimizerMap.get(minmer)))
# yield (sites[0], self.minimizerMap[minmer])
# except Exception as e:
# print(e)
# print("getMatches()", sites)
if sites[0] not in yieldedSites and minmer in self.minimizerMap: # self.minimizerMap.get(minmer):
yieldedSites.append(sites[0])
yield (sites[0], self.minimizerMap[minmer])
def _to_typed_list(iterable):
l = typed.List()
for i in iterable:
l.append(i)
return l
def __init__(self, targetString, w, k, t):
""" The target string is a string/array of form "[ACGT]*".
Stores the lexicographically smallest k-mer in each window of length w, such that
w >= k positions. This smallest k-mer is termed a minmer.
If a minmer occurs in the target sequence more than t times as a minmer then it is
omitted from the index, i.e. if the given minmer (kmer) is a minmer
in more than t different locations in the target string. Note, a minmer may be the
minmer for more than t distinct windows and not be pruned, we remove minmers
only if they have more than t distinct occurrences as minmers in the sequence.
"""
self.targetString = targetString
self.w = w
self.k = k
self.t = t # If a minmer occurs more than t times then its entry is removed from the index
# This is a heuristic to remove repetitive minmers that would create many spurious alignments between
# repeats
# Hash of minmers to query locations, stored as a map whose keys
# are minmers and whose values are lists of the start indexes of
# occurrences of the corresponding minmer in the targetString,
# sorted in ascending order of index in the targetString.
#
# For example if k = 2 and w = 4 and targetString = "GATTACATTT"
#
# GATTACATTT
# GATT (AT)
# ATTA (AT)
# TTAC (AC)
# TACA (AC)
# ACAT (AC)
# CATT (AT)
# ATTT (AT)
#
# then self.minimizerMap = { "AT":(1,6), "AC":(4,) }
# Declare the container attributes using Numba's pre-defined types
self.minimizerMap = typed.Dict.empty(*mm_kv_types)
self.minmerOccurrences = typed.Dict.empty(*mo_kv_types)
# Code to complete to build index - you are free to define additional functions
for i in range(len(targetString) - self.w + 1): # , self.w):
# Iterate each possible window of width w in the targetString
minmer = ""
candidateMinmers = typed.List() # list()
for j in range(self.w - self.k + 1):
# Then iterate each possible kmer of length k in the window w
candidateMinmer = targetString[i + j:i + j + self.k]
candidateMinmers.append((candidateMinmer, i + j))
try:
# Take the lexico min, if present
minmerTuple = min(candidateMinmers)
minmer = minmerTuple[0]
site = minmerTuple[1]
if 'N' not in minmer:
try:
# Log the minmer occurrences to consider only rare ones
self.minmerOccurrences[minmer] += 1
except:
# REF: https://numba.pydata.org/numba-doc/latest/user/troubleshoot.html
# self.minmerOccurrences[minmer] = [np.float64(x) for x in range(0)] # list()
# self.minmerOccurrences[minmer].append(site)
self.minmerOccurrences[minmer] = 1
if self.minmerOccurrences[minmer] > self.t:
# Exclude minmers that are present in more than t sites
try:
del self.minimizerMap[minmer]
except:
# CITATION: https://stackoverflow.com/questions/19522990/python-catch-exception-and-continue-try-block
pass
else:
#continue
# print('Excluded minmer:', minmer)
pass
except:
# Nothing to add if the candidate list had no min
# continue
pass
try:
if 'N' not in minmer and site not in self.minimizerMap[minmer]:
# Prevent duplicate sites from adjacent windows
self.minimizerMap[minmer].append(site)
except:
if 'N' not in minmer:
self.minimizerMap[minmer] = typed.List([site])
def typed_list(l):
tl = typed.List()
for e in l:
tl.append(e)
return tl
def process_html(
html: str,
store_url: str,
html_path: str,
full_html_path: str,
json_path: str,
title: Optional[str] = None,
image: Optional[str] = None,
) -> Optional[str]:
try:
soup = BeautifulSoup(html, "lxml")
data = {"path": html_path}
if image is None:
# Let's see if header has img metadata
og_img = soup.find("meta", attrs={"property": "og:image"})
twitter_img = soup.find("meta", attrs={"name": "twitter:image"})
if og_img is None and twitter_img is None:
# Header has no img metadata.
og_img_idx = html.find("og:image")
if og_img_idx != -1:
idx_s = og_img_idx + 11
idx_e = html.find('"', idx_s)
if idx_e == -1:
idx_e = html.find("'", idx_s)
image = html[og_img_idx + 11:idx_e]
else:
twitter_img_idx = html.find('name="twitter:image"')
if twitter_img_idx != -1:
idx_s = og_img_idx + 16
idx_e = html.find('"', idx_s)
if idx_e == -1:
idx_e = html.find("'", idx_s)
image = html[og_img_idx + 11:idx_e]
else:
return "no_image_metadata"
else:
if og_img is not None:
image = og_img.attrs["content"]
else:
image = twitter_img.attrs["content"]
data["image"] = image
description = soup.find("meta", attrs={"property": "og:description"})
if description is None:
description = soup.find("meta",
attrs={"name": "twitter:description"})
if description is None:
description = soup.find("meta", attrs={"name": "description"})
data["description"] = description.attrs["content"].strip().replace(
"\n", " ")
found_title = soup.find("meta", attrs={
"property": "og:title"
}).attrs["content"]
if found_title is None:
found_title = soup.find("meta", attrs={
"name": "twitter:title"
}).attrs["content"]
if found_title is None:
found_title = soup.find("title").text
if found_title is None and title is None:
return "no_title_metadata"
if found_title is not None:
title = found_title
final_html_with_body = process_tree(html)
title_arr = typed.List(title.split())
extracted_text, unbranded_title = extract_all_text(
final_html_with_body, title_arr)
final_html = get_rid_of_the_body(final_html_with_body)
if unbranded_title:
data["title"] = unbranded_title
else:
data["title"] = title
data["extracted_text"] = extracted_text
data["store_url"] = store_url
asyncio.run(save_json(json_path, data))
asyncio.run(save_html(html_path, final_html))
asyncio.run(save_html(full_html_path, html))
print(f"Finished processing {html_path}.")
return None
except Exception:
print(f"Failed at processing {html_path}.")
return None
def numba_arange_ranges(ranges):
expanded = nba_typed.List()
for t in prange(len(ranges)):
e = np.arange(ranges[t])
expanded.append(e)
return expanded
