def parse_packet(packet):
version = bin_to_dec(packet[:3])
type_id = bin_to_dec(packet[3:6])
# If Literal
if type_id == 4:
return get_literal_value(packet[6:])
# If operator
sub_packets = parse_operator(packet[6:])
values = [parse_packet(sub_packet) for sub_packet in sub_packets]
if type_id == 0:
return sum(values)
if type_id == 1:
return prod(values)
if type_id == 2:
return min(values)
if type_id == 3:
return max(values)
if type_id == 5:
return 1 if values[0] > values[1] else 0
if type_id == 6:
return 1 if values[0] < values[1] else 0
if type_id == 7:
return 1 if values[0] == values[1] else 0
def bin_cardinality(signal, M, D):
'''
Computes delayed WHT observations and declares cardinality based on that.
2 is a stand-in for any cardinality > 1. For debugging purposes.
Arguments
---------
signal : InputSignal
The input signal object.
M : numpy.ndarray, shape (n, b)
The subsampling matrix; takes on binary values.
D : numpy.ndarray of ints, shape (num_delays, n).
The delays matrix.
Returns
-------
cardinality : numpy.ndarray
0 or 1 if the bin is a zeroton or singleton resp.; 2 if multiton.
singleton_indices : list
A list (in decimal form for compactness) of the k values of the singletons.
Length matches the number of 1s in cardinality.
'''
b = M.shape[1]
U = compute_delayed_wht(signal, M, D)
cardinality = np.ones((signal.n, ), dtype=np.int) # vector of indicators
singleton_indices = []
cutoff = 2 * signal.noise_sd**2 * (2**(signal.n - b)) * D.shape[0]
if signal.noise_sd > 0:
K = binary_ints(signal.n)
S = (-1)**(D @ K)
for i, col in enumerate(U.T):
sgn = 1
print("Column: ", col)
# <col, col> = |col|^2 = |U|^2
if np.inner(col, col) <= cutoff:
cardinality[i] = 0
else:
if signal.noise_sd == 0:
k = singleton_detection_noiseless(col)
else:
selection = np.where(
[bin_to_dec(row) == i for row in (M.T.dot(K)).T])[0]
k, sgn = singleton_detection(col,
method="mle",
selection=selection,
S_slice=S[:, selection],
n=signal.n)
rho = np.mean(np.abs(col))
residual = col - sgn * rho * (-1)**np.dot(D, k)
print("Residual: ", residual)
if np.inner(residual, residual) > cutoff:
cardinality[i] = 2
else:
singleton_indices.append(bin_to_dec(k))
return cardinality, singleton_indices
def parse_packet(packet):
version = bin_to_dec(packet[:3])
type_id = bin_to_dec(packet[3:6])
# If Literal
if type_id == 4:
return version
# If operator
sub_packets = parse_operator(packet[6:])
for sub_packet in sub_packets:
version += parse_packet(sub_packet)
return version
def get_literal_value(packet):
rest = packet
result = ""
while True:
current = rest[:5]
result += rest[1:5]
rest = rest[5:]
if current[-5] == "0":
return bin_to_dec(result)
def _transform(self, gene):
if gene in self.genotype_dictionary:
return self.genotype_dictionary[gene]
try:
# maximum depth
depth = 30
gene_index = 0
expr = self._start_node
# for each level
level = 0
while level < depth:
i = 0
new_expr = ''
# parse every character in the expr
while i < len(expr):
found = False
for key in self._grammar:
# if there is a keyword in the grammar, replace it with rule in production
if (expr[i:i + len(key)] == key):
found = True
# count how many transformation possibility exists
possibility = len(self._grammar[key])
# how many binary digit needed to represent the possibilities
digit_needed = utils.bin_digit_needed(possibility)
# if end of gene, then start over from the beginning
if (gene_index + digit_needed) > len(gene):
gene_index = 0
# get part of gene that will be used
used_gene = gene[gene_index:gene_index +
digit_needed]
gene_index = gene_index + digit_needed
rule_index = utils.bin_to_dec(
used_gene) % possibility
new_expr += self._grammar[key][rule_index]
i += len(key) - 1
if not found:
new_expr += expr[i:i + 1]
i += 1
expr = new_expr
level = level + 1
# add to cache for future usage
self.genotype_dictionary[gene] = expr
except:
return ''
return expr
def _transform(self, gene):
if gene in self.genotype_dictionary:
return self.genotype_dictionary[gene]
try:
# maximum depth
depth = 30
gene_index = 0
expr = self._start_node
# for each level
level = 0
while level < depth:
i=0
new_expr = ''
# parse every character in the expr
while i<len(expr):
found = False
for key in self._grammar:
# if there is a keyword in the grammar, replace it with rule in production
if (expr[i:i+len(key)] == key):
found = True
# count how many transformation possibility exists
possibility = len(self._grammar[key])
# how many binary digit needed to represent the possibilities
digit_needed = utils.bin_digit_needed(possibility)
# if end of gene, then start over from the beginning
if(gene_index+digit_needed)>len(gene):
gene_index = 0
# get part of gene that will be used
used_gene = gene[gene_index:gene_index+digit_needed]
gene_index = gene_index + digit_needed
rule_index = utils.bin_to_dec(used_gene) % possibility
new_expr += self._grammar[key][rule_index]
i+= len(key)-1
if not found:
new_expr += expr[i:i+1]
i += 1
expr = new_expr
level = level+1
# add to cache for future usage
self.genotype_dictionary[gene] = expr
except:
return ''
return expr
def subsample_indices(M, d):
'''
Query generator: creates indices for signal subsamples.
Arguments
---------
M : numpy.ndarray, shape (n, b)
The subsampling matrix; takes on binary values.
d : numpy.ndarray, shape (n,)
The subsampling offset; takes on binary values.
Returns
-------
indices : numpy.ndarray, shape (B,)
The (decimal) subsample indices. Mostly for debugging purposes.
'''
L = binary_ints(M.shape[1])
inds_binary = np.mod(np.dot(M, L).T + d, 2).T
return bin_to_dec(inds_binary)
def transform(self, signal, verbose=False, report=False):
'''
Full SPRIGHT encoding and decoding. Implements Algorithms 1 and 2 from [2].
(numbers) in the comments indicate equation numbers in [2].
Arguments
---------
signal : Signal object.
The signal to be transformed / compared to.
verbose : boolean
Whether to print intermediate steps.
Returns
-------
wht : ndarray
The WHT constructed by subsampling and peeling.
'''
# check the condition for p_failure > eps
# upper bound on the number of peeling rounds, error out after that point
num_peeling = 0
result = []
wht = np.zeros_like(signal.signal_t)
b = get_b(signal, method=self.query_method)
peeling_max = 2 ** b
Ms = get_Ms(signal.n, b, method=self.query_method)
Us, Ss = [], []
singletons = {}
multitons = []
if report:
used = set()
if self.delays_method is not "nso":
num_delays = signal.n + 1
else:
num_delays = signal.n * int(np.log2(signal.n)) # idk
K = binary_ints(signal.n)
# subsample, make the observation [U] and offset signature [S] matrices
for M in Ms:
D = get_D(signal.n, method=self.delays_method, num_delays=num_delays)
if verbose:
print("------")
print("a delay matrix")
print(D)
U, used_i = compute_delayed_wht(signal, M, D)
Us.append(U)
Ss.append((-1) ** (D @ K)) # offset signature matrix
if report:
used = used.union(used_i)
cutoff = 2 * signal.noise_sd ** 2 * (2 ** (signal.n - b)) * num_delays # noise threshold
if verbose:
print('cutoff: {}'.format(cutoff))
# K is the binary representation of all integers from 0 to 2 ** n - 1.
select_froms = np.array([[bin_to_dec(row) for row in M.T.dot(K).T] for M in Ms])
# `select_froms` is the collection of 'j' values and associated indices
# so that we can quickly choose from the coefficient locations such that M.T @ k = j as in (20)
# example: ball j goes to bin at "select_froms[i][j]"" in stage i
# begin peeling
# index convention for peeling: 'i' goes over all M/U/S values
# i.e. it refers to the index of the subsampling group (zero-indexed - off by one from the paper).
# 'j' goes over all columns of the WHT subsample matrix, going from 0 to 2 ** b - 1.
# e.g. (i, j) = (0, 2) refers to subsampling group 0, and aliased bin 2 (10 in binary)
# which in the example of section 3.2 is the multiton X[0110] + X[1010] + W1[10]
# a multiton will just store the (i, j)s in a list
# a singleton will map from the (i, j)s to the true (binary) values k.
# e.g. the singleton (0, 0), which in the example of section 3.2 is X[0100] + W1[00]
# would be stored as the dictionary entry (0, 0): array([0, 1, 0, 0]).
there_were_multitons = True
while there_were_multitons and num_peeling < peeling_max:
if verbose:
print('-----')
print('the measurement matrix')
for U in Us:
print(U)
# first step: find all the singletons and multitons.
singletons = {} # dictionary from (i, j) values to the true index of the singleton, k.
multitons = [] # list of (i, j) values indicating where multitons are.
for i, (U, S, select_from) in enumerate(zip(Us, Ss, select_froms)):
for j, col in enumerate(U.T):
# note that np.inner(x, x) is used as norm-squared: marginally faster than taking norm and squaring
if np.inner(col, col) > cutoff:
selection = np.where(select_from == j)[0] # pick all the k such that M.T @ k = j
k, sgn = singleton_detection(
col,
method=self.reconstruct_method,
selection=selection,
S_slice=S[:, selection],
n=signal.n
) # find the best fit singleton
k_dec = bin_to_dec(k)
rho = np.dot(S[:,k_dec], col)*sgn/len(col)
residual = col - sgn * rho * S[:,k_dec]
if verbose:
print((i, j), np.inner(residual, residual))
if np.inner(residual, residual) > cutoff:
multitons.append((i, j))
else: # declare as singleton
singletons[(i, j)] = (k, rho, sgn)
if verbose:
print('amplitude: {}'.format(rho))
# all singletons and multitons are discovered
if verbose:
print('singletons:')
for ston in singletons.items():
print("\t{0} {1}\n".format(ston, bin_to_dec(ston[1][0])))
print("Multitons : {0}\n".format(multitons))
# raise RuntimeError("stop")
# WARNING: this is not a correct thing to do
# in the last iteration of peeling, everything will be singletons and there
# will be no multitons
if len(multitons) == 0: # no more multitons, and can construct final WHT
there_were_multitons = False
# balls to peel
balls_to_peel = set()
ball_values = {}
ball_sgn = {}
for (i, j) in singletons:
k, rho, sgn = singletons[(i, j)]
ball = bin_to_dec(k)
balls_to_peel.add(ball)
ball_values[ball] = rho
ball_sgn[ball] = sgn
if verbose:
print('these balls will be peeled')
print(balls_to_peel)
# peel
for ball in balls_to_peel:
num_peeling += 1
k = dec_to_bin(ball, signal.n)
potential_peels = [(l, bin_to_dec(M.T.dot(k))) for l, M in enumerate(Ms)]
result.append((k, ball_sgn[ball]*ball_values[ball]))
for peel in potential_peels:
signature_in_stage = Ss[peel[0]][:,ball]
to_subtract = ball_sgn[ball] * ball_values[ball] * signature_in_stage
Us[peel[0]][:,peel[1]] -= to_subtract
if verbose:
print('this is subtracted:')
print(to_subtract)
print("Peeled ball {0} off bin {1}".format(bin_to_dec(k), peel))
loc = set()
for k, value in result: # iterating over (i, j)s
idx = bin_to_dec(k) # converting 'k's of singletons to decimals
loc.add(idx)
if wht[idx] == 0:
wht[idx] = value
else:
wht[idx] = (wht[idx] + value) / 2
# average out noise; e.g. in the example in 3.2, U1[11] and U2[11] are the same singleton,
# so averaging them reduces the effect of noise.
wht /= 2 ** (signal.n - b)
if not report:
return wht
else:
return wht, len(used), loc