def spa_model(plaintext, key):
hd_arr = np.zeros(16)
hw_arr = np.zeros(16)
zo_arr = np.zeros(16)
byte_list = [0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 15, 11, 7]
for byte in byte_list:
print 'byte: ' + str(byte)
plaintext_nth_byte = plaintext[byte]
# XOR the plaintext byte with key byte and put the result through the S-BOX
xored_plaintxt_key_bytes = np.bitwise_xor(plaintext_nth_byte, key[byte])
# Use the Hamming Weight model to calculate the hypothetical power consumption of
# the SBOX operation.
hw_arr[byte] = num_ones(sbox_hex[xored_plaintxt_key_bytes])
# Use the Hamming Distance model
hd_arr[byte] = num_ones(np.bitwise_xor(xored_plaintxt_key_bytes, sbox_hex[xored_plaintxt_key_bytes]))
# Other power model! 0 -> 1 transition
zo_arr[byte] = num_ones(~xored_plaintxt_key_bytes & sbox_hex[xored_plaintxt_key_bytes])
print '%x, %x, %d, %d, %d' %(xored_plaintxt_key_bytes, sbox_hex[xored_plaintxt_key_bytes], hw_arr[byte], hd_arr[byte], zo_arr[byte])
plt.plot(hd_arr, 'r')
plt.plot(hw_arr, 'b')
plt.plot(zo_arr, 'g')
plt.show()
def detect_corruption(self, fname):
"""
single disk corruption detection
:param fname: data name in RAID6 system
:return: corrupted disk index; for p, self.N-2; for q, self.N-1
"""
# all disks, including P, Q
byte_ndarray = self._read_n(fname, self.N)
data_ndarray = byte_ndarray[:-2]
self._logger.info("byte_ndarray=\n{}".format(byte_ndarray))
P = byte_ndarray[-2:-1]
self._logger.info("p={}".format(P))
Q = byte_ndarray[-1]
self._logger.info("Q={}".format(Q))
P_prime = utils.gen_p(data_ndarray, ndim=2)
self._logger.info("P_prime={}".format(P_prime))
Q_prime = utils.gen_q(data_ndarray, ndim=2)
self._logger.info("Q_prime={}".format(Q_prime))
P_star = np.bitwise_xor(P, P_prime)
Q_star = np.bitwise_xor(Q, Q_prime)
P_nonzero = np.count_nonzero(P_star)
Q_nonzero = np.count_nonzero(Q_star)
if P_nonzero == 0 and Q_nonzero == 0:
print("no corruption")
return None
elif P_nonzero == 0 and Q_nonzero != 0:
print("Q corruption")
return self.N - 1
elif P_nonzero != 0 and Q_nonzero == 0:
print("P corruption")
return self.N - 2
else:
index = self._get_corrupted_data_disk(P_star, Q_star)
print("data disk {} corruption".format(index))
return index
def _unmask(self, buf, mask):
# Unmask a frame
if numpy:
plen = len(buf)
pstart = 0
pend = plen
b = c = ''.encode('ascii')
if plen >= 4:
dtype=numpy.dtype('<u4')
if sys.byteorder == 'big':
dtype = dtype.newbyteorder('>')
mask = numpy.frombuffer(mask, dtype, count=1)
data = numpy.frombuffer(buf, dtype, count=int(plen / 4))
#b = numpy.bitwise_xor(data, mask).data
b = numpy.bitwise_xor(data, mask).tostring()
if plen % 4:
dtype=numpy.dtype('B')
if sys.byteorder == 'big':
dtype = dtype.newbyteorder('>')
mask = numpy.frombuffer(mask, dtype, count=(plen % 4))
data = numpy.frombuffer(buf, dtype,
offset=plen - (plen % 4), count=(plen % 4))
c = numpy.bitwise_xor(data, mask).tostring()
return b + c
else:
# Slower fallback
if sys.hexversion < 0x3000000:
mask = [ ord(c) for c in mask ]
data = array.array('B')
data.fromstring(buf)
for i in range(len(data)):
data[i] ^= mask[i % 4]
return data.tostring()
def test_getTraining(self):
test = np.array(Image.open(self.filename), 'uint8')
train = np.array(Image.open(self.filename), 'uint8')
np.bitwise_xor(test,train,test)
image = np.ones(test.shape,test.dtype)
le.getTraining(image,self.filename, self.filename)
assert((test == image).all(), True)
def iota(ain, rnd):
# Initialize empty arrays
aout = np.zeros((5,5,64), dtype = int)
bit = np.zeros(dtype = int, shape = (5,5,64))
rc = np.zeros(dtype = int, shape = 168)
# Linear Feedback Shift Register
w = np.array([1,0,0,0,0,0,0,0], dtype = int)
rc[0] = w[0]
for i in range(1, 7*24):
a = np.bitwise_xor(w[0], w[4])
b = np.bitwise_xor(w[5], w[6])
tail = np.bitwise_xor(a, b)
w = [w[1],w[2],w[3],w[4],w[5],w[6],w[7], tail]
rc[i] = w[0]
# Calculate bits
for l in range(7):
q = pow(2, l) - 1
t = l + 7*rnd
bit[0][0][q] = rc[l + 7*rnd]
# Calculate aout
for i in range(5):
for j in range(5):
for k in range(64):
aout[i][j][k] = np.bitwise_xor(ain[i][j][k], bit[i][j][k])
return aout
def count_sensitive_neighborhood_hash(g):
""" Compute the count sensitive neighborhood hashed
version of a graph.
"""
gnh = g.copy()
g = array_labels_to_str(g)
#iterate over every node in the graph
for node in iter(g.nodes()):
neighbors_labels = [g.node[n]["label"] for n in g.neighbors_iter(node)]
#if node has no neighboors, nh is its own label
if len(neighbors_labels) > 0:
#count number of unique labels
c = Counter(neighbors_labels)
count_weighted_neighbors_labels = []
for label, c in c.iteritems():
label = str_to_array(label)
c_bin = np.array( list(np.binary_repr( c, len(label) ) ), dtype=np.int64 )
label = np.bitwise_xor( label, c_bin)
label = np.roll( label, c )
count_weighted_neighbors_labels.append( label )
x = count_weighted_neighbors_labels[0]
for l in count_weighted_neighbors_labels[1:]:
x = np.bitwise_xor( x, l)
node_label = str_to_array(g.node[node]["label"])
csnh = np.bitwise_xor( np.roll( node_label, 1 ), x )
else:
csnh = str_to_array(g.node[node]["label"])
gnh.node[node]["label"] = csnh
return gnh
def main():
mp.figure(1)
mp.clf()
ax = mp.subplot(211)
#ax = mp.subplot(211)
#x, y = testData()
x, y, flag = jupiter1()
cin = np.arange(len(x))
#import pdb; pdb.set_trace()
idx = noise.singlePointDifferenceSigmaClip(y, 4, initialClip=flag)
mp.plot(cin[~flag],y[~flag], 'k.')
mp.plot(cin[idx], y[idx], 'ro', ms=10)
mp.plot(cin[flag], y[flag], 'gs')
mp.axvline(499, color='b')
mp.subplot(212, sharex=ax)
mp.plot(cin, y - np.roll(y, -1), 'ko-')
mp.axvline(499, color='b')
outliers = np.where(np.bitwise_xor(idx, flag))[0]
outliers = np.bitwise_xor(idx, flag).astype(int)
mp.figure(2)
mp.clf()
mp.plot(np.convolve(outliers, outliers, mode='same'))
def reconciliate(matlabEng, arrDelta, arrData_bin_2, n, k, m, strCoder):
"""
Given the delta, try to deduce data1 via reconciliation
"""
arrMsg_bin_2 = None
arrCodeword_bin_2 = None
if (strCoder == CODER_GOLAY):
n = 23
k = 12
arrMsg_bin_2 = golay.decode(matlabEng,
np.bitwise_xor(arrData_bin_2, arrDelta),
n)
arrCodeword_bin_2 = golay.encode(matlabEng, arrMsg_bin_2, k)
elif (strCoder == CODER_RS):
arrMsg_bin_2 = rs.decode(matlabEng,
np.bitwise_xor(arrData_bin_2, arrDelta),
n, k, m)
arrCodeword_bin_2 = rs.encode(matlabEng, arrMsg_bin_2, n, k, m)
elif (strCoder == CODER_HAMMING):
arrMsg_bin_2 = fec.decode(matlabEng,
np.bitwise_xor(arrData_bin_2, arrDelta),
n, k, strCoder)
arrCodeword_bin_2 = fec.encode(matlabEng, arrMsg_bin_2, n, k)
else:
raise ValueError("Unkown coder")
# deduce data 1 from data 2 + delta
arrDeducedData_bin_1 = np.bitwise_xor(arrCodeword_bin_2, arrDelta)
return arrDeducedData_bin_1
def unmask(buf, f):
s2a = lambda s: [ord(c) for c in s]
s2b = lambda s: s
pstart = f['hlen'] + 4
pend = pstart + f['length']
if numpy:
b = c = s2b('')
if f['length'] >= 4:
mask = numpy.frombuffer(buf, dtype=numpy.dtype('<u4'),
offset=f['hlen'], count=1)
data = numpy.frombuffer(buf, dtype=numpy.dtype('<u4'),
offset=pstart, count=int(f['length'] / 4))
b = numpy.bitwise_xor(data, mask).tostring()
if f['length'] % 4:
mask = numpy.frombuffer(buf, dtype=numpy.dtype('B'),
offset=f['hlen'], count=(f['length'] % 4))
data = numpy.frombuffer(buf, dtype=numpy.dtype('B'),
offset=pend - (f['length'] % 4),
count=(f['length'] % 4))
c = numpy.bitwise_xor(data, mask).tostring()
return b + c
else:
data = array.array('B')
mask = s2a(f['mask'])
data.fromstring(buf[pstart:pend])
for i in range(len(data)):
data[i] ^= mask[i % 4]
return data.tostring()
def sign_change(img):
img = img.reshape(28, 28)
top = numpy.bitwise_xor(img, shift_up(img))
top = [top[:, i].sum() / 255 for i in xrange(img.shape[1])]
side = numpy.bitwise_xor(img, shift_left(img))
side = [side[i, :].sum() / 255 for i in xrange(img.shape[1])]
return numpy.array(top), numpy.array(side)
def chi(ain):
aout = np.zeros((5,5,64), dtype = int) # Initialize empty 5x5x64 array
for i in range(5):
for j in range(5):
for k in range(64):
xor = np.bitwise_xor(ain[(i+1)%5][j][k], 1)
mul = xor * (ain[(i+2)%5][j][k])
aout[i][j][k] = np.bitwise_xor(ain[i][j][k], mul)
return aout
def train_and_test(X_train,Y_train,X_test,Y_test,dtype,alpha,beta_percentile,cv=True): k = X_train.shape[0] n = X_test.shape[0] A = np.zeros(shape=(k,k)) lambd = 1 part1 = np.column_stack((X_train['guarantee_type'], X_train['gender'])) part2 = np.column_stack((X_train['age'], X_train['account_value'], X_train['withdrawal_rate'],X_train['maturity'])) for i in range(k): tmp = np.array([X_train[i],]*k, dtype=dtype) tmp1 = np.column_stack((tmp['guarantee_type'], tmp['gender'])) #copy this row k times tmp2 = np.column_stack((tmp['age'], tmp['account_value'], tmp['withdrawal_rate'],tmp['maturity'])) result1 = lambd*np.sum(np.bitwise_xor(part1, tmp1), axis=1) result2 = np.sum((part2-tmp2)**2,axis=1) A[i] = np.sqrt(result1+result2) # tmp = np.triu(A) # tmp = tmp[tmp!=0] max_dist = A.max() B = np.zeros(shape=(k+1,k+1)) B[k] = B[:,k] = 1 B[k,k] = 0 part1 = np.column_stack((X_test['guarantee_type'], X_test['gender'])) part2 = np.column_stack((X_test['age'], X_test['account_value'], X_test['withdrawal_rate'],X_test['maturity'])) weight_mat = np.zeros((k+1,n)) weight_mat[k] = 1 beta = max_dist*beta_percentile for j in range(k): B[j][:k] = alpha+np.exp(-3*A[j]/beta) tmp = np.array([X_train[j],]*n, dtype=dtype) tmp1 = np.column_stack((tmp['guarantee_type'], tmp['gender'])) #copy this row k times tmp2 = np.column_stack((tmp['age'], tmp['account_value'], tmp['withdrawal_rate'],tmp['maturity'])) result1 = lambd*np.sum(np.bitwise_xor(part1, tmp1), axis=1) result2 = np.sum((part2-tmp2)**2,axis=1) weight_mat[j] = alpha+np.exp(np.sqrt(result1+result2)*(-3/beta)) Y_train = np.append(Y_train, 0) const = np.dot(np.linalg.inv(B),Y_train) Y_hat = np.dot(const, weight_mat) if cv == True: RMSE = np.sqrt(sum((Y_test-Y_hat)**2)/n) return RMSE else: RMSE = np.sqrt(sum((Y_test-Y_hat)**2)/(n+k)) MAD = sum(abs(Y_test-Y_hat))/(n+k) total = sum(Y_hat)+sum(Y_train) benchmark = sum(Y_test)+sum(Y_train) APD = abs(total-benchmark) RPD = APD/benchmark return APD,RPD,MAD,RMSE
def xor_arrays(cls, source, target):
"""
Moved to its own function for profiling purposes.
May want to consider moving out to inline.
The source array will be xored into place into
the target array
Arguments:
source -- Numpy array source array
target -- Numpy array target array
"""
numpy.bitwise_xor(source, target, target)
def key_expansion(self, cipher_key):
if self.is_verbose:
print "Computing key schedule..."
key_schedule = np.copy(cipher_key)
n_k = len(cipher_key)
n_r = n_k + 6
if self.is_verbose:
print
print " After After Rcon XOR"
print " i Previous RotWord SubWord Value w/ Rcon w[i-Nk] Final"
print "=== ======== ======== ======== ======== ======== ======== ========"
for i in range(n_k, N_B * (n_r + 1)):
prev = key_schedule[i - 1]
first = key_schedule[i - n_k]
temp = prev
after_rot_word = None
after_sub_word = None
rcon_val = None
after_xor_rcon = None
if i % n_k == 0:
after_rot_word = rot_word(prev)
after_sub_word = sub_word(after_rot_word)
rcon_val = rcon(i / n_k)
after_xor_rcon = np.bitwise_xor(after_sub_word, rcon_val)
temp = after_xor_rcon
elif n_k > 6 and i % n_k == 4:
after_sub_word = sub_word(prev)
temp = after_sub_word
final = np.bitwise_xor(first, temp)
key_schedule = np.append(key_schedule, final.reshape(1, 4), axis=0)
if self.is_verbose:
print "{:02}: {} {} {} {} {} {} {}".format(
i,
w2s(prev),
w2s(after_rot_word),
w2s(after_sub_word),
w2s(rcon_val),
w2s(after_xor_rcon),
w2s(first),
w2s(final),
)
if self.is_verbose:
print
return key_schedule, n_k, n_r
def expand_key(self, key):
self.round_keys = []
self.round_keys.append(key)
for i in range(1, NUM_ROUNDS+1):
# Always transpose, words are columns and not rows
previous_key = self.round_keys[i-1].transpose()
round_key = np.zeros(key.shape, dtype=np.uint8)
round_key[0] = np.bitwise_xor(self.g(previous_key[3], i),
previous_key[0])
round_key[1] = np.bitwise_xor(round_key[0], previous_key[1])
round_key[2] = np.bitwise_xor(round_key[1], previous_key[2])
round_key[3] = np.bitwise_xor(round_key[2], previous_key[3])
self.round_keys.append(round_key.transpose())
def add_noise(y, x, noise_y_rate, noise_x_rate):
assert(noise_y_rate >= 0 and noise_y_rate <= 1)
assert(noise_x_rate >= 0 and noise_x_rate <= 1)
tmp_y = (y + np.ones_like(y)) / 2
noise_y = np.random.random(y.shape) < noise_y_rate
new_y = np.bitwise_xor(tmp_y, noise_y)
new_y = (2 * new_y) - np.ones_like(new_y)
noise_x = np.random.random(x.shape) < noise_x_rate
new_x = np.bitwise_xor(x, noise_x)
return (new_y, new_x)
def ErrorCorrection(self,syndrome,inBits):
# LUT based error correction
# errVector = find( errorSyndromes == syndrome );
# if numel(errVector)>1
# disp('Help!');
# end
# if isempty(errVector)
# failure = true;
# else
# inBits = bitxor(inBits,errorVectors(errVector(1),:));
# failure = false;
# end
corrected = 0;
if(syndrome):
# Meggitt algorithm, only correct data bits (16), not the parity bits
for ndx in range(16):
# The first (most significant) bit is one
if (np.bitwise_and(syndrome,512)):
# If the first bit is a one and the last 5 (least significant bits)
# are zero, this indicates an error at the current bit (ndx) position
if (np.bitwise_and(syndrome,31) == 0):
# The code can correct bursts up to 5 bits long. Check to see if
# the error is a burst or not. If it isn't a burst, it isn't
# correctable, return immediately with a failure.
tmp = np.bitwise_and(syndrome,480);
if ~(tmp == 480 | tmp == 448 | tmp == 384 | tmp == 256 | tmp == 0):
break;
# The error appears to be a burst error, attempt to correct
inBits[ndx] = np.bitwise_xor(inBits(ndx),1);
# Shift the syndrome
syndrome = np.left_shift(syndrome,1);
else:
# Least significant bits do not indicate the current bit (ndx) is
# an error in a burst. Continue shifting the syndrome and then apply the
# generator polynomial.
syndrome = np.left_shift(syndrome,1);
syndrome = np.bitwise_xor(syndrome,441);
else:
# Not a one at the first (most significant) syndrome bit, applying generator polynomial
# is trivial in this case.
syndrome = np.left_shift(syndrome,1);
# If after this process the syndrome is not zero, there was a
# uncorrectable error.
if (np.bitwise_and(syndrome,1023)==0):
corrected = 1;
return (inBits, corrected);
def ring(x, y, height, thickness, gaussian_width):
"""
Circular ring (annulus) with Gaussian fall-off after the solid ring-shaped region.
"""
radius = height/2.0
half_thickness = thickness/2.0
distance_from_origin = np.sqrt(x**2+y**2)
distance_outside_outer_disk = distance_from_origin - radius - half_thickness
distance_inside_inner_disk = radius - half_thickness - distance_from_origin
ring = 1.0-np.bitwise_xor(np.greater_equal(distance_inside_inner_disk,0.0),
np.greater_equal(distance_outside_outer_disk,0.0))
sigmasq = gaussian_width*gaussian_width
if sigmasq==0.0:
inner_falloff = x*0.0
outer_falloff = x*0.0
else:
with float_error_ignore():
inner_falloff = np.exp(np.divide(-distance_inside_inner_disk*distance_inside_inner_disk, 2.0*sigmasq))
outer_falloff = np.exp(np.divide(-distance_outside_outer_disk*distance_outside_outer_disk, 2.0*sigmasq))
return np.maximum(inner_falloff,np.maximum(outer_falloff,ring))
def decrypt_bytes(bytes_in, key):
# get the function from the cuda_kernel
func = sm.get_function("decipher")
iv = np.array([1,2], dtype=np.uint32)
ha = np.fromstring(hashlib.md5(key).digest(), np.uint32)
output = np.empty_like(bytes_in)
prev_decrypt = iv
length = len(bytes_in)
# assign a number < 1024 for the number of threads for each block
num_threads = 256
# number of blocks is the total length divided by 2, divided for the number of threads per block and + 1 in case of rest
# /2 because in decipher each part of the message is processed in pair
num_blocks = ((length / 2) / num_threads) + 1
# call the function decipher and generate the output [v1,v2] nump array
decipher_output = np.empty(length, dtype=np.uint32)
func(np.int32(32), drv.In(bytes_in), drv.In(ha), drv.InOut(decipher_output), np.int32(length), block=(num_threads,1,1), grid=(num_blocks,1,1))
# now in decipher_output there are all the v0 and v1 couple
i = 0;
while(i < length - 1):
output[i:i+2] = np.bitwise_xor(decipher_output[i:i+2], prev_decrypt)
prev_decrypt = bytes_in[i:i+2]
i += 2
return output
def test(self):
N, D = self.XY_test.shape
Y = self.XY_test[:,-2:]
pred = self.classify()
TP = FP = FN = TN = 0
res = np.empty((N,))
for i in xrange(N):
l1 = pred[i]
l2 = pred[N+i]
#print 'pred',l1,l2
#print 'Y',Y[i,0],Y[i,1]
if l1 == l2 and Y[i,0] == Y[i,1]:
TP += 1
elif l1 == l2 and Y[i,0] != Y[i,1]:
FP += 1
elif l1 != l2 and Y[i,0] == Y[i,1]:
FN += 1
elif l1 != l2 and Y[i,0] != Y[i,1]:
TN += 1
res[i] = 1-np.bitwise_xor(int(np.sign(abs(l1-l2))),int(np.sign(abs(Y[i,0]-Y[i,1]))))
print 'TP:',TP
print 'FP:',FP
print 'FN:',FN
print 'TN:',TN
print 'Accuracy:',float(np.sum(res))/N
sys.stdout.flush()
def execute(self, slot, subindex, roi, result):
assert slot == self.Output, "Unknown output slot: {}".format( slot.name )
if self.SelectedLabel.value == 0:
result[:] = 0
else:
# Can't use writeInto() here because dtypes don't match.
inputLabels = self.Input(roi.start, roi.stop).wait()
# Use two in-place bitwise operations instead of numpy.where
# This avoids the temporary variable created by (inputLabels == x)
#result[:] = numpy.where( inputLabels == self.SelectedLabel.value, 1, 0 )
numpy.bitwise_xor(inputLabels, self.SelectedLabel.value, out=inputLabels) # All
numpy.logical_not(inputLabels, out=inputLabels)
result[:] = inputLabels # Copy from uint32 to uint8
return result
def unmake_packet(whitened_payload_with_crc, whitener_offset=0, dewhitening=True):
"""
Return (payload)
@param whitened_payload_with_crc: string
"""
whitener_offset=0
dewhitening = True
if dewhitening:
i = frombuffer(whitened_payload_with_crc, dtype = byte)
j = frombuffer(random_mask[0:len(whitened_payload_with_crc)], dtype = byte)
try:
payload = (bitwise_xor(i, j)).tostring()
except:
print "Error: receiving arguments do not have equal length!"
len(i)
len(j)
else:
payload = (whitened_payload_with_crc)
if 0:
print "payload_with_crc =", string_to_hex_list(payload_with_crc)
print "ok = %r, len(payload) = %d" % (ok, len(payload))
print "payload =", string_to_hex_list(payload)
return payload
def get_synthetic_clusters_dataset(n_clusters = 4, n_dims = 20, n_training = 1000, n_test = 200,
sparsity = 0.5, flip_noise = 0.1, seed = 3425):
"""
A dataset consisting of clustered binary data with "bit-flip" noise, and the corresponding cluster labels.
This should be trivially solvable by any classifier, and serves as a basic test of whether your classifier is
completely broken or not.
:param n_clusters:
:param n_dims:
:param n_samples_training:
:param n_samples_test:
:param sparsity:
:param flip_noise:
:param seed:
:return:
"""
rng = np.random.RandomState(seed)
labels = rng.randint(n_clusters, size = n_training+n_test) # (n_samples, )
centers = rng.rand(n_clusters, n_dims) < sparsity # (n_samples, n_dims)
input_data = centers[labels]
input_data = np.bitwise_xor(input_data, rng.rand(*input_data.shape) < flip_noise)
return DataSet(
training_set = DataCollection(input_data[:n_training], labels[:n_training]),
test_set = DataCollection(input_data[n_training:], labels[n_training:]),
name = 'Synthetic Clusters Dataset'
)
def k_cross_validation(k, N, X, y, classifier):
indexSet = set([i for i in range(N)])
partitionDict = dict()
for i in range(k):
partitionDict[i] = [np.matrix(np.empty((0, np.size(X, 1)), X[0, 0].dtype)),
np.matrix(np.empty((0, 1), y[0, 0].dtype))]
for _ in range(N):
partitionKey = _ % k
i = random.sample(indexSet, 1)[0]
indexSet.remove(i)
partitionDict[partitionKey][0] = np.append(partitionDict[partitionKey][0], X[i], axis=0)
partitionDict[partitionKey][1] = np.append(partitionDict[partitionKey][1], y[i], axis=0)
for key in partitionDict:
validationPartition = partitionDict[key]
validation_X = validationPartition[0]
validation_y = validationPartition[1]
train_X = np.matrix(np.empty((0, np.size(X, 1)), X[0, 0].dtype))
train_y = np.matrix(np.empty((0, 1), y[0, 0].dtype))
for otherKey in partitionDict:
if otherKey != key:
trainingPartition = partitionDict[otherKey]
train_X = np.append(train_X, trainingPartition[0], axis=0)
train_y = np.append(train_y, trainingPartition[1], axis=0)
classifier.train(train_X, train_y)
y_hat = np.matrix(classifier.predict(validation_X), dtype=validation_y.dtype)
print("validation error rate is: {0}".format(
(np.sum(np.bitwise_xor(y_hat, validation_y)) / np.size(validation_y, 0))))
def process_frame(frame, filename=None):
##frame[:,:,2]=0
##frame[:,:,1]=0
##frame[:,:,0]=0
##print frame
# Keep a copy of the frame, we need it later.
orig_frame = numpy.copy(frame)
new_frame_for_show = numpy.copy(frame)
new_frame_for_show[:,:,:2]=0
# Remove the R and G layers
new_frame = frame[:,:,2]
#mpimg.imsave('filteredRGB'+filename, new_frame_for_show, format='png')
new_new_frame = cv2.filter2D(new_frame, -1, smoothing_kernel2D(25))
#mpimg.imsave('SmoothedFilteredRGB'+filename, new_frame, format='png')
# Histogram plot, helps choose threshold
#hist = cv2.calcHist([new_new_frame], [0], None, 256, [0,256])
#plt.hist(new_new_frame.ravel(), 256, [0,256]);
#plt.show()
#print "NEW_FRAME", type(new_new_frame)
#print "Max, new_new_frame", numpy.amax(new_new_frame)
# Binarize the image
thresholding_filter = numpy.vectorize(lambda x: 255 if x>30 else 0)
filtered_frame = thresholding_filter(new_new_frame)
#print "max, filtered", numpy.amax(filtered_frame)
#print filtered_frame
# Find and Mark center of frame
cx, cy = FindCenter(filtered_frame)
if cx and cy:
filtered_frame[int(cy)-5:int(cy)+5, int(cx)-5:int(cx)+5] = 128
mask = numpy.ndarray(filtered_frame.shape, dtype=int)
mask[:,:] = 255
#print "MASK:", mask
inverted_filter = numpy.bitwise_xor(mask, filtered_frame)
frame_ball_marked = numpy.bitwise_and(orig_frame, numpy.dstack([inverted_filter]*3))
# Show grey scale
if filename:
plt.imshow(filtered_frame, cmap=cm.Greys_r)
plt.savefig(filename+".png", format='png')
#plt.show()
plt.clf()
plt.imshow(frame_ball_marked)
#plt.imshow(frame)
#plt.savefig(filename+"Marked.png", format='png')
else:
#plt.show()
pass
return cx, cy
def label_state(comm):
""" Label/personalize the random generator state for the local rank."""
def get_mask(rank):
""" Get a uint32 mask array to use to xor the random generator state.
We do not simply return the rank, as this small change to the
state of the Mersenne Twister only causes small changes in
the generated sequence. (The generators will eventually
diverge, but this takes a while.) So we scramble the mask up
a lot more, still deterministically, using a cryptographic hash.
See: http://en.wikipedia.org/wiki/Mersenne_twister#Disadvantages
"""
# Since we will be converting to/from bytes, endianness is important.
uint32be = np.dtype('>u4')
x = np.empty([624 // 8, 2], dtype=uint32be)
# The hash of the rank catted with an increasing index
# (stuffed into big-endian uint32s) are hashed with SHA-256 to make
# the XOR mask for 8 consecutive uint32 words for a 624-word
# Mersenne Twister state.
x[:, 0] = rank
x[:, 1] = np.arange(624 // 8)
mask_buffer = b''.join(sha256(row).digest() for row in x)
# And convert back to native-endian.
mask = np.frombuffer(mask_buffer, dtype=uint32be).astype(np.uint32)
return mask
rank = comm.Get_rank()
mask = get_mask(rank)
# For the Mersenne Twister used by numpy, the state is a 5-tuple,
# with the important part being an array of 624 uint32 values.
# We xor the mask into that array, and leave the rest of the tuple alone.
s0, orig_array, s2, s3, s4 = np.random.get_state()
mod_array = np.bitwise_xor(orig_array, mask)
np.random.set_state((s0, mod_array, s2, s3, s4))
def compare(self, im):
"""Show differences between images
:Parameters:
- `im`: the comparison image
@Returns: a difference image
"""
header_diff = HeaderDifference(self, im)
hdus = (('science', self.data, im.data),
('mask', self.mask, im.mask),
('weight', self.weight, im.weight))
diff_im = DESImage()
for ext, im1, im2 in hdus:
if ext=='science':
diff_im.data = im1-im2
elif ext=='mask':
diff_im.mask = np.bitwise_xor(im1, im2)
elif ext=='weight':
diff_im.weight = im1-im2
comparison = DESImageComparison(header_diff, diff_im)
return comparison
def computeDelta(matlabEng, arrData_bin_1, n, k, m, strCoder):
"""
compute the delta for reconciliation
"""
# find corresponding codeword of data 1
arrMsg_bin_1 = None
arrCodeword_bin_1 = None
if(strCoder == CODER_GOLAY):
n = 23
k = 12
arrMsg_bin_1 = golay.decode(matlabEng, arrData_bin_1, n)
arrCodeword_bin_1 = golay.encode(matlabEng, arrMsg_bin_1, k)
elif (strCoder == CODER_RS):
arrMsg_bin_1 = rs.decode(matlabEng, arrData_bin_1, n, k, m)
arrCodeword_bin_1 = rs.encode(matlabEng, arrMsg_bin_1, n, k, m)
elif(strCoder == CODER_HAMMING):
arrMsg_bin_1 = fec.decode(matlabEng, arrData_bin_1,
n, k, strCoder)
arrCodeword_bin_1 = fec.encode(matlabEng, arrMsg_bin_1,
n, k, strCoder)
else:
raise ValueError("Unkown coder")
# compute the difference
arrDelta = np.bitwise_xor(arrData_bin_1, arrCodeword_bin_1)
return arrDelta
def arithmetic_phase(self, operation, node):
"""
Arithmetic phase
"""
for case in Switch(operation):
if case('AND'):
self.output[:, node] = np.bitwise_and(self.input1[:, node], self.input2[:, node])
break
if case('OR'):
self.output[:, node] = np.bitwise_or(self.input1[:, node], self.input2[:, node])
break
if case('XOR'):
self.output[:, node] = np.bitwise_xor(self.input1[:, node], self.input2[:, node])
break
if case('NOR'):
self.output[:, node] = np.bitwise_nor(self.input1[:, node]) # Single operand instruction
break
if case('ADD'):
self.output[:, node] = np.add(self.input1[:, node], self.input2[:, node])
break
if case('SUB'):
self.output[:, node] = np.subtract(self.input1[:, node], self.input2[:, node])
break
if case('MULT'):
self.output[:, node] = np.multiply(self.input1[:, node], self.input2[:, node])
break
if case(): # Default
print("Error! Undefined instruction ", operation)
exit()
def query_with_distances(self, v, n):
"""Find indices of `n` most similar vectors from the index to query vector `v`."""
if self._metric == 'hamming':
v = numpy.packbits(v)
if self._metric != 'jaccard':
# use same precision for query as for index
v = numpy.ascontiguousarray(v, dtype = self.index.dtype)
# HACK we ignore query length as that's a constant not affecting the final ordering
if self._metric == 'angular':
# argmax_a cossim(a, b) = argmax_a dot(a, b) / |a||b| = argmin_a -dot(a, b)
dists = -numpy.dot(self.index, v)
elif self._metric == 'euclidean':
# argmin_a (a - b)^2 = argmin_a a^2 - 2ab + b^2 = argmin_a a^2 - 2ab
dists = self.lengths - 2 * numpy.dot(self.index, v)
elif self._metric == 'hamming':
diff = numpy.bitwise_xor(v, self.index)
pc = BruteForceBLAS.popcount
den = float(len(v) * 8)
dists = [sum([pc[part] for part in point]) / den for point in diff]
elif self._metric == 'jaccard':
dists = [pd[self._metric]['distance'](v, e) for e in self.index]
else:
assert False, "invalid metric" # shouldn't get past the constructor!
nearest_indices = numpy.argpartition(dists, n)[:n] # partition-sort by distance, get `n` closest
indices = [idx for idx in nearest_indices if pd[self._metric]["distance_valid"](dists[idx])]
def fix(index):
ep = self.index[index]
ev = v
if self._metric == "hamming":
ep = numpy.unpackbits(ep)
ev = numpy.unpackbits(ev)
return (index, pd[self._metric]['distance'](ep, ev))
return map(fix, indices)
def get_groupacc(finetuned_model_pr, concept_arraynew2, f_train, f_val,
concept, n_concept, n_cluster, n0, verbose):
"""Gets the group accuracy for discovered concepts."""
print(finetuned_model_pr.summary())
min_weight = finetuned_model_pr.layers[-5].get_weights()[0]
sim_array = np.zeros((n_cluster, n_concept))
for j in range(n_cluster):
sim_array[j, :] = np.mean(np.matmul(concept_arraynew2[j, :100, :],
min_weight),
axis=0)
posneg = np.zeros(5)
sim_array_0mean = sim_array - np.mean(sim_array, axis=0)
max_cluster = np.argmax(np.abs(sim_array_0mean), axis=0)
for count in range(5):
posneg[count] = sim_array_0mean[max_cluster[count], count] > 0
loss_table = np.zeros((5, 5))
for count in range(5):
for count2 in range(5):
# count2 = max_cluster[count]
mean0 = np.mean(
np.matmul(f_train,
min_weight[:, count])[concept[:n0,
count2] == 0]) * 100
mean1 = np.mean(
np.matmul(f_train,
min_weight[:, count])[concept[:n0,
count2] == 1]) * 100
if mean0 < mean1:
pos = 1
else:
pos = -1
best_err = 1e10
best_bias = 0
a = int((mean1 - mean0) / 20)
if a == 0:
a = pos
for bias in range(int(mean0), int(mean1), a):
if pos == 1:
if np.sum(
np.bitwise_xor(
concept[:n0, count2],
np.matmul(f_train, min_weight[:, count]) >
bias / 100.)) < best_err:
best_err = np.sum(
np.bitwise_xor(
concept[:n0, count2],
np.matmul(f_train, min_weight[:, count]) >
bias / 100.))
best_bias = bias
else:
if np.sum(
np.bitwise_xor(
concept[:n0, count2],
np.matmul(f_train, min_weight[:, count]) <
bias / 100.)) < best_err:
best_err = np.sum(
np.bitwise_xor(
concept[:n0, count2],
np.matmul(f_train, min_weight[:, count]) <
bias / 100.))
best_bias = bias
if pos == 1:
loss_table[count, count2] = np.sum(
np.bitwise_xor(
concept[n0:, count2],
np.matmul(f_val, min_weight[:, count]) >
best_bias / 100.)) / 12000
if verbose:
print(
np.sum(
np.bitwise_xor(
concept[n0:, count2],
np.matmul(f_val, min_weight[:, count]) >
best_bias / 100.)) / 12000)
else:
loss_table[count, count2] = np.sum(
np.bitwise_xor(
concept[n0:, count2],
np.matmul(f_val, min_weight[:, count]) <
best_bias / 100.)) / 12000
if verbose:
print(
np.sum(
np.bitwise_xor(
concept[n0:, count2],
np.matmul(f_val, min_weight[:, count]) <
best_bias / 100.)) / 12000)
print(np.amin(loss_table, axis=0))
acc = np.mean(np.amin(loss_table, axis=0))
print(acc)
return min_weight, acc
def numpy_bitwise_xor(x1, x2):
x1 = dpnp.asnumpy(x1) if isinstance(x1, dparray) else x1
x2 = dpnp.asnumpy(x2) if isinstance(x2, dparray) else x2
return numpy.bitwise_xor(x1, x2)
def hammingB(x, y):
# This function computes the normalised hamming distance between two binary vectors
# h = 1.0 - float(np.sum(np.logical_xor(X, Y)))/len(X)
# count is faster than sum
return 1.0 - float(np.count_nonzero(np.bitwise_xor(x, y))) / len(x)
def bitwiseXOR(img1, img2):
new = np.bitwise_xor(img1, img2)
return new
def hamming(x, y):
x = np.asarray(x, np.bool)
y = np.asarray(y, np.bool)
return np.bitwise_xor(x, y).sum()
size[0], size[1], suffix),
dtype=ref_dtype)
out = np.fromfile('data/output/{}.bin'.format(suffix), dtype=out_dtype)
# ref=ref.reshape((64,*size))
# out=out.reshape((64,*size))
# print(ref[8])
# print(out[8])
# for i in range(64):
# print((ref[i]-out[i]).sum())
for i in range(5):
print(bin(ref[i]), " ", end='')
print("\n")
for i in range(5):
print(bin(out[i]), " ", end='')
print("\n")
mean = np.bitwise_xor(ref, out).mean()
elif 'conv' in suffix and suffix not in ['conv0', 'conv16']:
ref = np.fromfile('data/ref/{}x{}/{}.bin'.format(
size[0], size[1], suffix),
dtype=ref_dtype).reshape((*size, 64))
out = np.fromfile('data/output/{}.bin'.format(suffix),
dtype=out_dtype).reshape((*size, 64))
# out2=out
out2 = np.zeros_like(out, dtype=np.int32)
for k in range(0, 64):
for j in range(0, size[0]):
for i in range(0, size[1]):
T = 9 * 64
if (i == 0 or i == size[1] - 1):
T -= 3 * 64
if (j == 0 or j == size[0] - 1):
def i4_sobol(dim_num, seed):
"""
I4_SOBOL generates a new quasirandom Sobol vector with each call.
Discussion:
The routine adapts the ideas of Antonov and Saleev.
Licensing:
This code is distributed under the MIT license.
Modified:
22 February 2011
Author:
Original FORTRAN77 version by Bennett Fox.
MATLAB version by John Burkardt.
PYTHON version by Corrado Chisari
Reference:
Antonov, Saleev,
USSR Computational Mathematics and Mathematical Physics,
olume 19, 19, pages 252 - 256.
Paul Bratley, Bennett Fox,
Algorithm 659:
Implementing Sobol's Quasirandom Sequence Generator,
ACM Transactions on Mathematical Software,
Volume 14, Number 1, pages 88-100, 1988.
Bennett Fox,
Algorithm 647:
Implementation and Relative Efficiency of Quasirandom
Sequence Generators,
ACM Transactions on Mathematical Software,
Volume 12, Number 4, pages 362-376, 1986.
Ilya Sobol,
USSR Computational Mathematics and Mathematical Physics,
Volume 16, pages 236-242, 1977.
Ilya Sobol, Levitan,
The Production of Points Uniformly Distributed in a Multidimensional
Cube (in Russian),
Preprint IPM Akad. Nauk SSSR,
Number 40, Moscow 1976.
Parameters:
Input, integer DIM_NUM, the number of spatial dimensions.
DIM_NUM must satisfy 1 <= DIM_NUM <= 40.
Input/output, integer SEED, the "seed" for the sequence.
This is essentially the index in the sequence of the quasirandom
value to be generated. On output, SEED has been set to the
appropriate next value, usually simply SEED+1.
If SEED is less than 0 on input, it is treated as though it were 0.
An input value of 0 requests the first (0-th) element of the sequence.
Output, real QUASI(DIM_NUM), the next quasirandom vector.
"""
global atmost
global dim_max
global dim_num_save
global initialized
global lastq
global log_max
global maxcol
global poly
global recipd
global seed_save
global v
if not initialized or dim_num != dim_num_save:
initialized = 1
dim_max = 40
dim_num_save = -1
log_max = 30
seed_save = -1
#
# Initialize (part of) V.
#
v = np.zeros((dim_max, log_max))
v[0:40, 0] = np.transpose([
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
])
v[2:40, 1] = np.transpose([
1, 3, 1, 3, 1, 3, 3, 1, 3, 1, 3, 1, 3, 1, 1, 3, 1, 3, 1, 3, 1, 3,
3, 1, 3, 1, 3, 1, 3, 1, 1, 3, 1, 3, 1, 3, 1, 3
])
v[3:40, 2] = np.transpose([
7, 5, 1, 3, 3, 7, 5, 5, 7, 7, 1, 3, 3, 7, 5, 1, 1, 5, 3, 3, 1, 7,
5, 1, 3, 3, 7, 5, 1, 1, 5, 7, 7, 5, 1, 3, 3
])
v[5:40, 3] = np.transpose([
1, 7, 9, 13, 11, 1, 3, 7, 9, 5, 13, 13, 11, 3, 15, 5, 3, 15, 7, 9,
13, 9, 1, 11, 7, 5, 15, 1, 15, 11, 5, 3, 1, 7, 9
])
v[7:40, 4] = np.transpose([
9, 3, 27, 15, 29, 21, 23, 19, 11, 25, 7, 13, 17, 1, 25, 29, 3, 31,
11, 5, 23, 27, 19, 21, 5, 1, 17, 13, 7, 15, 9, 31, 9
])
v[13:40, 5] = np.transpose([
37, 33, 7, 5, 11, 39, 63, 27, 17, 15, 23, 29, 3, 21, 13, 31, 25, 9,
49, 33, 19, 29, 11, 19, 27, 15, 25
])
v[19:40, 6] = np.transpose([
13, 33, 115, 41, 79, 17, 29, 119, 75, 73, 105, 7, 59, 65, 21, 3,
113, 61, 89, 45, 107
])
v[37:40, 7] = np.transpose([7, 23, 39])
#
# Set POLY.
#
poly = [
1, 3, 7, 11, 13, 19, 25, 37, 59, 47, 61, 55, 41, 67, 97, 91, 109,
103, 115, 131, 193, 137, 145, 143, 241, 157, 185, 167, 229, 171,
213, 191, 253, 203, 211, 239, 247, 285, 369, 299
]
atmost = 2**log_max - 1
#
# Find the number of bits in ATMOST.
#
maxcol = i4_bit_hi1(atmost)
#
# Initialize row 1 of V.
#
v[0, 0:maxcol] = 1
# Things to do only if the dimension changed.
if dim_num != dim_num_save:
#
# Check parameters.
#
if (dim_num < 1 or dim_max < dim_num):
print('I4_SOBOL - Fatal error!')
print(' The spatial dimension DIM_NUM should satisfy:')
print(' 1 <= DIM_NUM <= %d' % dim_max)
print(' But this input value is DIM_NUM = %d' % dim_num)
return None
dim_num_save = dim_num
#
# Initialize the remaining rows of V.
#
for i in range(2, dim_num + 1):
#
# The bits of the integer POLY(I) gives the form of polynomial
# I.
#
# Find the degree of polynomial I from binary encoding.
#
j = poly[i - 1]
m = 0
while True:
j = math.floor(j / 2.)
if (j <= 0):
break
m = m + 1
#
# Expand this bit pattern to separate components of the logical
# array INCLUD.
#
j = poly[i - 1]
includ = np.zeros(m)
for k in range(m, 0, -1):
j2 = math.floor(j / 2.)
includ[k - 1] = (j != 2 * j2)
j = j2
#
# Calculate the remaining elements of row I as explained
# in Bratley and Fox, section 2.
#
for j in range(m + 1, maxcol + 1):
newv = v[i - 1, j - m - 1]
l_var = 1
for k in range(1, m + 1):
l_var = 2 * l_var
if (includ[k - 1]):
newv = np.bitwise_xor(int(newv),
int(l_var * v[i - 1, j - k - 1]))
v[i - 1, j - 1] = newv
#
# Multiply columns of V by appropriate power of 2.
#
l_var = 1
for j in range(maxcol - 1, 0, -1):
l_var = 2 * l_var
v[0:dim_num, j - 1] = v[0:dim_num, j - 1] * l_var
#
# RECIPD is 1/(common denominator of the elements in V).
#
recipd = 1.0 / (2 * l_var)
lastq = np.zeros(dim_num)
seed = int(math.floor(seed))
if (seed < 0):
seed = 0
if (seed == 0):
l_var = 1
lastq = np.zeros(dim_num)
elif (seed == seed_save + 1):
#
# Find the position of the right-hand zero in SEED.
#
l_var = i4_bit_lo0(seed)
elif (seed <= seed_save):
seed_save = 0
lastq = np.zeros(dim_num)
for seed_temp in range(int(seed_save), int(seed)):
l_var = i4_bit_lo0(seed_temp)
for i in range(1, dim_num + 1):
lastq[i - 1] = np.bitwise_xor(int(lastq[i - 1]),
int(v[i - 1, l_var - 1]))
l_var = i4_bit_lo0(seed)
elif (seed_save + 1 < seed):
for seed_temp in range(int(seed_save + 1), int(seed)):
l_var = i4_bit_lo0(seed_temp)
for i in range(1, dim_num + 1):
lastq[i - 1] = np.bitwise_xor(int(lastq[i - 1]),
int(v[i - 1, l_var - 1]))
l_var = i4_bit_lo0(seed)
#
# Check that the user is not calling too many times!
#
if maxcol < l_var:
print('I4_SOBOL - Fatal error!')
print(' Too many calls!')
print(' MAXCOL = %d\n' % maxcol)
print(' L = %d\n' % l_var)
return None
#
# Calculate the new components of QUASI.
#
quasi = np.zeros(dim_num)
for i in range(1, dim_num + 1):
quasi[i - 1] = lastq[i - 1] * recipd
lastq[i - 1] = np.bitwise_xor(int(lastq[i - 1]),
int(v[i - 1, l_var - 1]))
seed_save = seed
seed = seed + 1
return [quasi, seed]
def create_dataset(skip, n_sample=60000):
"""Creates toy dataset and save to disk."""
assert n_sample > 15 # or it will clearly bug
concept = np.reshape(np.random.randint(
2, size=15 * n_sample), (-1, 15)).astype(
np.bool_) # shape (n_sample, 15), type boolean, (60000, 15)
concept[:15, :15] = np.eye(15) # hardcode some entries to 1 # TODO WHY
fig = Figure(figsize=(3, 3))
canvas = FigureCanvas(fig)
axes = fig.gca()
axes.set_xlim([0, 10])
axes.set_ylim([0, 10])
axes.axis('off')
width, height = fig.get_size_inches() * fig.get_dpi()
width = int(width)
height = int(height)
location = [(1.3, 1.3), (3.3, 1.3), (5.3, 1.3), (7.3, 1.3), (1.3, 3.3),
(3.3, 3.3), (5.3, 3.3), (7.3, 3.3), (1.3, 5.3), (3.3, 5.3),
(5.3, 5.3), (7.3, 5.3), (1.3, 7.3), (3.3, 7.3),
(5.3, 7.3)] # all possible locations for shape to appear
x = np.zeros(
(n_sample, width, height, 3),
dtype='uint8') # specify data type to save memory / avoid warning
color_array = [
'green', 'red', 'blue', 'black', 'orange', 'purple', 'yellow'
]
if not skip:
for i in range(n_sample):
##############################################
fig = Figure(figsize=(3, 3))
canvas = FigureCanvas(fig)
axes = fig.gca()
axes.set_xlim([0, 10])
axes.set_ylim([0, 10])
axes.axis('off')
# added by Tony, without reset all x will draw on same canvas, bug
##############################################
location_bool = np.zeros(
15
) # [Added by Tony, or it will inf loop, bug]: for each image, record whether the location has shape already, to avoid overlap
if i % 1000 == 0:
print('{} images are created'.format(i))
if concept[
i,
5] == 1: # consider concept 5, of image i. 1 means that concept needs to appear in the image generated
a = np.random.randint(
15
) # select a random int to index into variable "location"
while location_bool[
a] == 1: # if the place has already been taken by other shape
a = np.random.randint(
15
) # sample it again # these three lines amount to finding a random place in image that is not taken
location_bool[a] = 1 # mark that the place chosen is chosen
axes.plot(
location[a][0], # x pos
location[a][1], # y pos
'x', # shape
color=color_array[np.random.randint(100) %
7], # a random color
markersize=8, # size of shape
mew=4,
) # marker edge width
# ms=8) # redundant argument. bug.
if concept[i, 6] == 1: # repeat the same thing for concept 6
a = np.random.randint(15)
while location_bool[a] == 1:
a = np.random.randint(15)
location_bool[a] = 1
axes.plot(
location[a][0],
location[a][1],
'3',
color=color_array[np.random.randint(100) % 7],
markersize=8,
mew=4,
)
# ms=8)
if concept[i, 7] == 1:
a = np.random.randint(15)
while location_bool[a] == 1:
a = np.random.randint(15)
location_bool[a] = 1
axes.plot(
location[a][0],
location[a][1],
's',
color=color_array[np.random.randint(100) % 7],
markersize=8,
mew=4,
)
# ms=8)
if concept[i, 8] == 1:
a = np.random.randint(15)
while location_bool[a] == 1:
a = np.random.randint(15)
location_bool[a] = 1
axes.plot(
location[a][0],
location[a][1],
'p',
color=color_array[np.random.randint(100) % 7],
markersize=8,
mew=4,
)
# ms=8)
if concept[i, 9] == 1:
a = np.random.randint(15)
while location_bool[a] == 1:
a = np.random.randint(15)
location_bool[a] = 1
axes.plot(
location[a][0],
location[a][1],
'_',
color=color_array[np.random.randint(100) % 7],
markersize=8,
mew=4,
)
# ms=8)
if concept[i, 10] == 1:
a = np.random.randint(15)
while location_bool[a] == 1:
a = np.random.randint(15)
location_bool[a] = 1
axes.plot(
location[a][0],
location[a][1],
'd',
color=color_array[np.random.randint(100) % 7],
markersize=8,
mew=4,
)
# ms=8)
if concept[i, 11] == 1:
a = np.random.randint(15)
while location_bool[a] == 1:
a = np.random.randint(15)
location_bool[a] = 1
axes.plot(
location[a][0],
location[a][1],
'd',
color=color_array[np.random.randint(100) % 7],
markersize=8,
mew=4,
)
# ms=8)
if concept[i, 12] == 1:
a = np.random.randint(15)
while location_bool[a] == 1:
a = np.random.randint(15)
location_bool[a] = 1
axes.plot(
location[a][0],
location[a][1],
11,
color=color_array[np.random.randint(100) % 7],
markersize=8,
mew=4,
)
# ms=8)
if concept[i, 13] == 1:
a = np.random.randint(15)
while location_bool[a] == 1:
a = np.random.randint(15)
location_bool[a] = 1
axes.plot(
location[a][0],
location[a][1],
'o',
color=color_array[np.random.randint(100) % 7],
markersize=8,
mew=4,
)
# ms=8)
if concept[i, 14] == 1:
a = np.random.randint(15)
while location_bool[a] == 1:
a = np.random.randint(15)
location_bool[a] = 1
axes.plot(
location[a][0],
location[a][1],
'.',
color=color_array[np.random.randint(100) % 7],
markersize=8,
mew=4,
)
# ms=8)
if concept[i, 0] == 1:
a = np.random.randint(15)
while location_bool[a] == 1:
a = np.random.randint(15)
location_bool[a] = 1
axes.plot(
location[a][0],
location[a][1],
'+',
color=color_array[np.random.randint(100) % 7],
markersize=8,
mew=4,
)
# ms=8)
if concept[i, 1] == 1:
a = np.random.randint(15)
while location_bool[a] == 1:
a = np.random.randint(15)
location_bool[a] = 1
axes.plot(
location[a][0],
location[a][1],
'1',
color=color_array[np.random.randint(100) % 7],
markersize=8,
mew=4,
)
# ms=8)
if concept[i, 2] == 1:
a = np.random.randint(15)
while location_bool[a] == 1:
a = np.random.randint(15)
location_bool[a] = 1
axes.plot(
location[a][0],
location[a][1],
'*',
color=color_array[np.random.randint(100) % 7],
markersize=30,
mew=3,
)
# ms=5)
if concept[i, 3] == 1:
a = np.random.randint(15)
while location_bool[a] == 1:
a = np.random.randint(15)
location_bool[a] = 1
axes.plot(
location[a][0],
location[a][1],
'<',
color=color_array[np.random.randint(100) % 7],
markersize=8,
mew=4,
)
# ms=8)
if concept[i, 4] == 1:
a = np.random.randint(15)
while location_bool[a] == 1:
a = np.random.randint(15)
location_bool[a] = 1
axes.plot(
location[a][0],
location[a][1],
'h',
color=color_array[np.random.randint(100) % 7],
markersize=8,
mew=4,
)
# ms=8)
canvas.draw()
image = np.fromstring(canvas.tostring_rgb(),
dtype='uint8').reshape(width, height, 3)
x[i, :, :, :] = image
fig.clf(
) # they also did not close the graph: their cluster probably has 500G memory so it does not matter :)
# imgplot = plt.imshow(image)
# plt.show()
# create label by booling functions
y = np.zeros((n_sample, 15))
y[:, 0] = ((1 - concept[:, 0] * concept[:, 2]) + concept[:, 3]) > 0
y[:, 1] = concept[:, 1] + (concept[:, 2] * concept[:, 3])
y[:, 2] = (concept[:, 3] * concept[:, 4]) + (concept[:, 1] *
concept[:, 2])
y[:, 3] = np.bitwise_xor(concept[:, 0], concept[:, 1])
y[:, 4] = concept[:, 1] + concept[:, 4]
y[:, 5] = (1 - (concept[:, 0] + concept[:, 3] + concept[:, 4])) > 0
y[:, 6] = np.bitwise_xor(concept[:, 1] * concept[:, 2], concept[:, 4])
y[:, 7] = concept[:, 0] * concept[:, 4] + concept[:, 1]
y[:, 8] = concept[:, 2]
y[:, 9] = np.bitwise_xor(concept[:, 0] + concept[:, 1], concept[:, 3])
y[:, 10] = (1 - (concept[:, 2] + concept[:, 4])) > 0
y[:, 11] = concept[:, 0] + concept[:, 3] + concept[:, 4]
y[:, 12] = np.bitwise_xor(concept[:, 1], concept[:, 2])
y[:, 13] = (1 - (concept[:, 0] * concept[:, 4] + concept[:, 3])) > 0
y[:, 14] = np.bitwise_xor(concept[:, 4], concept[:, 3])
np.save('x_data.npy', x)
np.save('y_data.npy', y)
np.save('concept_data.npy', concept)
return width, height
def compute_W(patch, Y, A, C, CdotA, radius, dims):
"""compute background according to ring model
solves the problem
min_{W,b0} ||X-W*X|| with X = Y - A*C - b0*1'
subject to
W(i,j) = 0 for each pixel j that is not in ring around pixel i
Problem parallelizes over pixels i
Fluctuating background activity is W*X, constant baselines b0.
Parameters:
----------
Y: np.ndarray (2D or 3D)
movie, raw data in 2D or 3D (pixels x time).
A: np.ndarray or sparse matrix
spatial footprint of each neuron.
C: np.ndarray
calcium activity of each neuron.
dims: tuple
x, y[, z] movie dimensions
radius: int
radius of ring
data_fits_in_memory: [optional] bool
If true, use faster but more memory consuming computation
Returns:
--------
W: scipy.sparse.csr_matrix (pixels x pixels)
estimate of weight matrix for fluctuating background
b0: np.ndarray (pixels,)
estimate of constant background baselines
"""
from time import time
start = time()
ring = disk(radius + 1, dtype=bool)
ring[1:-1, 1:-1] = np.bitwise_xor(ring[1:-1, 1:-1], disk(radius, dtype=bool))
ringidx = np.vstack(np.where(ring)).T - radius - 1
if isinstance(A, np.ndarray) and isinstance(C, np.ndarray):
tmp = C.mean(0).dot(A)
elif isinstance(A, h5py._hl.dataset.Dataset) and isinstance(C, h5py._hl.dataset.Dataset):
print("TODO")
sys.exit()
pass
tmp2 = np.zeros_like(tmp)
chunk_size = Y.chunks[0]
for i in range(0, Y.shape[0]+chunk_size,chunk_size): tmp2 += Y[i:i+chunk_size,:].sum(0)
tmp2 = tmp2/float(Y.shape[0])
# constante fluorescence is mean(Y) - A * mean(C)
b0 = tmp2 - tmp
if isinstance(CdotA, np.ndarray):
X = (Y - CdotA - b0).T # (pixels, time)
elif isinstance(CdotA, h5py._hl.dataset.Dataset):
print("TODO")
sys.exit()
pass
W = np.zeros((np.prod(dims),np.prod(dims)))
pixels_pos = np.array([x for x in itertools.product(range(dims[0]), range(dims[1]))])
xpos_ring = np.vstack(pixels_pos[:,0])+ringidx[:,0]
ypos_ring = np.vstack(pixels_pos[:,1])+ringidx[:,1]
xypos_ring = np.dstack((xpos_ring, ypos_ring))
inside = (xpos_ring>=0) * (ypos_ring>=0) * (xpos_ring<dims[0]) * (ypos_ring<dims[1])
for i,(r,c) in enumerate(itertools.product(range(dims[0]), range(dims[1]))):
# xy = np.array([r,c]) + ringidx
# inside = np.prod((xy >= 0) * (xy < dims), 1)
# index = xy[inside==1]
index = xypos_ring[i, inside[i]==1]
flatten_index = np.ravel_multi_index(index.T, dims)
B = X[flatten_index]
tmp0 = B.dot(B.T)
tmp0 = tmp0 + 1.e-9*np.eye(index.shape[0])
tmp = np.linalg.inv(tmp0)
tmp2 = X[i].dot(B.T)
tmp3 = tmp2.dot(tmp)
W[i,flatten_index] = tmp3
end = time()
# print("Time to update weight ", time() - start)
return W, b0
def generate_error_map(seg, truth):
error_map = np.bitwise_xor(seg, truth)
return error_map
def i4_sobol(dim_num, seed):
"""
Parameters:
Input, integer DIM_NUM, the number of spatial dimensions.
DIM_NUM must satisfy 1 <= DIM_NUM <= 40.
Input/output, integer SEED, the "seed" for the sequence.
This is essentially the index in the sequence of the quasirandom
value to be generated. On output, SEED has been set to the
appropriate next value, usually simply SEED+1.
If SEED is less than 0 on input, it is treated as though it were 0.
An input value of 0 requests the first (0-th) element of the sequence.
Output, real QUASI(DIM_NUM), the next quasirandom vector.
"""
global atmost
global dim_max
global dim_num_save
global initialized
global lastq
global log_max
global maxcol
global poly
global recipd
global seed_save
global v
if 'initialized' not in list(globals().keys()):
initialized = 0
dim_num_save = -1
if not initialized or dim_num != dim_num_save:
initialized = 1
dim_max = 40
dim_num_save = -1
log_max = 30
seed_save = -1
# Initialize (part of) V.
v = np.zeros((dim_max, log_max))
v[0:40, 0] = np.transpose([
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
])
v[2:40, 1] = np.transpose([
1, 3, 1, 3, 1, 3, 3, 1, 3, 1, 3, 1, 3, 1, 1, 3, 1, 3, 1, 3, 1, 3,
3, 1, 3, 1, 3, 1, 3, 1, 1, 3, 1, 3, 1, 3, 1, 3
])
v[3:40, 2] = np.transpose([
7, 5, 1, 3, 3, 7, 5, 5, 7, 7, 1, 3, 3, 7, 5, 1, 1, 5, 3, 3, 1, 7,
5, 1, 3, 3, 7, 5, 1, 1, 5, 7, 7, 5, 1, 3, 3
])
v[5:40, 3] = np.transpose([
1, 7, 9, 13, 11, 1, 3, 7, 9, 5, 13, 13, 11, 3, 15, 5, 3, 15, 7, 9,
13, 9, 1, 11, 7, 5, 15, 1, 15, 11, 5, 3, 1, 7, 9
])
v[7:40, 4] = np.transpose([
9, 3, 27, 15, 29, 21, 23, 19, 11, 25, 7, 13, 17, 1, 25, 29, 3, 31,
11, 5, 23, 27, 19, 21, 5, 1, 17, 13, 7, 15, 9, 31, 9
])
v[13:40, 5] = np.transpose([
37, 33, 7, 5, 11, 39, 63, 27, 17, 15, 23, 29, 3, 21, 13, 31, 25, 9,
49, 33, 19, 29, 11, 19, 27, 15, 25
])
v[19:40, 6] = np.transpose([
13, 33, 115, 41, 79, 17, 29, 119, 75, 73, 105, 7, 59, 65, 21, 3,
113, 61, 89, 45, 107
])
v[37:40, 7] = np.transpose([7, 23, 39])
# Set POLY.
poly = [
1, 3, 7, 11, 13, 19, 25, 37, 59, 47, 61, 55, 41, 67, 97, 91, 109,
103, 115, 131, 193, 137, 145, 143, 241, 157, 185, 167, 229, 171,
213, 191, 253, 203, 211, 239, 247, 285, 369, 299
]
atmost = 2**log_max - 1
# Find the number of bits in ATMOST.
maxcol = i4_bit_hi1(atmost)
# Initialize row 1 of V.
v[0, 0:maxcol] = 1
# Things to do only if the dimension changed.
if dim_num != dim_num_save:
# Check parameters.
if dim_num < 1 or dim_max < dim_num:
print('I4_SOBOL - Fatal error!')
print(' The spatial dimension DIM_NUM should satisfy:')
print(' 1 <= DIM_NUM <= %d' % dim_max)
print(' But this input value is DIM_NUM = %d' % dim_num)
return
dim_num_save = dim_num
# Initialize the remaining rows of V.
for i in range(2, dim_num + 1):
# The bits of the integer POLY(I) gives the form of polynomial I.
# Find the degree of polynomial I from binary encoding.
j = poly[i - 1]
m = 0
j //= 2
while j > 0:
j //= 2
m += 1
# Expand this bit pattern to separate components of the logical array INCLUD.
j = poly[i - 1]
includ = np.zeros(m)
for k in range(m, 0, -1):
j2 = j // 2
includ[k - 1] = (j != 2 * j2)
j = j2
# Calculate the remaining elements of row I as explained
# in Bratley and Fox, section 2.
for j in range(m + 1, maxcol + 1):
newv = v[i - 1, j - m - 1]
l = 1
for k in range(1, m + 1):
l *= 2
if includ[k - 1]:
newv = np.bitwise_xor(int(newv),
int(l * v[i - 1, j - k - 1]))
v[i - 1, j - 1] = newv
# Multiply columns of V by appropriate power of 2.
l = 1
for j in range(maxcol - 1, 0, -1):
l *= 2
v[0:dim_num, j - 1] = v[0:dim_num, j - 1] * l
# RECIPD is 1/(common denominator of the elements in V).
recipd = 1.0 / (2 * l)
lastq = np.zeros(dim_num)
seed = int(np.floor(seed))
if seed < 0:
seed = 0
l = 1
if seed == 0:
lastq = np.zeros(dim_num)
elif seed == seed_save + 1:
# Find the position of the right-hand zero in SEED.
l = i4_bit_lo0(seed)
elif seed <= seed_save:
seed_save = 0
lastq = np.zeros(dim_num)
for seed_temp in range(int(seed_save), int(seed)):
l = i4_bit_lo0(seed_temp)
for i in range(1, dim_num + 1):
lastq[i - 1] = np.bitwise_xor(int(lastq[i - 1]),
int(v[i - 1, l - 1]))
l = i4_bit_lo0(seed)
elif seed_save + 1 < seed:
for seed_temp in range(int(seed_save + 1), int(seed)):
l = i4_bit_lo0(seed_temp)
for i in range(1, dim_num + 1):
lastq[i - 1] = np.bitwise_xor(int(lastq[i - 1]),
int(v[i - 1, l - 1]))
l = i4_bit_lo0(seed)
# Check that the user is not calling too many times!
if maxcol < l:
print('I4_SOBOL - Fatal error!')
print(' Too many calls!')
print(' MAXCOL = %d\n' % maxcol)
print(' L = %d\n' % l)
return
# Calculate the new components of QUASI.
quasi = np.zeros(dim_num)
for i in range(1, dim_num + 1):
quasi[i - 1] = lastq[i - 1] * recipd
lastq[i - 1] = np.bitwise_xor(int(lastq[i - 1]), int(v[i - 1, l - 1]))
seed_save = seed
seed += 1
return [quasi, seed]
def sign_extend(value, nbits):
value = np.asarray(value)
sign_bit = 1 << (nbits - 1)
return np.bitwise_xor(value, sign_bit) - sign_bit
