def spa_model(plaintext, key): hd_arr = np.zeros(16) hw_arr = np.zeros(16) zo_arr = np.zeros(16) byte_list = [0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 15, 11, 7] for byte in byte_list: print 'byte: ' + str(byte) plaintext_nth_byte = plaintext[byte] # XOR the plaintext byte with key byte and put the result through the S-BOX xored_plaintxt_key_bytes = np.bitwise_xor(plaintext_nth_byte, key[byte]) # Use the Hamming Weight model to calculate the hypothetical power consumption of # the SBOX operation. hw_arr[byte] = num_ones(sbox_hex[xored_plaintxt_key_bytes]) # Use the Hamming Distance model hd_arr[byte] = num_ones(np.bitwise_xor(xored_plaintxt_key_bytes, sbox_hex[xored_plaintxt_key_bytes])) # Other power model! 0 -> 1 transition zo_arr[byte] = num_ones(~xored_plaintxt_key_bytes & sbox_hex[xored_plaintxt_key_bytes]) print '%x, %x, %d, %d, %d' %(xored_plaintxt_key_bytes, sbox_hex[xored_plaintxt_key_bytes], hw_arr[byte], hd_arr[byte], zo_arr[byte]) plt.plot(hd_arr, 'r') plt.plot(hw_arr, 'b') plt.plot(zo_arr, 'g') plt.show()
def detect_corruption(self, fname): """ single disk corruption detection :param fname: data name in RAID6 system :return: corrupted disk index; for p, self.N-2; for q, self.N-1 """ # all disks, including P, Q byte_ndarray = self._read_n(fname, self.N) data_ndarray = byte_ndarray[:-2] self._logger.info("byte_ndarray=\n{}".format(byte_ndarray)) P = byte_ndarray[-2:-1] self._logger.info("p={}".format(P)) Q = byte_ndarray[-1] self._logger.info("Q={}".format(Q)) P_prime = utils.gen_p(data_ndarray, ndim=2) self._logger.info("P_prime={}".format(P_prime)) Q_prime = utils.gen_q(data_ndarray, ndim=2) self._logger.info("Q_prime={}".format(Q_prime)) P_star = np.bitwise_xor(P, P_prime) Q_star = np.bitwise_xor(Q, Q_prime) P_nonzero = np.count_nonzero(P_star) Q_nonzero = np.count_nonzero(Q_star) if P_nonzero == 0 and Q_nonzero == 0: print("no corruption") return None elif P_nonzero == 0 and Q_nonzero != 0: print("Q corruption") return self.N - 1 elif P_nonzero != 0 and Q_nonzero == 0: print("P corruption") return self.N - 2 else: index = self._get_corrupted_data_disk(P_star, Q_star) print("data disk {} corruption".format(index)) return index
def _unmask(self, buf, mask): # Unmask a frame if numpy: plen = len(buf) pstart = 0 pend = plen b = c = ''.encode('ascii') if plen >= 4: dtype=numpy.dtype('<u4') if sys.byteorder == 'big': dtype = dtype.newbyteorder('>') mask = numpy.frombuffer(mask, dtype, count=1) data = numpy.frombuffer(buf, dtype, count=int(plen / 4)) #b = numpy.bitwise_xor(data, mask).data b = numpy.bitwise_xor(data, mask).tostring() if plen % 4: dtype=numpy.dtype('B') if sys.byteorder == 'big': dtype = dtype.newbyteorder('>') mask = numpy.frombuffer(mask, dtype, count=(plen % 4)) data = numpy.frombuffer(buf, dtype, offset=plen - (plen % 4), count=(plen % 4)) c = numpy.bitwise_xor(data, mask).tostring() return b + c else: # Slower fallback if sys.hexversion < 0x3000000: mask = [ ord(c) for c in mask ] data = array.array('B') data.fromstring(buf) for i in range(len(data)): data[i] ^= mask[i % 4] return data.tostring()
def test_getTraining(self): test = np.array(Image.open(self.filename), 'uint8') train = np.array(Image.open(self.filename), 'uint8') np.bitwise_xor(test,train,test) image = np.ones(test.shape,test.dtype) le.getTraining(image,self.filename, self.filename) assert((test == image).all(), True)
def iota(ain, rnd): # Initialize empty arrays aout = np.zeros((5,5,64), dtype = int) bit = np.zeros(dtype = int, shape = (5,5,64)) rc = np.zeros(dtype = int, shape = 168) # Linear Feedback Shift Register w = np.array([1,0,0,0,0,0,0,0], dtype = int) rc[0] = w[0] for i in range(1, 7*24): a = np.bitwise_xor(w[0], w[4]) b = np.bitwise_xor(w[5], w[6]) tail = np.bitwise_xor(a, b) w = [w[1],w[2],w[3],w[4],w[5],w[6],w[7], tail] rc[i] = w[0] # Calculate bits for l in range(7): q = pow(2, l) - 1 t = l + 7*rnd bit[0][0][q] = rc[l + 7*rnd] # Calculate aout for i in range(5): for j in range(5): for k in range(64): aout[i][j][k] = np.bitwise_xor(ain[i][j][k], bit[i][j][k]) return aout
def count_sensitive_neighborhood_hash(g): """ Compute the count sensitive neighborhood hashed version of a graph. """ gnh = g.copy() g = array_labels_to_str(g) #iterate over every node in the graph for node in iter(g.nodes()): neighbors_labels = [g.node[n]["label"] for n in g.neighbors_iter(node)] #if node has no neighboors, nh is its own label if len(neighbors_labels) > 0: #count number of unique labels c = Counter(neighbors_labels) count_weighted_neighbors_labels = [] for label, c in c.iteritems(): label = str_to_array(label) c_bin = np.array( list(np.binary_repr( c, len(label) ) ), dtype=np.int64 ) label = np.bitwise_xor( label, c_bin) label = np.roll( label, c ) count_weighted_neighbors_labels.append( label ) x = count_weighted_neighbors_labels[0] for l in count_weighted_neighbors_labels[1:]: x = np.bitwise_xor( x, l) node_label = str_to_array(g.node[node]["label"]) csnh = np.bitwise_xor( np.roll( node_label, 1 ), x ) else: csnh = str_to_array(g.node[node]["label"]) gnh.node[node]["label"] = csnh return gnh
def main(): mp.figure(1) mp.clf() ax = mp.subplot(211) #ax = mp.subplot(211) #x, y = testData() x, y, flag = jupiter1() cin = np.arange(len(x)) #import pdb; pdb.set_trace() idx = noise.singlePointDifferenceSigmaClip(y, 4, initialClip=flag) mp.plot(cin[~flag],y[~flag], 'k.') mp.plot(cin[idx], y[idx], 'ro', ms=10) mp.plot(cin[flag], y[flag], 'gs') mp.axvline(499, color='b') mp.subplot(212, sharex=ax) mp.plot(cin, y - np.roll(y, -1), 'ko-') mp.axvline(499, color='b') outliers = np.where(np.bitwise_xor(idx, flag))[0] outliers = np.bitwise_xor(idx, flag).astype(int) mp.figure(2) mp.clf() mp.plot(np.convolve(outliers, outliers, mode='same'))
def reconciliate(matlabEng, arrDelta, arrData_bin_2, n, k, m, strCoder): """ Given the delta, try to deduce data1 via reconciliation """ arrMsg_bin_2 = None arrCodeword_bin_2 = None if (strCoder == CODER_GOLAY): n = 23 k = 12 arrMsg_bin_2 = golay.decode(matlabEng, np.bitwise_xor(arrData_bin_2, arrDelta), n) arrCodeword_bin_2 = golay.encode(matlabEng, arrMsg_bin_2, k) elif (strCoder == CODER_RS): arrMsg_bin_2 = rs.decode(matlabEng, np.bitwise_xor(arrData_bin_2, arrDelta), n, k, m) arrCodeword_bin_2 = rs.encode(matlabEng, arrMsg_bin_2, n, k, m) elif (strCoder == CODER_HAMMING): arrMsg_bin_2 = fec.decode(matlabEng, np.bitwise_xor(arrData_bin_2, arrDelta), n, k, strCoder) arrCodeword_bin_2 = fec.encode(matlabEng, arrMsg_bin_2, n, k) else: raise ValueError("Unkown coder") # deduce data 1 from data 2 + delta arrDeducedData_bin_1 = np.bitwise_xor(arrCodeword_bin_2, arrDelta) return arrDeducedData_bin_1
def unmask(buf, f): s2a = lambda s: [ord(c) for c in s] s2b = lambda s: s pstart = f['hlen'] + 4 pend = pstart + f['length'] if numpy: b = c = s2b('') if f['length'] >= 4: mask = numpy.frombuffer(buf, dtype=numpy.dtype('<u4'), offset=f['hlen'], count=1) data = numpy.frombuffer(buf, dtype=numpy.dtype('<u4'), offset=pstart, count=int(f['length'] / 4)) b = numpy.bitwise_xor(data, mask).tostring() if f['length'] % 4: mask = numpy.frombuffer(buf, dtype=numpy.dtype('B'), offset=f['hlen'], count=(f['length'] % 4)) data = numpy.frombuffer(buf, dtype=numpy.dtype('B'), offset=pend - (f['length'] % 4), count=(f['length'] % 4)) c = numpy.bitwise_xor(data, mask).tostring() return b + c else: data = array.array('B') mask = s2a(f['mask']) data.fromstring(buf[pstart:pend]) for i in range(len(data)): data[i] ^= mask[i % 4] return data.tostring()
def sign_change(img): img = img.reshape(28, 28) top = numpy.bitwise_xor(img, shift_up(img)) top = [top[:, i].sum() / 255 for i in xrange(img.shape[1])] side = numpy.bitwise_xor(img, shift_left(img)) side = [side[i, :].sum() / 255 for i in xrange(img.shape[1])] return numpy.array(top), numpy.array(side)
def chi(ain): aout = np.zeros((5,5,64), dtype = int) # Initialize empty 5x5x64 array for i in range(5): for j in range(5): for k in range(64): xor = np.bitwise_xor(ain[(i+1)%5][j][k], 1) mul = xor * (ain[(i+2)%5][j][k]) aout[i][j][k] = np.bitwise_xor(ain[i][j][k], mul) return aout
def train_and_test(X_train,Y_train,X_test,Y_test,dtype,alpha,beta_percentile,cv=True): k = X_train.shape[0] n = X_test.shape[0] A = np.zeros(shape=(k,k)) lambd = 1 part1 = np.column_stack((X_train['guarantee_type'], X_train['gender'])) part2 = np.column_stack((X_train['age'], X_train['account_value'], X_train['withdrawal_rate'],X_train['maturity'])) for i in range(k): tmp = np.array([X_train[i],]*k, dtype=dtype) tmp1 = np.column_stack((tmp['guarantee_type'], tmp['gender'])) #copy this row k times tmp2 = np.column_stack((tmp['age'], tmp['account_value'], tmp['withdrawal_rate'],tmp['maturity'])) result1 = lambd*np.sum(np.bitwise_xor(part1, tmp1), axis=1) result2 = np.sum((part2-tmp2)**2,axis=1) A[i] = np.sqrt(result1+result2) # tmp = np.triu(A) # tmp = tmp[tmp!=0] max_dist = A.max() B = np.zeros(shape=(k+1,k+1)) B[k] = B[:,k] = 1 B[k,k] = 0 part1 = np.column_stack((X_test['guarantee_type'], X_test['gender'])) part2 = np.column_stack((X_test['age'], X_test['account_value'], X_test['withdrawal_rate'],X_test['maturity'])) weight_mat = np.zeros((k+1,n)) weight_mat[k] = 1 beta = max_dist*beta_percentile for j in range(k): B[j][:k] = alpha+np.exp(-3*A[j]/beta) tmp = np.array([X_train[j],]*n, dtype=dtype) tmp1 = np.column_stack((tmp['guarantee_type'], tmp['gender'])) #copy this row k times tmp2 = np.column_stack((tmp['age'], tmp['account_value'], tmp['withdrawal_rate'],tmp['maturity'])) result1 = lambd*np.sum(np.bitwise_xor(part1, tmp1), axis=1) result2 = np.sum((part2-tmp2)**2,axis=1) weight_mat[j] = alpha+np.exp(np.sqrt(result1+result2)*(-3/beta)) Y_train = np.append(Y_train, 0) const = np.dot(np.linalg.inv(B),Y_train) Y_hat = np.dot(const, weight_mat) if cv == True: RMSE = np.sqrt(sum((Y_test-Y_hat)**2)/n) return RMSE else: RMSE = np.sqrt(sum((Y_test-Y_hat)**2)/(n+k)) MAD = sum(abs(Y_test-Y_hat))/(n+k) total = sum(Y_hat)+sum(Y_train) benchmark = sum(Y_test)+sum(Y_train) APD = abs(total-benchmark) RPD = APD/benchmark return APD,RPD,MAD,RMSE
def xor_arrays(cls, source, target): """ Moved to its own function for profiling purposes. May want to consider moving out to inline. The source array will be xored into place into the target array Arguments: source -- Numpy array source array target -- Numpy array target array """ numpy.bitwise_xor(source, target, target)
def key_expansion(self, cipher_key): if self.is_verbose: print "Computing key schedule..." key_schedule = np.copy(cipher_key) n_k = len(cipher_key) n_r = n_k + 6 if self.is_verbose: print print " After After Rcon XOR" print " i Previous RotWord SubWord Value w/ Rcon w[i-Nk] Final" print "=== ======== ======== ======== ======== ======== ======== ========" for i in range(n_k, N_B * (n_r + 1)): prev = key_schedule[i - 1] first = key_schedule[i - n_k] temp = prev after_rot_word = None after_sub_word = None rcon_val = None after_xor_rcon = None if i % n_k == 0: after_rot_word = rot_word(prev) after_sub_word = sub_word(after_rot_word) rcon_val = rcon(i / n_k) after_xor_rcon = np.bitwise_xor(after_sub_word, rcon_val) temp = after_xor_rcon elif n_k > 6 and i % n_k == 4: after_sub_word = sub_word(prev) temp = after_sub_word final = np.bitwise_xor(first, temp) key_schedule = np.append(key_schedule, final.reshape(1, 4), axis=0) if self.is_verbose: print "{:02}: {} {} {} {} {} {} {}".format( i, w2s(prev), w2s(after_rot_word), w2s(after_sub_word), w2s(rcon_val), w2s(after_xor_rcon), w2s(first), w2s(final), ) if self.is_verbose: print return key_schedule, n_k, n_r
def expand_key(self, key): self.round_keys = [] self.round_keys.append(key) for i in range(1, NUM_ROUNDS+1): # Always transpose, words are columns and not rows previous_key = self.round_keys[i-1].transpose() round_key = np.zeros(key.shape, dtype=np.uint8) round_key[0] = np.bitwise_xor(self.g(previous_key[3], i), previous_key[0]) round_key[1] = np.bitwise_xor(round_key[0], previous_key[1]) round_key[2] = np.bitwise_xor(round_key[1], previous_key[2]) round_key[3] = np.bitwise_xor(round_key[2], previous_key[3]) self.round_keys.append(round_key.transpose())
def add_noise(y, x, noise_y_rate, noise_x_rate): assert(noise_y_rate >= 0 and noise_y_rate <= 1) assert(noise_x_rate >= 0 and noise_x_rate <= 1) tmp_y = (y + np.ones_like(y)) / 2 noise_y = np.random.random(y.shape) < noise_y_rate new_y = np.bitwise_xor(tmp_y, noise_y) new_y = (2 * new_y) - np.ones_like(new_y) noise_x = np.random.random(x.shape) < noise_x_rate new_x = np.bitwise_xor(x, noise_x) return (new_y, new_x)
def ErrorCorrection(self,syndrome,inBits): # LUT based error correction # errVector = find( errorSyndromes == syndrome ); # if numel(errVector)>1 # disp('Help!'); # end # if isempty(errVector) # failure = true; # else # inBits = bitxor(inBits,errorVectors(errVector(1),:)); # failure = false; # end corrected = 0; if(syndrome): # Meggitt algorithm, only correct data bits (16), not the parity bits for ndx in range(16): # The first (most significant) bit is one if (np.bitwise_and(syndrome,512)): # If the first bit is a one and the last 5 (least significant bits) # are zero, this indicates an error at the current bit (ndx) position if (np.bitwise_and(syndrome,31) == 0): # The code can correct bursts up to 5 bits long. Check to see if # the error is a burst or not. If it isn't a burst, it isn't # correctable, return immediately with a failure. tmp = np.bitwise_and(syndrome,480); if ~(tmp == 480 | tmp == 448 | tmp == 384 | tmp == 256 | tmp == 0): break; # The error appears to be a burst error, attempt to correct inBits[ndx] = np.bitwise_xor(inBits(ndx),1); # Shift the syndrome syndrome = np.left_shift(syndrome,1); else: # Least significant bits do not indicate the current bit (ndx) is # an error in a burst. Continue shifting the syndrome and then apply the # generator polynomial. syndrome = np.left_shift(syndrome,1); syndrome = np.bitwise_xor(syndrome,441); else: # Not a one at the first (most significant) syndrome bit, applying generator polynomial # is trivial in this case. syndrome = np.left_shift(syndrome,1); # If after this process the syndrome is not zero, there was a # uncorrectable error. if (np.bitwise_and(syndrome,1023)==0): corrected = 1; return (inBits, corrected);
def ring(x, y, height, thickness, gaussian_width): """ Circular ring (annulus) with Gaussian fall-off after the solid ring-shaped region. """ radius = height/2.0 half_thickness = thickness/2.0 distance_from_origin = np.sqrt(x**2+y**2) distance_outside_outer_disk = distance_from_origin - radius - half_thickness distance_inside_inner_disk = radius - half_thickness - distance_from_origin ring = 1.0-np.bitwise_xor(np.greater_equal(distance_inside_inner_disk,0.0), np.greater_equal(distance_outside_outer_disk,0.0)) sigmasq = gaussian_width*gaussian_width if sigmasq==0.0: inner_falloff = x*0.0 outer_falloff = x*0.0 else: with float_error_ignore(): inner_falloff = np.exp(np.divide(-distance_inside_inner_disk*distance_inside_inner_disk, 2.0*sigmasq)) outer_falloff = np.exp(np.divide(-distance_outside_outer_disk*distance_outside_outer_disk, 2.0*sigmasq)) return np.maximum(inner_falloff,np.maximum(outer_falloff,ring))
def decrypt_bytes(bytes_in, key): # get the function from the cuda_kernel func = sm.get_function("decipher") iv = np.array([1,2], dtype=np.uint32) ha = np.fromstring(hashlib.md5(key).digest(), np.uint32) output = np.empty_like(bytes_in) prev_decrypt = iv length = len(bytes_in) # assign a number < 1024 for the number of threads for each block num_threads = 256 # number of blocks is the total length divided by 2, divided for the number of threads per block and + 1 in case of rest # /2 because in decipher each part of the message is processed in pair num_blocks = ((length / 2) / num_threads) + 1 # call the function decipher and generate the output [v1,v2] nump array decipher_output = np.empty(length, dtype=np.uint32) func(np.int32(32), drv.In(bytes_in), drv.In(ha), drv.InOut(decipher_output), np.int32(length), block=(num_threads,1,1), grid=(num_blocks,1,1)) # now in decipher_output there are all the v0 and v1 couple i = 0; while(i < length - 1): output[i:i+2] = np.bitwise_xor(decipher_output[i:i+2], prev_decrypt) prev_decrypt = bytes_in[i:i+2] i += 2 return output
def test(self): N, D = self.XY_test.shape Y = self.XY_test[:,-2:] pred = self.classify() TP = FP = FN = TN = 0 res = np.empty((N,)) for i in xrange(N): l1 = pred[i] l2 = pred[N+i] #print 'pred',l1,l2 #print 'Y',Y[i,0],Y[i,1] if l1 == l2 and Y[i,0] == Y[i,1]: TP += 1 elif l1 == l2 and Y[i,0] != Y[i,1]: FP += 1 elif l1 != l2 and Y[i,0] == Y[i,1]: FN += 1 elif l1 != l2 and Y[i,0] != Y[i,1]: TN += 1 res[i] = 1-np.bitwise_xor(int(np.sign(abs(l1-l2))),int(np.sign(abs(Y[i,0]-Y[i,1])))) print 'TP:',TP print 'FP:',FP print 'FN:',FN print 'TN:',TN print 'Accuracy:',float(np.sum(res))/N sys.stdout.flush()
def execute(self, slot, subindex, roi, result): assert slot == self.Output, "Unknown output slot: {}".format( slot.name ) if self.SelectedLabel.value == 0: result[:] = 0 else: # Can't use writeInto() here because dtypes don't match. inputLabels = self.Input(roi.start, roi.stop).wait() # Use two in-place bitwise operations instead of numpy.where # This avoids the temporary variable created by (inputLabels == x) #result[:] = numpy.where( inputLabels == self.SelectedLabel.value, 1, 0 ) numpy.bitwise_xor(inputLabels, self.SelectedLabel.value, out=inputLabels) # All numpy.logical_not(inputLabels, out=inputLabels) result[:] = inputLabels # Copy from uint32 to uint8 return result
def unmake_packet(whitened_payload_with_crc, whitener_offset=0, dewhitening=True): """ Return (payload) @param whitened_payload_with_crc: string """ whitener_offset=0 dewhitening = True if dewhitening: i = frombuffer(whitened_payload_with_crc, dtype = byte) j = frombuffer(random_mask[0:len(whitened_payload_with_crc)], dtype = byte) try: payload = (bitwise_xor(i, j)).tostring() except: print "Error: receiving arguments do not have equal length!" len(i) len(j) else: payload = (whitened_payload_with_crc) if 0: print "payload_with_crc =", string_to_hex_list(payload_with_crc) print "ok = %r, len(payload) = %d" % (ok, len(payload)) print "payload =", string_to_hex_list(payload) return payload
def get_synthetic_clusters_dataset(n_clusters = 4, n_dims = 20, n_training = 1000, n_test = 200, sparsity = 0.5, flip_noise = 0.1, seed = 3425): """ A dataset consisting of clustered binary data with "bit-flip" noise, and the corresponding cluster labels. This should be trivially solvable by any classifier, and serves as a basic test of whether your classifier is completely broken or not. :param n_clusters: :param n_dims: :param n_samples_training: :param n_samples_test: :param sparsity: :param flip_noise: :param seed: :return: """ rng = np.random.RandomState(seed) labels = rng.randint(n_clusters, size = n_training+n_test) # (n_samples, ) centers = rng.rand(n_clusters, n_dims) < sparsity # (n_samples, n_dims) input_data = centers[labels] input_data = np.bitwise_xor(input_data, rng.rand(*input_data.shape) < flip_noise) return DataSet( training_set = DataCollection(input_data[:n_training], labels[:n_training]), test_set = DataCollection(input_data[n_training:], labels[n_training:]), name = 'Synthetic Clusters Dataset' )
def k_cross_validation(k, N, X, y, classifier): indexSet = set([i for i in range(N)]) partitionDict = dict() for i in range(k): partitionDict[i] = [np.matrix(np.empty((0, np.size(X, 1)), X[0, 0].dtype)), np.matrix(np.empty((0, 1), y[0, 0].dtype))] for _ in range(N): partitionKey = _ % k i = random.sample(indexSet, 1)[0] indexSet.remove(i) partitionDict[partitionKey][0] = np.append(partitionDict[partitionKey][0], X[i], axis=0) partitionDict[partitionKey][1] = np.append(partitionDict[partitionKey][1], y[i], axis=0) for key in partitionDict: validationPartition = partitionDict[key] validation_X = validationPartition[0] validation_y = validationPartition[1] train_X = np.matrix(np.empty((0, np.size(X, 1)), X[0, 0].dtype)) train_y = np.matrix(np.empty((0, 1), y[0, 0].dtype)) for otherKey in partitionDict: if otherKey != key: trainingPartition = partitionDict[otherKey] train_X = np.append(train_X, trainingPartition[0], axis=0) train_y = np.append(train_y, trainingPartition[1], axis=0) classifier.train(train_X, train_y) y_hat = np.matrix(classifier.predict(validation_X), dtype=validation_y.dtype) print("validation error rate is: {0}".format( (np.sum(np.bitwise_xor(y_hat, validation_y)) / np.size(validation_y, 0))))
def process_frame(frame, filename=None): ##frame[:,:,2]=0 ##frame[:,:,1]=0 ##frame[:,:,0]=0 ##print frame # Keep a copy of the frame, we need it later. orig_frame = numpy.copy(frame) new_frame_for_show = numpy.copy(frame) new_frame_for_show[:,:,:2]=0 # Remove the R and G layers new_frame = frame[:,:,2] #mpimg.imsave('filteredRGB'+filename, new_frame_for_show, format='png') new_new_frame = cv2.filter2D(new_frame, -1, smoothing_kernel2D(25)) #mpimg.imsave('SmoothedFilteredRGB'+filename, new_frame, format='png') # Histogram plot, helps choose threshold #hist = cv2.calcHist([new_new_frame], [0], None, 256, [0,256]) #plt.hist(new_new_frame.ravel(), 256, [0,256]); #plt.show() #print "NEW_FRAME", type(new_new_frame) #print "Max, new_new_frame", numpy.amax(new_new_frame) # Binarize the image thresholding_filter = numpy.vectorize(lambda x: 255 if x>30 else 0) filtered_frame = thresholding_filter(new_new_frame) #print "max, filtered", numpy.amax(filtered_frame) #print filtered_frame # Find and Mark center of frame cx, cy = FindCenter(filtered_frame) if cx and cy: filtered_frame[int(cy)-5:int(cy)+5, int(cx)-5:int(cx)+5] = 128 mask = numpy.ndarray(filtered_frame.shape, dtype=int) mask[:,:] = 255 #print "MASK:", mask inverted_filter = numpy.bitwise_xor(mask, filtered_frame) frame_ball_marked = numpy.bitwise_and(orig_frame, numpy.dstack([inverted_filter]*3)) # Show grey scale if filename: plt.imshow(filtered_frame, cmap=cm.Greys_r) plt.savefig(filename+".png", format='png') #plt.show() plt.clf() plt.imshow(frame_ball_marked) #plt.imshow(frame) #plt.savefig(filename+"Marked.png", format='png') else: #plt.show() pass return cx, cy
def label_state(comm): """ Label/personalize the random generator state for the local rank.""" def get_mask(rank): """ Get a uint32 mask array to use to xor the random generator state. We do not simply return the rank, as this small change to the state of the Mersenne Twister only causes small changes in the generated sequence. (The generators will eventually diverge, but this takes a while.) So we scramble the mask up a lot more, still deterministically, using a cryptographic hash. See: http://en.wikipedia.org/wiki/Mersenne_twister#Disadvantages """ # Since we will be converting to/from bytes, endianness is important. uint32be = np.dtype('>u4') x = np.empty([624 // 8, 2], dtype=uint32be) # The hash of the rank catted with an increasing index # (stuffed into big-endian uint32s) are hashed with SHA-256 to make # the XOR mask for 8 consecutive uint32 words for a 624-word # Mersenne Twister state. x[:, 0] = rank x[:, 1] = np.arange(624 // 8) mask_buffer = b''.join(sha256(row).digest() for row in x) # And convert back to native-endian. mask = np.frombuffer(mask_buffer, dtype=uint32be).astype(np.uint32) return mask rank = comm.Get_rank() mask = get_mask(rank) # For the Mersenne Twister used by numpy, the state is a 5-tuple, # with the important part being an array of 624 uint32 values. # We xor the mask into that array, and leave the rest of the tuple alone. s0, orig_array, s2, s3, s4 = np.random.get_state() mod_array = np.bitwise_xor(orig_array, mask) np.random.set_state((s0, mod_array, s2, s3, s4))
def compare(self, im): """Show differences between images :Parameters: - `im`: the comparison image @Returns: a difference image """ header_diff = HeaderDifference(self, im) hdus = (('science', self.data, im.data), ('mask', self.mask, im.mask), ('weight', self.weight, im.weight)) diff_im = DESImage() for ext, im1, im2 in hdus: if ext=='science': diff_im.data = im1-im2 elif ext=='mask': diff_im.mask = np.bitwise_xor(im1, im2) elif ext=='weight': diff_im.weight = im1-im2 comparison = DESImageComparison(header_diff, diff_im) return comparison
def computeDelta(matlabEng, arrData_bin_1, n, k, m, strCoder): """ compute the delta for reconciliation """ # find corresponding codeword of data 1 arrMsg_bin_1 = None arrCodeword_bin_1 = None if(strCoder == CODER_GOLAY): n = 23 k = 12 arrMsg_bin_1 = golay.decode(matlabEng, arrData_bin_1, n) arrCodeword_bin_1 = golay.encode(matlabEng, arrMsg_bin_1, k) elif (strCoder == CODER_RS): arrMsg_bin_1 = rs.decode(matlabEng, arrData_bin_1, n, k, m) arrCodeword_bin_1 = rs.encode(matlabEng, arrMsg_bin_1, n, k, m) elif(strCoder == CODER_HAMMING): arrMsg_bin_1 = fec.decode(matlabEng, arrData_bin_1, n, k, strCoder) arrCodeword_bin_1 = fec.encode(matlabEng, arrMsg_bin_1, n, k, strCoder) else: raise ValueError("Unkown coder") # compute the difference arrDelta = np.bitwise_xor(arrData_bin_1, arrCodeword_bin_1) return arrDelta
def arithmetic_phase(self, operation, node): """ Arithmetic phase """ for case in Switch(operation): if case('AND'): self.output[:, node] = np.bitwise_and(self.input1[:, node], self.input2[:, node]) break if case('OR'): self.output[:, node] = np.bitwise_or(self.input1[:, node], self.input2[:, node]) break if case('XOR'): self.output[:, node] = np.bitwise_xor(self.input1[:, node], self.input2[:, node]) break if case('NOR'): self.output[:, node] = np.bitwise_nor(self.input1[:, node]) # Single operand instruction break if case('ADD'): self.output[:, node] = np.add(self.input1[:, node], self.input2[:, node]) break if case('SUB'): self.output[:, node] = np.subtract(self.input1[:, node], self.input2[:, node]) break if case('MULT'): self.output[:, node] = np.multiply(self.input1[:, node], self.input2[:, node]) break if case(): # Default print("Error! Undefined instruction ", operation) exit()
def query_with_distances(self, v, n): """Find indices of `n` most similar vectors from the index to query vector `v`.""" if self._metric == 'hamming': v = numpy.packbits(v) if self._metric != 'jaccard': # use same precision for query as for index v = numpy.ascontiguousarray(v, dtype = self.index.dtype) # HACK we ignore query length as that's a constant not affecting the final ordering if self._metric == 'angular': # argmax_a cossim(a, b) = argmax_a dot(a, b) / |a||b| = argmin_a -dot(a, b) dists = -numpy.dot(self.index, v) elif self._metric == 'euclidean': # argmin_a (a - b)^2 = argmin_a a^2 - 2ab + b^2 = argmin_a a^2 - 2ab dists = self.lengths - 2 * numpy.dot(self.index, v) elif self._metric == 'hamming': diff = numpy.bitwise_xor(v, self.index) pc = BruteForceBLAS.popcount den = float(len(v) * 8) dists = [sum([pc[part] for part in point]) / den for point in diff] elif self._metric == 'jaccard': dists = [pd[self._metric]['distance'](v, e) for e in self.index] else: assert False, "invalid metric" # shouldn't get past the constructor! nearest_indices = numpy.argpartition(dists, n)[:n] # partition-sort by distance, get `n` closest indices = [idx for idx in nearest_indices if pd[self._metric]["distance_valid"](dists[idx])] def fix(index): ep = self.index[index] ev = v if self._metric == "hamming": ep = numpy.unpackbits(ep) ev = numpy.unpackbits(ev) return (index, pd[self._metric]['distance'](ep, ev)) return map(fix, indices)
def get_groupacc(finetuned_model_pr, concept_arraynew2, f_train, f_val, concept, n_concept, n_cluster, n0, verbose): """Gets the group accuracy for discovered concepts.""" print(finetuned_model_pr.summary()) min_weight = finetuned_model_pr.layers[-5].get_weights()[0] sim_array = np.zeros((n_cluster, n_concept)) for j in range(n_cluster): sim_array[j, :] = np.mean(np.matmul(concept_arraynew2[j, :100, :], min_weight), axis=0) posneg = np.zeros(5) sim_array_0mean = sim_array - np.mean(sim_array, axis=0) max_cluster = np.argmax(np.abs(sim_array_0mean), axis=0) for count in range(5): posneg[count] = sim_array_0mean[max_cluster[count], count] > 0 loss_table = np.zeros((5, 5)) for count in range(5): for count2 in range(5): # count2 = max_cluster[count] mean0 = np.mean( np.matmul(f_train, min_weight[:, count])[concept[:n0, count2] == 0]) * 100 mean1 = np.mean( np.matmul(f_train, min_weight[:, count])[concept[:n0, count2] == 1]) * 100 if mean0 < mean1: pos = 1 else: pos = -1 best_err = 1e10 best_bias = 0 a = int((mean1 - mean0) / 20) if a == 0: a = pos for bias in range(int(mean0), int(mean1), a): if pos == 1: if np.sum( np.bitwise_xor( concept[:n0, count2], np.matmul(f_train, min_weight[:, count]) > bias / 100.)) < best_err: best_err = np.sum( np.bitwise_xor( concept[:n0, count2], np.matmul(f_train, min_weight[:, count]) > bias / 100.)) best_bias = bias else: if np.sum( np.bitwise_xor( concept[:n0, count2], np.matmul(f_train, min_weight[:, count]) < bias / 100.)) < best_err: best_err = np.sum( np.bitwise_xor( concept[:n0, count2], np.matmul(f_train, min_weight[:, count]) < bias / 100.)) best_bias = bias if pos == 1: loss_table[count, count2] = np.sum( np.bitwise_xor( concept[n0:, count2], np.matmul(f_val, min_weight[:, count]) > best_bias / 100.)) / 12000 if verbose: print( np.sum( np.bitwise_xor( concept[n0:, count2], np.matmul(f_val, min_weight[:, count]) > best_bias / 100.)) / 12000) else: loss_table[count, count2] = np.sum( np.bitwise_xor( concept[n0:, count2], np.matmul(f_val, min_weight[:, count]) < best_bias / 100.)) / 12000 if verbose: print( np.sum( np.bitwise_xor( concept[n0:, count2], np.matmul(f_val, min_weight[:, count]) < best_bias / 100.)) / 12000) print(np.amin(loss_table, axis=0)) acc = np.mean(np.amin(loss_table, axis=0)) print(acc) return min_weight, acc
def numpy_bitwise_xor(x1, x2): x1 = dpnp.asnumpy(x1) if isinstance(x1, dparray) else x1 x2 = dpnp.asnumpy(x2) if isinstance(x2, dparray) else x2 return numpy.bitwise_xor(x1, x2)
def hammingB(x, y): # This function computes the normalised hamming distance between two binary vectors # h = 1.0 - float(np.sum(np.logical_xor(X, Y)))/len(X) # count is faster than sum return 1.0 - float(np.count_nonzero(np.bitwise_xor(x, y))) / len(x)
def bitwiseXOR(img1, img2): new = np.bitwise_xor(img1, img2) return new
def hamming(x, y): x = np.asarray(x, np.bool) y = np.asarray(y, np.bool) return np.bitwise_xor(x, y).sum()
size[0], size[1], suffix), dtype=ref_dtype) out = np.fromfile('data/output/{}.bin'.format(suffix), dtype=out_dtype) # ref=ref.reshape((64,*size)) # out=out.reshape((64,*size)) # print(ref[8]) # print(out[8]) # for i in range(64): # print((ref[i]-out[i]).sum()) for i in range(5): print(bin(ref[i]), " ", end='') print("\n") for i in range(5): print(bin(out[i]), " ", end='') print("\n") mean = np.bitwise_xor(ref, out).mean() elif 'conv' in suffix and suffix not in ['conv0', 'conv16']: ref = np.fromfile('data/ref/{}x{}/{}.bin'.format( size[0], size[1], suffix), dtype=ref_dtype).reshape((*size, 64)) out = np.fromfile('data/output/{}.bin'.format(suffix), dtype=out_dtype).reshape((*size, 64)) # out2=out out2 = np.zeros_like(out, dtype=np.int32) for k in range(0, 64): for j in range(0, size[0]): for i in range(0, size[1]): T = 9 * 64 if (i == 0 or i == size[1] - 1): T -= 3 * 64 if (j == 0 or j == size[0] - 1):
def i4_sobol(dim_num, seed): """ I4_SOBOL generates a new quasirandom Sobol vector with each call. Discussion: The routine adapts the ideas of Antonov and Saleev. Licensing: This code is distributed under the MIT license. Modified: 22 February 2011 Author: Original FORTRAN77 version by Bennett Fox. MATLAB version by John Burkardt. PYTHON version by Corrado Chisari Reference: Antonov, Saleev, USSR Computational Mathematics and Mathematical Physics, olume 19, 19, pages 252 - 256. Paul Bratley, Bennett Fox, Algorithm 659: Implementing Sobol's Quasirandom Sequence Generator, ACM Transactions on Mathematical Software, Volume 14, Number 1, pages 88-100, 1988. Bennett Fox, Algorithm 647: Implementation and Relative Efficiency of Quasirandom Sequence Generators, ACM Transactions on Mathematical Software, Volume 12, Number 4, pages 362-376, 1986. Ilya Sobol, USSR Computational Mathematics and Mathematical Physics, Volume 16, pages 236-242, 1977. Ilya Sobol, Levitan, The Production of Points Uniformly Distributed in a Multidimensional Cube (in Russian), Preprint IPM Akad. Nauk SSSR, Number 40, Moscow 1976. Parameters: Input, integer DIM_NUM, the number of spatial dimensions. DIM_NUM must satisfy 1 <= DIM_NUM <= 40. Input/output, integer SEED, the "seed" for the sequence. This is essentially the index in the sequence of the quasirandom value to be generated. On output, SEED has been set to the appropriate next value, usually simply SEED+1. If SEED is less than 0 on input, it is treated as though it were 0. An input value of 0 requests the first (0-th) element of the sequence. Output, real QUASI(DIM_NUM), the next quasirandom vector. """ global atmost global dim_max global dim_num_save global initialized global lastq global log_max global maxcol global poly global recipd global seed_save global v if not initialized or dim_num != dim_num_save: initialized = 1 dim_max = 40 dim_num_save = -1 log_max = 30 seed_save = -1 # # Initialize (part of) V. # v = np.zeros((dim_max, log_max)) v[0:40, 0] = np.transpose([ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ]) v[2:40, 1] = np.transpose([ 1, 3, 1, 3, 1, 3, 3, 1, 3, 1, 3, 1, 3, 1, 1, 3, 1, 3, 1, 3, 1, 3, 3, 1, 3, 1, 3, 1, 3, 1, 1, 3, 1, 3, 1, 3, 1, 3 ]) v[3:40, 2] = np.transpose([ 7, 5, 1, 3, 3, 7, 5, 5, 7, 7, 1, 3, 3, 7, 5, 1, 1, 5, 3, 3, 1, 7, 5, 1, 3, 3, 7, 5, 1, 1, 5, 7, 7, 5, 1, 3, 3 ]) v[5:40, 3] = np.transpose([ 1, 7, 9, 13, 11, 1, 3, 7, 9, 5, 13, 13, 11, 3, 15, 5, 3, 15, 7, 9, 13, 9, 1, 11, 7, 5, 15, 1, 15, 11, 5, 3, 1, 7, 9 ]) v[7:40, 4] = np.transpose([ 9, 3, 27, 15, 29, 21, 23, 19, 11, 25, 7, 13, 17, 1, 25, 29, 3, 31, 11, 5, 23, 27, 19, 21, 5, 1, 17, 13, 7, 15, 9, 31, 9 ]) v[13:40, 5] = np.transpose([ 37, 33, 7, 5, 11, 39, 63, 27, 17, 15, 23, 29, 3, 21, 13, 31, 25, 9, 49, 33, 19, 29, 11, 19, 27, 15, 25 ]) v[19:40, 6] = np.transpose([ 13, 33, 115, 41, 79, 17, 29, 119, 75, 73, 105, 7, 59, 65, 21, 3, 113, 61, 89, 45, 107 ]) v[37:40, 7] = np.transpose([7, 23, 39]) # # Set POLY. # poly = [ 1, 3, 7, 11, 13, 19, 25, 37, 59, 47, 61, 55, 41, 67, 97, 91, 109, 103, 115, 131, 193, 137, 145, 143, 241, 157, 185, 167, 229, 171, 213, 191, 253, 203, 211, 239, 247, 285, 369, 299 ] atmost = 2**log_max - 1 # # Find the number of bits in ATMOST. # maxcol = i4_bit_hi1(atmost) # # Initialize row 1 of V. # v[0, 0:maxcol] = 1 # Things to do only if the dimension changed. if dim_num != dim_num_save: # # Check parameters. # if (dim_num < 1 or dim_max < dim_num): print('I4_SOBOL - Fatal error!') print(' The spatial dimension DIM_NUM should satisfy:') print(' 1 <= DIM_NUM <= %d' % dim_max) print(' But this input value is DIM_NUM = %d' % dim_num) return None dim_num_save = dim_num # # Initialize the remaining rows of V. # for i in range(2, dim_num + 1): # # The bits of the integer POLY(I) gives the form of polynomial # I. # # Find the degree of polynomial I from binary encoding. # j = poly[i - 1] m = 0 while True: j = math.floor(j / 2.) if (j <= 0): break m = m + 1 # # Expand this bit pattern to separate components of the logical # array INCLUD. # j = poly[i - 1] includ = np.zeros(m) for k in range(m, 0, -1): j2 = math.floor(j / 2.) includ[k - 1] = (j != 2 * j2) j = j2 # # Calculate the remaining elements of row I as explained # in Bratley and Fox, section 2. # for j in range(m + 1, maxcol + 1): newv = v[i - 1, j - m - 1] l_var = 1 for k in range(1, m + 1): l_var = 2 * l_var if (includ[k - 1]): newv = np.bitwise_xor(int(newv), int(l_var * v[i - 1, j - k - 1])) v[i - 1, j - 1] = newv # # Multiply columns of V by appropriate power of 2. # l_var = 1 for j in range(maxcol - 1, 0, -1): l_var = 2 * l_var v[0:dim_num, j - 1] = v[0:dim_num, j - 1] * l_var # # RECIPD is 1/(common denominator of the elements in V). # recipd = 1.0 / (2 * l_var) lastq = np.zeros(dim_num) seed = int(math.floor(seed)) if (seed < 0): seed = 0 if (seed == 0): l_var = 1 lastq = np.zeros(dim_num) elif (seed == seed_save + 1): # # Find the position of the right-hand zero in SEED. # l_var = i4_bit_lo0(seed) elif (seed <= seed_save): seed_save = 0 lastq = np.zeros(dim_num) for seed_temp in range(int(seed_save), int(seed)): l_var = i4_bit_lo0(seed_temp) for i in range(1, dim_num + 1): lastq[i - 1] = np.bitwise_xor(int(lastq[i - 1]), int(v[i - 1, l_var - 1])) l_var = i4_bit_lo0(seed) elif (seed_save + 1 < seed): for seed_temp in range(int(seed_save + 1), int(seed)): l_var = i4_bit_lo0(seed_temp) for i in range(1, dim_num + 1): lastq[i - 1] = np.bitwise_xor(int(lastq[i - 1]), int(v[i - 1, l_var - 1])) l_var = i4_bit_lo0(seed) # # Check that the user is not calling too many times! # if maxcol < l_var: print('I4_SOBOL - Fatal error!') print(' Too many calls!') print(' MAXCOL = %d\n' % maxcol) print(' L = %d\n' % l_var) return None # # Calculate the new components of QUASI. # quasi = np.zeros(dim_num) for i in range(1, dim_num + 1): quasi[i - 1] = lastq[i - 1] * recipd lastq[i - 1] = np.bitwise_xor(int(lastq[i - 1]), int(v[i - 1, l_var - 1])) seed_save = seed seed = seed + 1 return [quasi, seed]
def create_dataset(skip, n_sample=60000): """Creates toy dataset and save to disk.""" assert n_sample > 15 # or it will clearly bug concept = np.reshape(np.random.randint( 2, size=15 * n_sample), (-1, 15)).astype( np.bool_) # shape (n_sample, 15), type boolean, (60000, 15) concept[:15, :15] = np.eye(15) # hardcode some entries to 1 # TODO WHY fig = Figure(figsize=(3, 3)) canvas = FigureCanvas(fig) axes = fig.gca() axes.set_xlim([0, 10]) axes.set_ylim([0, 10]) axes.axis('off') width, height = fig.get_size_inches() * fig.get_dpi() width = int(width) height = int(height) location = [(1.3, 1.3), (3.3, 1.3), (5.3, 1.3), (7.3, 1.3), (1.3, 3.3), (3.3, 3.3), (5.3, 3.3), (7.3, 3.3), (1.3, 5.3), (3.3, 5.3), (5.3, 5.3), (7.3, 5.3), (1.3, 7.3), (3.3, 7.3), (5.3, 7.3)] # all possible locations for shape to appear x = np.zeros( (n_sample, width, height, 3), dtype='uint8') # specify data type to save memory / avoid warning color_array = [ 'green', 'red', 'blue', 'black', 'orange', 'purple', 'yellow' ] if not skip: for i in range(n_sample): ############################################## fig = Figure(figsize=(3, 3)) canvas = FigureCanvas(fig) axes = fig.gca() axes.set_xlim([0, 10]) axes.set_ylim([0, 10]) axes.axis('off') # added by Tony, without reset all x will draw on same canvas, bug ############################################## location_bool = np.zeros( 15 ) # [Added by Tony, or it will inf loop, bug]: for each image, record whether the location has shape already, to avoid overlap if i % 1000 == 0: print('{} images are created'.format(i)) if concept[ i, 5] == 1: # consider concept 5, of image i. 1 means that concept needs to appear in the image generated a = np.random.randint( 15 ) # select a random int to index into variable "location" while location_bool[ a] == 1: # if the place has already been taken by other shape a = np.random.randint( 15 ) # sample it again # these three lines amount to finding a random place in image that is not taken location_bool[a] = 1 # mark that the place chosen is chosen axes.plot( location[a][0], # x pos location[a][1], # y pos 'x', # shape color=color_array[np.random.randint(100) % 7], # a random color markersize=8, # size of shape mew=4, ) # marker edge width # ms=8) # redundant argument. bug. if concept[i, 6] == 1: # repeat the same thing for concept 6 a = np.random.randint(15) while location_bool[a] == 1: a = np.random.randint(15) location_bool[a] = 1 axes.plot( location[a][0], location[a][1], '3', color=color_array[np.random.randint(100) % 7], markersize=8, mew=4, ) # ms=8) if concept[i, 7] == 1: a = np.random.randint(15) while location_bool[a] == 1: a = np.random.randint(15) location_bool[a] = 1 axes.plot( location[a][0], location[a][1], 's', color=color_array[np.random.randint(100) % 7], markersize=8, mew=4, ) # ms=8) if concept[i, 8] == 1: a = np.random.randint(15) while location_bool[a] == 1: a = np.random.randint(15) location_bool[a] = 1 axes.plot( location[a][0], location[a][1], 'p', color=color_array[np.random.randint(100) % 7], markersize=8, mew=4, ) # ms=8) if concept[i, 9] == 1: a = np.random.randint(15) while location_bool[a] == 1: a = np.random.randint(15) location_bool[a] = 1 axes.plot( location[a][0], location[a][1], '_', color=color_array[np.random.randint(100) % 7], markersize=8, mew=4, ) # ms=8) if concept[i, 10] == 1: a = np.random.randint(15) while location_bool[a] == 1: a = np.random.randint(15) location_bool[a] = 1 axes.plot( location[a][0], location[a][1], 'd', color=color_array[np.random.randint(100) % 7], markersize=8, mew=4, ) # ms=8) if concept[i, 11] == 1: a = np.random.randint(15) while location_bool[a] == 1: a = np.random.randint(15) location_bool[a] = 1 axes.plot( location[a][0], location[a][1], 'd', color=color_array[np.random.randint(100) % 7], markersize=8, mew=4, ) # ms=8) if concept[i, 12] == 1: a = np.random.randint(15) while location_bool[a] == 1: a = np.random.randint(15) location_bool[a] = 1 axes.plot( location[a][0], location[a][1], 11, color=color_array[np.random.randint(100) % 7], markersize=8, mew=4, ) # ms=8) if concept[i, 13] == 1: a = np.random.randint(15) while location_bool[a] == 1: a = np.random.randint(15) location_bool[a] = 1 axes.plot( location[a][0], location[a][1], 'o', color=color_array[np.random.randint(100) % 7], markersize=8, mew=4, ) # ms=8) if concept[i, 14] == 1: a = np.random.randint(15) while location_bool[a] == 1: a = np.random.randint(15) location_bool[a] = 1 axes.plot( location[a][0], location[a][1], '.', color=color_array[np.random.randint(100) % 7], markersize=8, mew=4, ) # ms=8) if concept[i, 0] == 1: a = np.random.randint(15) while location_bool[a] == 1: a = np.random.randint(15) location_bool[a] = 1 axes.plot( location[a][0], location[a][1], '+', color=color_array[np.random.randint(100) % 7], markersize=8, mew=4, ) # ms=8) if concept[i, 1] == 1: a = np.random.randint(15) while location_bool[a] == 1: a = np.random.randint(15) location_bool[a] = 1 axes.plot( location[a][0], location[a][1], '1', color=color_array[np.random.randint(100) % 7], markersize=8, mew=4, ) # ms=8) if concept[i, 2] == 1: a = np.random.randint(15) while location_bool[a] == 1: a = np.random.randint(15) location_bool[a] = 1 axes.plot( location[a][0], location[a][1], '*', color=color_array[np.random.randint(100) % 7], markersize=30, mew=3, ) # ms=5) if concept[i, 3] == 1: a = np.random.randint(15) while location_bool[a] == 1: a = np.random.randint(15) location_bool[a] = 1 axes.plot( location[a][0], location[a][1], '<', color=color_array[np.random.randint(100) % 7], markersize=8, mew=4, ) # ms=8) if concept[i, 4] == 1: a = np.random.randint(15) while location_bool[a] == 1: a = np.random.randint(15) location_bool[a] = 1 axes.plot( location[a][0], location[a][1], 'h', color=color_array[np.random.randint(100) % 7], markersize=8, mew=4, ) # ms=8) canvas.draw() image = np.fromstring(canvas.tostring_rgb(), dtype='uint8').reshape(width, height, 3) x[i, :, :, :] = image fig.clf( ) # they also did not close the graph: their cluster probably has 500G memory so it does not matter :) # imgplot = plt.imshow(image) # plt.show() # create label by booling functions y = np.zeros((n_sample, 15)) y[:, 0] = ((1 - concept[:, 0] * concept[:, 2]) + concept[:, 3]) > 0 y[:, 1] = concept[:, 1] + (concept[:, 2] * concept[:, 3]) y[:, 2] = (concept[:, 3] * concept[:, 4]) + (concept[:, 1] * concept[:, 2]) y[:, 3] = np.bitwise_xor(concept[:, 0], concept[:, 1]) y[:, 4] = concept[:, 1] + concept[:, 4] y[:, 5] = (1 - (concept[:, 0] + concept[:, 3] + concept[:, 4])) > 0 y[:, 6] = np.bitwise_xor(concept[:, 1] * concept[:, 2], concept[:, 4]) y[:, 7] = concept[:, 0] * concept[:, 4] + concept[:, 1] y[:, 8] = concept[:, 2] y[:, 9] = np.bitwise_xor(concept[:, 0] + concept[:, 1], concept[:, 3]) y[:, 10] = (1 - (concept[:, 2] + concept[:, 4])) > 0 y[:, 11] = concept[:, 0] + concept[:, 3] + concept[:, 4] y[:, 12] = np.bitwise_xor(concept[:, 1], concept[:, 2]) y[:, 13] = (1 - (concept[:, 0] * concept[:, 4] + concept[:, 3])) > 0 y[:, 14] = np.bitwise_xor(concept[:, 4], concept[:, 3]) np.save('x_data.npy', x) np.save('y_data.npy', y) np.save('concept_data.npy', concept) return width, height
def compute_W(patch, Y, A, C, CdotA, radius, dims): """compute background according to ring model solves the problem min_{W,b0} ||X-W*X|| with X = Y - A*C - b0*1' subject to W(i,j) = 0 for each pixel j that is not in ring around pixel i Problem parallelizes over pixels i Fluctuating background activity is W*X, constant baselines b0. Parameters: ---------- Y: np.ndarray (2D or 3D) movie, raw data in 2D or 3D (pixels x time). A: np.ndarray or sparse matrix spatial footprint of each neuron. C: np.ndarray calcium activity of each neuron. dims: tuple x, y[, z] movie dimensions radius: int radius of ring data_fits_in_memory: [optional] bool If true, use faster but more memory consuming computation Returns: -------- W: scipy.sparse.csr_matrix (pixels x pixels) estimate of weight matrix for fluctuating background b0: np.ndarray (pixels,) estimate of constant background baselines """ from time import time start = time() ring = disk(radius + 1, dtype=bool) ring[1:-1, 1:-1] = np.bitwise_xor(ring[1:-1, 1:-1], disk(radius, dtype=bool)) ringidx = np.vstack(np.where(ring)).T - radius - 1 if isinstance(A, np.ndarray) and isinstance(C, np.ndarray): tmp = C.mean(0).dot(A) elif isinstance(A, h5py._hl.dataset.Dataset) and isinstance(C, h5py._hl.dataset.Dataset): print("TODO") sys.exit() pass tmp2 = np.zeros_like(tmp) chunk_size = Y.chunks[0] for i in range(0, Y.shape[0]+chunk_size,chunk_size): tmp2 += Y[i:i+chunk_size,:].sum(0) tmp2 = tmp2/float(Y.shape[0]) # constante fluorescence is mean(Y) - A * mean(C) b0 = tmp2 - tmp if isinstance(CdotA, np.ndarray): X = (Y - CdotA - b0).T # (pixels, time) elif isinstance(CdotA, h5py._hl.dataset.Dataset): print("TODO") sys.exit() pass W = np.zeros((np.prod(dims),np.prod(dims))) pixels_pos = np.array([x for x in itertools.product(range(dims[0]), range(dims[1]))]) xpos_ring = np.vstack(pixels_pos[:,0])+ringidx[:,0] ypos_ring = np.vstack(pixels_pos[:,1])+ringidx[:,1] xypos_ring = np.dstack((xpos_ring, ypos_ring)) inside = (xpos_ring>=0) * (ypos_ring>=0) * (xpos_ring<dims[0]) * (ypos_ring<dims[1]) for i,(r,c) in enumerate(itertools.product(range(dims[0]), range(dims[1]))): # xy = np.array([r,c]) + ringidx # inside = np.prod((xy >= 0) * (xy < dims), 1) # index = xy[inside==1] index = xypos_ring[i, inside[i]==1] flatten_index = np.ravel_multi_index(index.T, dims) B = X[flatten_index] tmp0 = B.dot(B.T) tmp0 = tmp0 + 1.e-9*np.eye(index.shape[0]) tmp = np.linalg.inv(tmp0) tmp2 = X[i].dot(B.T) tmp3 = tmp2.dot(tmp) W[i,flatten_index] = tmp3 end = time() # print("Time to update weight ", time() - start) return W, b0
def generate_error_map(seg, truth): error_map = np.bitwise_xor(seg, truth) return error_map
def i4_sobol(dim_num, seed): """ Parameters: Input, integer DIM_NUM, the number of spatial dimensions. DIM_NUM must satisfy 1 <= DIM_NUM <= 40. Input/output, integer SEED, the "seed" for the sequence. This is essentially the index in the sequence of the quasirandom value to be generated. On output, SEED has been set to the appropriate next value, usually simply SEED+1. If SEED is less than 0 on input, it is treated as though it were 0. An input value of 0 requests the first (0-th) element of the sequence. Output, real QUASI(DIM_NUM), the next quasirandom vector. """ global atmost global dim_max global dim_num_save global initialized global lastq global log_max global maxcol global poly global recipd global seed_save global v if 'initialized' not in list(globals().keys()): initialized = 0 dim_num_save = -1 if not initialized or dim_num != dim_num_save: initialized = 1 dim_max = 40 dim_num_save = -1 log_max = 30 seed_save = -1 # Initialize (part of) V. v = np.zeros((dim_max, log_max)) v[0:40, 0] = np.transpose([ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ]) v[2:40, 1] = np.transpose([ 1, 3, 1, 3, 1, 3, 3, 1, 3, 1, 3, 1, 3, 1, 1, 3, 1, 3, 1, 3, 1, 3, 3, 1, 3, 1, 3, 1, 3, 1, 1, 3, 1, 3, 1, 3, 1, 3 ]) v[3:40, 2] = np.transpose([ 7, 5, 1, 3, 3, 7, 5, 5, 7, 7, 1, 3, 3, 7, 5, 1, 1, 5, 3, 3, 1, 7, 5, 1, 3, 3, 7, 5, 1, 1, 5, 7, 7, 5, 1, 3, 3 ]) v[5:40, 3] = np.transpose([ 1, 7, 9, 13, 11, 1, 3, 7, 9, 5, 13, 13, 11, 3, 15, 5, 3, 15, 7, 9, 13, 9, 1, 11, 7, 5, 15, 1, 15, 11, 5, 3, 1, 7, 9 ]) v[7:40, 4] = np.transpose([ 9, 3, 27, 15, 29, 21, 23, 19, 11, 25, 7, 13, 17, 1, 25, 29, 3, 31, 11, 5, 23, 27, 19, 21, 5, 1, 17, 13, 7, 15, 9, 31, 9 ]) v[13:40, 5] = np.transpose([ 37, 33, 7, 5, 11, 39, 63, 27, 17, 15, 23, 29, 3, 21, 13, 31, 25, 9, 49, 33, 19, 29, 11, 19, 27, 15, 25 ]) v[19:40, 6] = np.transpose([ 13, 33, 115, 41, 79, 17, 29, 119, 75, 73, 105, 7, 59, 65, 21, 3, 113, 61, 89, 45, 107 ]) v[37:40, 7] = np.transpose([7, 23, 39]) # Set POLY. poly = [ 1, 3, 7, 11, 13, 19, 25, 37, 59, 47, 61, 55, 41, 67, 97, 91, 109, 103, 115, 131, 193, 137, 145, 143, 241, 157, 185, 167, 229, 171, 213, 191, 253, 203, 211, 239, 247, 285, 369, 299 ] atmost = 2**log_max - 1 # Find the number of bits in ATMOST. maxcol = i4_bit_hi1(atmost) # Initialize row 1 of V. v[0, 0:maxcol] = 1 # Things to do only if the dimension changed. if dim_num != dim_num_save: # Check parameters. if dim_num < 1 or dim_max < dim_num: print('I4_SOBOL - Fatal error!') print(' The spatial dimension DIM_NUM should satisfy:') print(' 1 <= DIM_NUM <= %d' % dim_max) print(' But this input value is DIM_NUM = %d' % dim_num) return dim_num_save = dim_num # Initialize the remaining rows of V. for i in range(2, dim_num + 1): # The bits of the integer POLY(I) gives the form of polynomial I. # Find the degree of polynomial I from binary encoding. j = poly[i - 1] m = 0 j //= 2 while j > 0: j //= 2 m += 1 # Expand this bit pattern to separate components of the logical array INCLUD. j = poly[i - 1] includ = np.zeros(m) for k in range(m, 0, -1): j2 = j // 2 includ[k - 1] = (j != 2 * j2) j = j2 # Calculate the remaining elements of row I as explained # in Bratley and Fox, section 2. for j in range(m + 1, maxcol + 1): newv = v[i - 1, j - m - 1] l = 1 for k in range(1, m + 1): l *= 2 if includ[k - 1]: newv = np.bitwise_xor(int(newv), int(l * v[i - 1, j - k - 1])) v[i - 1, j - 1] = newv # Multiply columns of V by appropriate power of 2. l = 1 for j in range(maxcol - 1, 0, -1): l *= 2 v[0:dim_num, j - 1] = v[0:dim_num, j - 1] * l # RECIPD is 1/(common denominator of the elements in V). recipd = 1.0 / (2 * l) lastq = np.zeros(dim_num) seed = int(np.floor(seed)) if seed < 0: seed = 0 l = 1 if seed == 0: lastq = np.zeros(dim_num) elif seed == seed_save + 1: # Find the position of the right-hand zero in SEED. l = i4_bit_lo0(seed) elif seed <= seed_save: seed_save = 0 lastq = np.zeros(dim_num) for seed_temp in range(int(seed_save), int(seed)): l = i4_bit_lo0(seed_temp) for i in range(1, dim_num + 1): lastq[i - 1] = np.bitwise_xor(int(lastq[i - 1]), int(v[i - 1, l - 1])) l = i4_bit_lo0(seed) elif seed_save + 1 < seed: for seed_temp in range(int(seed_save + 1), int(seed)): l = i4_bit_lo0(seed_temp) for i in range(1, dim_num + 1): lastq[i - 1] = np.bitwise_xor(int(lastq[i - 1]), int(v[i - 1, l - 1])) l = i4_bit_lo0(seed) # Check that the user is not calling too many times! if maxcol < l: print('I4_SOBOL - Fatal error!') print(' Too many calls!') print(' MAXCOL = %d\n' % maxcol) print(' L = %d\n' % l) return # Calculate the new components of QUASI. quasi = np.zeros(dim_num) for i in range(1, dim_num + 1): quasi[i - 1] = lastq[i - 1] * recipd lastq[i - 1] = np.bitwise_xor(int(lastq[i - 1]), int(v[i - 1, l - 1])) seed_save = seed seed += 1 return [quasi, seed]
def sign_extend(value, nbits): value = np.asarray(value) sign_bit = 1 << (nbits - 1) return np.bitwise_xor(value, sign_bit) - sign_bit