def ArtistsRate(JCR, period, istest=False): global constants cols = ["id", "reviews_avgnote", "reviews_all", "playlisters_logged", "starers_logged", "listeners_logged", "downloaders_logged"] if period == 'total': cols += ['days_since_publication'] COLUMNS = JCR.getColumns(cols) #REVIEWS RATE reviews_reduceunder = constants[period]['reviews_reduceunder'] #for album in JCR.iterAlbumJoinArtist() #bestreviewed COLUMNS['reviews_rate'] = computeBayesAvg(COLUMNS["reviews_avgnote"], COLUMNS['reviews_all'], constants[period]['reviews_bayes_const'], reviews_reduceunder) #LIKESRATE likes = COLUMNS["playlisters_logged"] + 1.5*COLUMNS["starers_logged"] #*1.5 to bring the 2 values-set to about the same level avg level ratios = np.true_divide(likes, COLUMNS["listeners_logged"]+1) likes_reduceunder = constants[period]['likes_reduceunder'] COLUMNS['likes_rate'] = computeBayesAvg(ratios, COLUMNS["listeners_logged"]+1, constants[period]['likes_bayes_const'], likes_reduceunder) #FINAL RATE COMBINING BY RANK RATE if period=='total': days = 1 + nomramlizeTo0_1(np.log2( COLUMNS['days_since_publication'] + 2 )) downloaders = np.true_divide(COLUMNS['downloaders_logged'], days) else: downloaders = COLUMNS['downloaders_logged'] COLUMNS['rate'] = getRanks(COLUMNS['likes_rate']) + 1.5*getRanks(COLUMNS['reviews_rate']) + getRanks(COLUMNS['downloaders_logged']) if istest: return COLUMNS else: return {'id': COLUMNS['id'], 'rate': COLUMNS['rate']}
def thresholdColor(img, colattr): (domchan, dommin, first, second) = colattr channels = cv2.split(img)#red, green, blue width, height, cha = img.shape mult = np.empty((width,height)).astype(np.uint8) mult.fill(255) red = channels[2].astype(np.uint8) green = channels[1].astype(np.uint8) blue = channels[0].astype(np.uint8) firsttype = np.zeros(img.shape) secondtype = np.zeros(img.shape) if domchan == "r": zerotype = (red > dommin) firsttype = np.true_divide(red,green)#r/g secondtype = np.true_divide(red,blue)#r/b elif domchan == "g": zerotype = (green > dommin) firsttype = np.true_divide(green,red)#g/r secondtype = np.true_divide(green,blue)#g/b zerotype = zerotype.astype(np.int) firsttype = (firsttype > first).astype(np.int)# & (firsttype < first[1]) secondtype = (secondtype > second).astype(np.int)# & (secondtype < second[1]) combined = cv2.bitwise_and(cv2.bitwise_and(zerotype, secondtype), firsttype) combined = cv2.multiply(combined.astype(np.uint8), mult) return combined
def find_entropy_threshold(img_gray, ranges): hist_item = cv2.calcHist([img_gray],[0],None,[256],[0,256]) # compute a mask of non zero items to avoid # computing a log(0) later on mask = hist_item != 0 entropies = [] for k in ranges: # [:k] is [0, k[, but [k:] is [k, end] range1 = hist_item[:k] range2 = hist_item[k:] sum_r1 = np.sum(range1) sum_r2 = np.sum(range2) # protect against div by zero / log(0) => nan if sum_r1 == 0 or sum_r2 == 0: # insert a 0 to keep the same len as "ranges" entropies.append(0) continue mask1 = mask[:k] mask2 = mask[k:] sum_ln_r1 = np.sum(np.multiply(range1[mask1], np.log(range1[mask1]))) sum_ln_r2 = np.sum(np.multiply(range2[mask2], np.log(range2[mask2]))) entropy = (np.log(sum_r1) + np.log(sum_r2) - np.true_divide(sum_ln_r1, sum_r1) - np.true_divide(sum_ln_r2, sum_r2)) entropies.append(entropy) entropies = np.array(entropies) # get the maximum "k" k = np.argmax(entropies) # return a value in the original range return ranges[k]
def theta(self, S, t, vol, r): if t > self.T: return np.zeros(len(S)) if self.op_type == 'c': return np.subtract( np.true_divide( np.multiply(-vol, np.multiply(S, norm.pdf(self.d1(S, t, vol, r)))), np.multiply(2, np.power( np.subtract(self.T, t), .5))), np.multiply(r, np.multiply(self.K, np.multiply( np.exp( np.multiply(-r, np.subtract(self.T, t))), norm.cdf(self.d2(S, t, vol, r)))))) else: return np.add( np.true_divide( np.multiply(-vol, np.multiply(S, norm.pdf(self.d1(S, t, vol, r)))), np.multiply(2, np.power( np.subtract(self.T, t), .5))), np.multiply(r, np.multiply(self.K, np.multiply( np.exp( np.multiply(-r, np.subtract(self.T, t))), norm.cdf( np.multiply(-1, self.d2(S, t, vol, r)))))))
def kappa_score(y_true, y_pred): """Calculate Cohen's kappa for classification tasks. See https://en.wikipedia.org/wiki/Cohen%27s_kappa Note that this implementation of Cohen's kappa expects binary labels. Args: y_true: Numpy array containing true values. y_pred: Numpy array containing predicted values. Returns: kappa: Numpy array containing kappa for each classification task. Raises: AssertionError: If y_true and y_pred are not the same size, or if class labels are not in [0, 1]. """ assert len(y_true) == len(y_pred), 'Number of examples does not match.' yt = np.asarray(y_true, dtype=int) yp = np.asarray(y_pred, dtype=int) assert np.array_equal(np.unique(yt), [0, 1]), ( 'Class labels must be binary: %s' % np.unique(yt)) observed_agreement = np.true_divide(np.count_nonzero(np.equal(yt, yp)), len(yt)) expected_agreement = np.true_divide( np.count_nonzero(yt == 1) * np.count_nonzero(yp == 1) + np.count_nonzero(yt == 0) * np.count_nonzero(yp == 0), len(yt) ** 2) kappa = np.true_divide(observed_agreement - expected_agreement, 1.0 - expected_agreement) return kappa
def compute_IICR_n_islands(n, M, t, s=True): # This method evaluates the lambda function in a vector # of time values t. # If 's' is True we are in the case when two individuals where # sampled from the same island. If 's' is false, then the two # individuals where sampled from different islands. # Computing constants gamma = np.true_divide(M, n - 1) delta = (1 + n * gamma) ** 2 - 4 * gamma alpha = 0.5 * (1 + n * gamma + np.sqrt(delta)) beta = 0.5 * (1 + n * gamma - np.sqrt(delta)) # Now we evaluate x_vector = t if s: numerator = (1 - beta) * np.exp(-alpha * x_vector) + (alpha - 1) * np.exp(-beta * x_vector) denominator = (alpha - gamma) * np.exp(-alpha * x_vector) + (gamma - beta) * np.exp(-beta * x_vector) else: numerator = beta * np.exp(-alpha * (x_vector)) - alpha * np.exp(-beta * (x_vector)) denominator = gamma * (np.exp(-alpha * (x_vector)) - np.exp(-beta * (x_vector))) lambda_t = np.true_divide(numerator, denominator) return lambda_t
def work(self, input_items, output_items): in0 = input_items[0] out = output_items[0] if len(in0) != self.frame_length: print "pilot receive input buffer size != frame length" if self.scounter==0: #start of a frame Y0 = in0[:self.pilot_length] Y = np.dot(self.prewhiten, Y0.transpose()) Y = Y.transpose() #=====================debugging msg======================== #print "!!!!! received = " #print Y #========================================================== corr = np.true_divide(np.dot(Y.transpose(),self.pilot_seq.conj()),self.pilot_length) #nt x nt #=====================debugging msg======================== #print "Estimated Channel! Estimated channel = " #print corr #========================================================== A = np.dot(corr,corr.transpose().conj()) B = np.true_divide(np.dot(Y.transpose(),Y.conj()),self.pilot_length) Omega = B-A Sigma = np.dot(np.linalg.pinv(Omega)-np.linalg.pinv(B),self.weight) #nt x nt #=====================debugging msg======================== #print "Pilot detected! Sigma =" #print Sigma #========================================================== out[0] = Sigma.reshape(self.nt*self.nt) self.scounter = self.scounter + len(in0) if self.scounter==self.frame_length: self.fcounter += 1 self.scounter = 0 return 1
def _hist_bin_doane(x): """ Doane's histogram bin estimator. Improved version of Sturges' formula which works better for non-normal data. See stats.stackexchange.com/questions/55134/doanes-formula-for-histogram-binning Parameters ---------- x : array_like Input data that is to be histogrammed, trimmed to range. May not be empty. Returns ------- h : An estimate of the optimal bin width for the given data. """ if x.size > 2: sg1 = np.sqrt(6.0 * (x.size - 2) / ((x.size + 1.0) * (x.size + 3))) sigma = np.std(x) if sigma > 0.0: # These three operations add up to # g1 = np.mean(((x - np.mean(x)) / sigma)**3) # but use only one temp array instead of three temp = x - np.mean(x) np.true_divide(temp, sigma, temp) np.power(temp, 3, temp) g1 = np.mean(temp) return x.ptp() / (1.0 + np.log2(x.size) + np.log2(1.0 + np.absolute(g1) / sg1)) return 0.0
def VImetric(ConfusionMatrix): '''Computes Meila's VI metric between two clusterings of the same data, e.g. KK clustering and the detcrit_groundtruth from the confusion matrix''' #nbklust = ConfusionMatrix.shape[0] #nbklustprime = ConfusionMatrix.shape[1] totalspikes = np.sum(ConfusionMatrix) #Pk = np.zeros((nbklust,1)) #Pkprime = np.zeros((nbklustprime, 1)) Pk = np.true_divide(np.sum(ConfusionMatrix, axis = 1),totalspikes) #for k in np.arange(nbklust): # Pk[k] = np.sum(ConfusionMatrix[k,:])/totalspikes Pkprime = np.true_divide(np.sum(ConfusionMatrix, axis = 0),totalspikes) #for kk in np.arange(nbklustprime): # Pkprime[kk] = np.sum(ConfusionMatrix[:,kk])/totalspikes PJoint = np.true_divide(ConfusionMatrix,totalspikes) HC = EntropyH(Pk) HCprime = EntropyH(Pkprime) Inff = MutualInf(Pk,Pkprime,PJoint) #mutual_inf = (infisum, Infie) VI = HC+HCprime - 2*Inff[0]; VImetrics = {'VI':VI, 'Mutual Inf': Inff, 'PJoint' : PJoint, 'PK': Pk, 'PKprime': Pkprime, 'HC': HC, 'HCprime': HCprime} return VImetrics
def __call__(self, values, clip=True, out=None): values = _prepare(values, clip=clip, out=out) np.multiply(values, np.log(self.exp + 1.), out=values) np.exp(values, out=values) np.subtract(values, 1., out=values) np.true_divide(values, self.exp, out=values) return values
def corrected_mean_ndvi(rgb, irview, roi): """Experimental function""" # Obtain the exposure values from the images rgb_exp, _ = read_header(rgb) ir_exp, _ = read_header(irview) # Obtain bands from rgb image red, green, blue = get_image_bands(rgb, roi=roi) # Obtain one of the bands from the IR image ir, _, _ = get_image_bands(irview, roi=roi) red = np.ma.average(red) green = np.ma.average(green) blue = np.ma.average(blue) ir = np.ma.average(ir) # Calculate the 'visible component' of the image pair visible = 0.3 * red + 0.59 * green + 0.11 * blue if rgb_exp != ir_exp: # Correct for exposure ir = np.true_divide(ir, math.sqrt(float(ir_exp))) red = np.true_divide(red, math.sqrt(float(rgb_exp))) visible = np.true_divide(visible, math.sqrt(float(rgb_exp))) # Calculate the NIR component: nir = ir - visible # Now calculate NDVI ndvi = (nir - red) / (nir + red) return ndvi
def this_create_total_dict(): input_dir='/tmp/fvs_fun/fv_deep_funneled/' fv_dict = dict() count=0; for name in os.listdir(input_dir): count=count+1 input_file = os.path.join(input_dir,name) with open(input_file) as f: #w, h = [float(x) for x in f.readline().split()] #array = [[float(x) for x in line.split()] for line in f] s= numpy.genfromtxt(input_file, delimiter=',') word1 = "".join(re.findall("[a-zA-Z]+", name)) word = word1[:-3] if word in fv_dict: fv_dict[word].append([numpy.true_divide(s,numpy.linalg.norm(s,ord=2))]) #fv_dict[word]= numpy.concatenate( (fv_dict[word], numpy.true_divide(s,numpy.linalg.norm(s,ord=2))) ,axis=1) else: fv_dict[word]=[numpy.true_divide(s,numpy.linalg.norm(s,ord=2))] print count if count>10: break; return fv_dict
def yes(features, knowledge, smooth): #calculate probability of yes p_y = math.log(np.true_divide(float(knowledge["nmb_y"]), (knowledge["nmb_y"] + knowledge["nmb_n"]))) p_f_y = p_y #calculate probability of no p_n = math.log(np.true_divide(float(knowledge["nmb_n"]), (knowledge["nmb_y"] + knowledge["nmb_n"]))) p_f_n = p_n for e in features: #calculate p(yes|features) if (not knowledge["frqy"].has_key(e)) and (not knowledge["frqn"].has_key(e)): continue tmp = (0 if (not knowledge["frqy"].has_key(e)) else knowledge["frqy"][e]) + smooth if tmp == 0: p_f_y =float('-inf') else: p_f_y = p_f_y + math.log(float(tmp)/(len(knowledge["frqy"]) * smooth + knowledge["total_f_y"])) #calculate p(no|features) tmp = (0 if (not knowledge["frqn"].has_key(e)) else knowledge["frqn"][e]) + smooth if tmp == 0: p_f_n =float('-inf') else: p_f_n = p_f_n + math.log(float(tmp)/(len(knowledge["frqn"]) * smooth + knowledge["total_f_n"])) if (p_f_y == float('-inf') and p_f_n == float('-inf')): rlt = 'yes' if p_y > p_n else 'n0' else: rlt = 'yes' if p_f_y > p_f_n else 'no' return rlt
def __call__(self, values, clip=None): if clip is None: clip = self.clip if isinstance(values, ma.MaskedArray): if clip: mask = False else: mask = values.mask values = values.filled(self.vmax) else: mask = False # Make sure scalars get broadcast to 1-d if np.isscalar(values): values = np.array([values], dtype=float) else: # copy because of in-place operations after values = np.array(values, copy=True, dtype=float) # Normalize based on vmin and vmax np.subtract(values, self.vmin, out=values) np.true_divide(values, self.vmax - self.vmin, out=values) # Clip to the 0 to 1 range if self.clip: values = np.clip(values, 0., 1., out=values) # Stretch values values = self.stretch(values, out=values, clip=False) # Convert to masked array for matplotlib return ma.array(values, mask=mask)
def hand_to_features(hand, convolution_function, play="river"): # hand: list representation of n hands e.g., ['Kc','Ac','6c','7h'] # returns the output of the filters diction = {"river": 7, "turn": 6, "flop": 5} flush_filter = numpy.ones((1, 13)) vertical_filter = numpy.ones((4, 1)) horizontal_filter = numpy.ones((1, 5)) if play not in diction.keys(): print "play must be river, turn or flop" quit() hand_matrix = string_to_vector(hand) column_sums = numpy.sum(hand_matrix, axis=0) column_sums[column_sums > 1] = 1 column_sums = column_sums.reshape((1, 14)) stripped_hand_matrix = numpy.delete(hand_matrix, 0, 1) straight_features = convolution_function(column_sums, horizontal_filter) straight_features_normalized = numpy.true_divide(straight_features, 5) two_three_four_features = convolution_function(stripped_hand_matrix, vertical_filter) two_three_four_features_normalized = numpy.true_divide(two_three_four_features, 4) flush_features = convolution_function(stripped_hand_matrix, flush_filter).T flush_features_normalized = numpy.true_divide(flush_features, diction[play]) return numpy.concatenate( (straight_features_normalized, flush_features_normalized, two_three_four_features_normalized), axis=1 )
def thresholdColor(img): NTIME = time.time() channels = cv2.split(img) red = channels[2].astype(np.uint8) green = channels[1].astype(np.uint8) blue = channels[0].astype(np.uint8) r_g = np.true_divide(red,green) r_b = np.true_divide(red,blue) g_r = np.true_divide(green,red) g_b = np.true_divide(green,blue) combs = [] for col in colors: (dom,minv,first,second)=color_dims[col] if dom == "r": comb = ((red > minv) & (r_g > first) & (r_b > second)).astype(np.uint8) elif dom == "g": comb = ((green > minv) & (g_r > first) & (g_b > second)).astype(np.uint8) combs.append(comb) BTIME = time.time() rospy.loginfo("IN") rospy.loginfo(BTIME-NTIME) return combs
def search(): print('*' * 80) print('Searching: ') boundary_of_category = dict() max_p_category = np.nanmax(jll, axis=0) # max probability in each category min_p_category = np.nanmin(jll, axis=0) # min probability in each category for categoryid in categoryid_set: print('\t%s\tSearching in %s' % (datetime.now(), categoryid)) idx = np.where(clf.classes_ == categoryid) tp = (y_true == categoryid) & (y_pred == categoryid) fp = (y_true != categoryid) & (y_pred == categoryid) fn = (y_true == categoryid) & (y_pred != categoryid) proba_tp = np.sort(max_proba[tp]) proba_fp = np.sort(max_proba[fp]) proba_fn = np.sort(max_proba[fn]) threshold = np.linspace(min_p_category[idx], max_p_category[idx], 100) tp_num = proba_tp.shape[0] - np.searchsorted(proba_tp, threshold) fp_num = proba_fp.shape[0] - np.searchsorted(proba_fp, threshold) fn_num = proba_fn.shape[0] + np.searchsorted(proba_tp, threshold) accuracy = np.true_divide(tp_num, (tp_num + fp_num)) recall = np.true_divide(tp_num, (tp_num + fn_num)) f1 = np.true_divide(2 * accuracy * recall, (accuracy + recall)) idx_max_f1 = np.nanargmax(f1) boundary_of_category[categoryid] = threshold[idx_max_f1] y_pred[(max_proba < threshold[idx_max_f1]) & (y_pred == categoryid)] = None if args.persistence: with codecs.open('boundary.json', encoding='utf-8', mode='w') as f: json.dump(obj=boundary_of_category, fp=f, ensure_ascii=False, encoding='utf-8', indent=4, separators=(',', ': '))
def persp(winsize, nf, dim=3): """ Assumptions: 2D: Device coordinates (eye coordinates) have (0,0) at the top left corner. z is just normalized. This is because in 2D I like to pretend the screen is a big bitmap. 3D: I am not going to bother explaining what is going on here. It is relatively simple but this was incredibly painful to implement on top of poorly written code. It is dull mathematics. One thing to note: If you are using your own geometry shader you MUST do some sort of clipping plane check on the output, because if you don't you will get weird artifacts around the near plane because of the asymptotic behavior. :param winsize: width, height :return: device coordinates normalized to the x,y axis """ w = np.true_divide(2, winsize[0]) h = np.true_divide(2, winsize[1]) n, f = nf if dim == 3: return np.array([[w*n, 0, 0, 0], [0, h*n, 0, 0], [0, 0, -(n+f)/float(f-n), -2*n*f/float(f-n)], [0, 0, -1, 0]], dtype="float32") else: return [[2*w, 0, -1], [0, -2*h, 1], [0, 0, 1]]
def plot_coding_forward(data, node): speeds,forwarded,coded = nodes_forwarded_coded(data, "coding", node) speeds = len(data['coding']['slaves']) * numpy.array(speeds) # Normalize total = numpy.add.reduce((forwarded, numpy.array(coded)*2)) forwarded_norm = numpy.true_divide(forwarded, total) coded_norm = numpy.true_divide(coded, total) total_norm = forwarded_norm + coded_norm # Create and setup a new figure fig = pylab.figure() ax = fig.add_subplot(111) ax.set_xlabel("Total Offered Load [kbit/s]") ax.set_ylabel("Packets Ratio") ax.set_title("Packets Forwarded/Coded") ax.grid(True) # Plot data ax.plot(speeds, forwarded_norm, linewidth=2, color=c['chameleon2']) ax.plot(speeds, coded_norm, linewidth=2, color=c['skyblue2']) ax.plot(speeds, total_norm, linewidth=2, color=c['scarletred2']) ax.legend(("Forwarded", "Coded", "Total"), loc='upper left', shadow=True) # Add figure to list of created figures figures["{}_fw_coded".format(node)] = fig
def handle_data(context, data): # get data d = get_data(data,context.stocks) if d == None: return prices = d[0] volumes = d[1] if not context.init: rebalance_portfolio(context, data, context.b_t) context.init = True return m = context.m #Lenth of b vector x_tilde = np.zeros(m) #x_tilde vector b_optimal = np.zeros(m) #b_vector #For each window, we calculate the optimal b_norm for k, w in enumerate(W_L): #Caluclate predicted x for that window size for i, stock in enumerate(context.stocks): #Predicted ratio of price change from t to t+1 if vwap == True: vol_weights = np.true_divide(volumes[W_MAX-w:,i],np.sum(volumes[W_MAX-w:,i])) vwa_price = np.average(prices[W_MAX-w:,i],None,vol_weights) x_tilde[i] = vwa_price/prices[W_MAX-1,i] else: x_tilde[i] = np.mean(prices[W_MAX-w:,i])/prices[W_MAX-1,i] #Update the b_w matrix context.b_w[:,k] = find_b_norm(context,x_tilde,context.b_w[:,k]) length_p = len(prices[:,0]) #Length of price vector p_t = prices[length_p-1,:] #Price vector today p_y = prices[length_p-2,:] #Price vector yesterday #Ratio of price change (1 x m) vector from t-1 to t x_t = np.true_divide(p_t,p_y) #w is length of W_L #Daily returns (1 x w) vector s_d = np.dot(x_t,context.b_w) #Cumulative returns (1 x w) vector context.s_w = np.multiply(context.s_w,s_d) #Calculate return_weights (1 x w) vector return_weights = np.true_divide(context.s_w[:],np.sum(context.s_w)) #Weight according to cum. returns #Calculate b_{t+1} (n x 1) vector from (m x w) * (w x 1) b_optimal = np.dot(context.b_w,np.transpose(return_weights)) #Calculate the weighted portfolio #SJUAR #log.info(b_optimal) rebalance_portfolio(context, data, b_optimal) # update portfolio context.b_t = b_optimal
def run_cv(data,label,weight): # run cross validation on training set to get stats # configure weights wp = len(np.where(label == 1)[0]) wd = len(np.where(label == 0)[0]) print 'Scale pos. weight: {}'.format(3.*np.true_divide(wd,wp)) # setup parameters for xgboost param = {} # use logistic regression loss, use raw prediction before logistic transformation # since we only need the rank param['objective'] = 'binary:logistic' # scale weight of positive examples param['scale_pos_weight'] = 3.*np.true_divide(wd,wp) param['eta'] = 0.05 param['eval_metric'] = 'error' param['silent'] = 1 param['nthread'] = 6 param['min_child_weight'] = 2 param['max_depth'] = 10 param['gamma'] = 0.0 param['colsample_bytree'] = 0.8 param['subsample'] = 0.9 param['reg_alpha'] = 1e-5 # boost 25 tres num_round = 300 test_error = [] test_falsepos = [] test_falseneg = [] scores = np.zeros((2,len(label))) # get folds skf = StratifiedKFold(label, 10, shuffle=True) for i, (train, test) in enumerate(skf): #print train, test Xtrain = data[train] ytrain = label[train] wtrain = label[train] Xtest = data[test] ytest = label[test] wtest = label[test] # make dmatrices from xgboost dtrain = xgb.DMatrix( Xtrain, label=ytrain ) dtest = xgb.DMatrix( Xtest ) #watchlist = [ (dtrain,'train') ] bst = xgb.train(param, dtrain, num_round) ypred = bst.predict(dtest) fold_error,fold_falsepos,fold_falseneg = compute_stats(ytest,ypred) test_error.append(fold_error) test_falsepos.append(fold_falsepos) test_falseneg.append(fold_falseneg) scores[0,test] = ytest scores[1,test] = ypred return test_error,test_falsepos,test_falseneg,scores
def thresholdColor(img): width,height,cha = img.shape mult = np.empty((width,height)).astype(np.uint8) mult.fill(255) channels = cv2.split(img) red = channels[2].astype(np.uint8) green = channels[1].astype(np.uint8) blue = channels[0].astype(np.uint8) r_g = np.true_divide(red,green) r_b = np.true_divide(red,blue) g_r = np.true_divide(green,red) g_b = np.true_divide(green,blue) combs = [] for col in colors: (dom,minv,first,second)=col if dom == "r": zerotype = red > minv firstype = r_g secondtype = r_b elif dom == "g": zerotype = green > minv firsttype = g_r secondtype = g_b zerotype = zerotype.astype(np.int) firsttype = (firsttype > first).astype(np.int) secondtype = (secondtype > second).astype(np.int) combined = cv2.bitwise_and(cv2.bitwise_and(zerotype,firsttype),secondtype) combined = cv2.multiply(combined.astype(np.uint8),mult) combs.append(combined) return combs
def get_clipped_resized_masks(boxes, input_masks): target_size = cfg.MASK_SIZE num_boxes = boxes.shape[0] num_channels = input_masks.shape[1] #masks = cymask.clip_resize_mask(sp, reg2sp, boxes, target_size) masks = np.zeros((num_boxes, num_channels, target_size, target_size)) totaltime=0. yv = np.arange(target_size).reshape(-1,1) xv = np.arange(target_size).reshape(1,-1) yv = np.true_divide(yv, float(target_size)-1.) xv = np.true_divide(xv, float(target_size)-1.) for k in range(num_boxes): box = boxes[k,:] xmin = box[0] ymin = box[1] w = box[2]-box[0] h = box[3]-box[1] xv1 = np.round(xmin + xv*w).astype(int) yv1 = np.round(ymin + yv*h).astype(int) for l in range(num_channels): M = input_masks[k,l,yv1, xv1] masks[k,l,:,:]=M #S = sp[box[1]:box[3]+1,box[0]:box[2]+1] #M = reg2sp[:,k] #M = M[S-1] #start = time.clock() #masks[k,:,:] = scipymisc.imresize(M, (target_size, target_size), 'nearest') #stop = time.clock() #totaltime+=stop-start return masks
def __init__(self, n=10, M=0.1): T2_StSI.__init__(self, n, M) [A, B, a, alpha, beta, E, AplusB, AminusB] = \ T2_StSI.compute_constants(self, n, M) [self.a, self.alpha, self.beta] = [a, alpha, beta] self.gamma = np.true_divide(M, n-1) self.c = np.true_divide(self.gamma, beta-alpha)
def this_create_total_dict(fromPercent, toPercent): input_dir="/tmp/fvs_fun/fv_deep_funneled/" total = len(os.listdir(input_dir)) start_index = int(fromPercent*total) end_index = int(toPercent*total) fv_dict = dict() count=0; for name in os.listdir(input_dir)[start_index:end_index]: count=count+1 input_file = os.path.join(input_dir,name) with open(input_file) as f: #w, h = [float(x) for x in f.readline().split()] #array = [[float(x) for x in line.split()] for line in f] s= numpy.genfromtxt(input_file, delimiter=',') word1 = "".join(re.findall("[a-zA-Z]+", name)) word = word1[:-3] if word in fv_dict: fv_dict[word].append([numpy.true_divide(s,numpy.linalg.norm(s,ord=2))]) #fv_dict[word]= numpy.concatenate( (fv_dict[word], numpy.true_divide(s,numpy.linalg.norm(s,ord=2))) ,axis=1) else: fv_dict[word]=[numpy.true_divide(s,numpy.linalg.norm(s,ord=2))] print count #if len(fv_dict.keys())>50: # break; return fv_dict
def normalize(ndarray_list): """ Normalize each ndarray and scale all > All ndarray must have same dimensions! Parameters : - ndarray_list : ndarray list of images Returns : - ndarray : ndarray in wich each pixel match to pixels normalize """ liste = [] lenght = len(ndarray_list) print "Taille de la liste d'image à traiter : "+ str(lenght) # 1) Normalize each flat field, i.e. divide it by its mean entry value for i in range(lenght): print "Normalize 1/2 : " + str(i+1) + "/" + str(lenght) mean = np.mean(ndarray_list[i]) # Find the mean of the ndarray print "Moyenne de la " + str(i+1) + " image : " + str(mean) liste.append(mean) ndarray_list[i] = np.true_divide(ndarray_list[i], mean) # Divide ndarray by its mean, normalizing it # 2) Scale all of the fields’ means so that their individual averages are equal to one another meanofmean = sum(liste) / len(liste) # Find the mean of the total set of ndarray_list print "Moyenne global de toutes les images : " + str(meanofmean) for i in range(lenght): print "Normalize 2/2 with : "+ str(i+1) + "/" + str(lenght) ndarray_list[i] = np.multiply(ndarray_list[i], np.true_divide(meanofmean, np.mean(ndarray_list[i]))) # Divide return ndarray_list
def compare_cdf_g(model, n_obs=10000, case='T2s', t_vector=np.arange(0, 100, 0.1), path2ms='./'): # Do a graphical comparison by plotting the theoretical cdf and the # empirical pdf T_list_ms = np.true_divide(model.T_list,2) M_list = model.M_list if case == 'T2s': cmd = create_ms_command_T2(n_obs, model.n, T_list_ms, M_list, 'same') theor_cdf = model.cdf_T2s elif case == 'T2d': cmd = create_ms_command_T2(n_obs, model.n, T_list_ms, M_list, 'disctint') theor_cdf = model.cdf_T2d else: return 1 obs = simulate_T2_ms(cmd, path2ms) obs = np.array(obs) * 2 delta = t_vector[-1] - t_vector[-2] bins = t_vector f_obs = np.histogram(obs, bins=bins)[0] cum_f_obs = [0] + list(f_obs.cumsum()) F_obs = np.true_divide(np.array(cum_f_obs), n_obs) F_theory = [theor_cdf(t) for t in bins] # Now we plot fig = plt.figure() ax = fig.add_subplot(111) ax.plot(bins, F_obs, '-r', label = 'Empirical cdf') ax.plot(bins, F_theory, '-b', label = 'Theoretical cdf') plt.legend() plt.show()
def make_bdt(data,label,weight): # create and save tree model # configure weights wp = len(np.where(label == 1)[0]) wd = len(np.where(label == 0)[0]) print 'Scale pos. weight: {}'.format(3.*np.true_divide(wd,wp)) # setup parameters for xgboost param = {} # use logistic regression loss, use raw prediction before logistic transformation # since we only need the rank param['objective'] = 'binary:logistic' # scale weight of positive examples param['scale_pos_weight'] = 3.*np.true_divide(wd,wp) param['eta'] = 0.05 param['eval_metric'] = 'error' param['silent'] = 1 param['nthread'] = 6 param['min_child_weight'] = 2 param['max_depth'] = 10 param['gamma'] = 0.0 param['colsample_bytree'] = 0.8 param['subsample'] = 0.9 param['reg_alpha'] = 1e-5 # boost 25 tres num_round = 300 # make dmatrices from xgboost dtrain = xgb.DMatrix( data, label=label ) bst = xgb.train(plst, dtrain, num_round) return bst
def sigmoid_inverse(z): # z = 0.998*z + 0.001 assert(np.max(z) <= 1.0 and np.min(z) >= 0.0) z = 0.998*z + 0.001 return np.log(np.true_divide( 1.0, (np.true_divide(1.0, z) - 1) ))
def general_work(self, input_items, output_items): in0 = input_items[0] flags = input_items[1] out = output_items[0] if len(in0) !=len(flags): print "TX:input length buffer DOESN'T EQUAL" len_in = min(len(in0), len(flags)) if self.state == 0: for i in range(len_in): if flags[i] == 1: print "TX: FLAG FOUND!!!", i self.state =1 self.consume(0,i+1) self.consume(1,i+1) return 0 if self.state == 1: Y = in0[-self.pilot_length:0] corr = np.true_divide(np.dot(Y.transpose(),self.pilot_seq.conj()),self.pilot_length) A = np.dot(corr,corr.transpose().conj()) B = np.true_divide(np.dot(Y.transpose(),Y.conj()),self.pilot_length) Omega_hat = B-A Omega = Omega_hat - self.NHAT + np.identity(self.nt) Sigma = np.dot(np.linalg.pinv(Omega)-np.linalg.pinv(B),self.weight) #nt x nt #=====================debugging msg======================== print "Pilot detected! TX sync Sigma =" print Sigma out[:]=Sigma.reshape(self.nt*self.nt) self.consume(0,len_in) self.consume(1,len_in) return 1
def div0( a, b ): """ ignore / 0, div0( [-1, 0, 1], 0 ) -> [0, 0, 0] """ with np.errstate(divide='ignore', invalid='ignore'): c = np.true_divide( a, b ) c[ ~ np.isfinite( c )] = 0 # -inf inf NaN return c
def run_analysis(opts): #% Set up paths: traceid = '%s_s2p'%(opts. traceid) #% Set up paths: acquisition_dir = os.path.join(opts.rootdir, opts.animalid, opts.session, opts.acquisition) traceid_dir = os.path.join(acquisition_dir, opts.retino_run, 'traces',traceid) file_dir = os.path.join(traceid_dir,'retino_analysis','files') run_dir = traceid_dir.split('/traces')[0] trace_arrays_dir = os.path.join(traceid_dir,'files') #Output paths fig_base_dir = os.path.join(traceid_dir,'retino_analysis','figures') if not os.path.exists(fig_base_dir): os.makedirs(fig_base_dir) file_out_dir = os.path.join(traceid_dir,'retino_analysis','files') if not os.path.exists(file_out_dir): os.makedirs(file_out_dir) # Get associated RUN info: runmeta_path = os.path.join(run_dir, '%s.json' % opts.retino_run) with open(runmeta_path, 'r') as r: runinfo = json.load(r) nslices = len(runinfo['slices']) nchannels = runinfo['nchannels'] nvolumes = runinfo['nvolumes'] ntiffs = runinfo['ntiffs'] frame_rate = runinfo['frame_rate'] #-----Get info from paradigm file para_file_dir = os.path.join(run_dir,'paradigm','files') if not os.path.exists(para_file_dir): os.makedirs(para_file_dir) para_files = [f for f in os.listdir(para_file_dir) if f.endswith('.json')]#assuming a single file for all tiffs in run if len(para_files) == 0: # Paradigm info not extracted yet: raw_para_files = [f for f in glob.glob(os.path.join(run_dir, 'raw*', 'paradigm_files', '*.mwk')) if not f.startswith('.')] print run_dir assert len(raw_para_files) == 1, "No raw .mwk file found, and no processed .mwk file found. Aborting!" raw_para_file = raw_para_files[0] print "Extracting .mwk trials: %s" % raw_para_file fn_base = os.path.split(raw_para_file)[1][:-4] trials = mw.extract_trials(raw_para_file, retinobar=True, trigger_varname='frame_trigger', verbose=True) para_fpath = mw.save_trials(trials, para_file_dir, fn_base) para_file = os.path.split(para_fpath)[-1] else: assert len(para_files) == 1, "Unable to find unique .mwk file..." para_file = para_files[0] print 'Getting paradigm file info from %s'%(os.path.join(para_file_dir, para_file)) with open(os.path.join(para_file_dir, para_file), 'r') as r: parainfo = json.load(r) #get masks curr_slice = 'Slice01'#hard-coding planar data for now masks_fn = os.path.join(file_out_dir,'masks.hdf5') mask_file = h5py.File(masks_fn, 'r') iscell = np.array(mask_file[curr_slice]['iscell']) mask_array = np.array(mask_file[curr_slice]['mask_array']) mask_file.close() #fid = 1 for fid in range(1,ntiffs+1): trace_file = [f for f in os.listdir(trace_arrays_dir) if 'File%03d'%(fid) in f and f.endswith('hdf5')][0] trace_fn = os.path.join(trace_arrays_dir,trace_file) print(trace_fn) rawfile = h5py.File(trace_fn, 'r') frametimes = np.array(rawfile[curr_slice]['frames_tsec']) roi_trace = np.transpose(rawfile[curr_slice]['traces']['pixel_value']['cell']) rawfile.close() stimulus = parainfo[str(fid)]['stimuli']['stimulus'] stimfreq = parainfo[str(fid)]['stimuli']['scale'] #make figure directory for stimulus type fig_out_dir = os.path.join(fig_base_dir, stimulus) if not os.path.exists(fig_out_dir): os.makedirs(fig_out_dir) #Get fft print('Getting fft....') fourier_data = np.fft.fft(roi_trace) nrois,nframes = roi_trace.shape #Get magnitude and phase data print('Analyzing phase and magnitude....') mag_data=abs(fourier_data) phase_data=np.angle(fourier_data) #label frequency bins freqs = np.fft.fftfreq(nframes, float(1/frame_rate)) idx = np.argsort(freqs) freqs=freqs[idx] #sort magnitude and phase data mag_data=mag_data[:,idx] phase_data=phase_data[:,idx] #excluding DC offset from data freqs=freqs[np.round(nframes/2)+1:] mag_data=mag_data[:,np.round(nframes/2)+1:] phase_data=phase_data[:,np.round(nframes/2)+1:] freq_idx=np.argmin(np.absolute(freqs-stimfreq))#find out index of stimulation freq top_freq_idx=np.where(freqs>1)[0][0]#find out index of 1Hz, to cut-off zoomed out plot max_mod_idx=np.argmax(mag_data[:,freq_idx],0)#best pixel index #unpack values from frequency analysis mag_array = mag_data[:,freq_idx] phase_array = phase_data[:,freq_idx] #get magnitude ratio tmp=np.copy(mag_data) np.delete(tmp,freq_idx,1) nontarget_mag_array=np.sum(tmp,1) mag_ratio_array=mag_array/nontarget_mag_array #bootstrap to get null-distribution for mag ratio nreps = 1000 zscore_bootstrap = np.empty((nrois,)) ratio_array = np.empty((nreps,)) #ridx = 76 for ridx in range(nrois): for rep in range(nreps): roi_mag = mag_data[ridx,:] shuffle_mag = np.random.permutation(roi_mag) mag_value = shuffle_mag[freq_idx] tmp = np.copy(shuffle_mag) np.delete(tmp,freq_idx,0) nontarget_mag_value=np.sum(tmp,0) ratio_array[rep] = mag_value/nontarget_mag_value zscore_bootstrap[ridx] = (mag_ratio_array[ridx] - np.mean(ratio_array))/np.std(ratio_array) #do regression, get some info from fit t=frametimes*(2*np.pi)*stimfreq phi=np.expand_dims(phase_array,1) varexp_array, beta_array, signal_fit = do_regression(t,phi,roi_trace,roi_trace.shape[0],roi_trace.shape[1]) print('Saving data to file') file_grp = h5py.File(os.path.join(file_out_dir,'File%03d_retino_data.hdf5'%(fid)), 'w') #file_grp.attrs['frame_rate'] = frame_rate file_grp.attrs['stimfreq'] = stimfreq #save data values to structure if 'mag_array' not in file_grp.keys(): magset = file_grp.create_dataset('/'.join([curr_slice,'mag_array']),mag_array.shape, mag_array.dtype) magset[...] = mag_array if 'phase_array' not in file_grp.keys(): phaseset = file_grp.create_dataset('/'.join([curr_slice,'phase_array']),phase_array.shape, phase_array.dtype) phaseset[...] = phase_array if 'mag_ratio_array' not in file_grp.keys(): ratioset = file_grp.create_dataset('/'.join([curr_slice,'mag_ratio_array']),mag_ratio_array.shape, mag_ratio_array.dtype) ratioset[...] = mag_ratio_array if 'ratio_bootstrap_zscore' not in file_grp.keys(): bstrapset = file_grp.create_dataset('/'.join([curr_slice,'ratio_bootstrap_zscore']),zscore_bootstrap.shape, zscore_bootstrap.dtype) bstrapset[...] = zscore_bootstrap if 'beta_array' not in file_grp.keys(): betaset = file_grp.create_dataset('/'.join([curr_slice,'beta_array']),beta_array.shape, beta_array.dtype) betaset[...] = beta_array if 'var_exp_array' not in file_grp.keys(): varset = file_grp.create_dataset('/'.join([curr_slice,'var_exp_array']),varexp_array.shape, varexp_array.dtype) varset[...] = varexp_array # Add fit signal to retino output: if 'signal_fit' not in file_grp.keys(): fitset = file_grp.create_dataset('/'.join([curr_slice,'signal_fit']), signal_fit.shape, signal_fit.dtype) fitset[...] = signal_fit if 'masks' not in file_grp.keys(): mset = file_grp.create_dataset('/'.join([curr_slice,'masks']),mask_array.shape, mask_array.dtype) mset[...] = mask_array file_grp.close() #VISUALIZE!!! print('Visualizing results') print('Output folder: %s'%(fig_out_dir)) #visualize pixel-based result #make figure directory for stimulus type fig_dir = os.path.join(fig_out_dir, 'File%03d_%s' % (fid, curr_slice),'spectrum') if not os.path.exists(fig_dir): os.makedirs(fig_dir) for midx in range(nrois): fig_name = 'full_spectrum_mask%04d.png' %(midx) fig=plt.figure() plt.plot(freqs,mag_data[midx,:]) plt.xlabel('Frequency (Hz)',fontsize=16) plt.ylabel('Magnitude',fontsize=16) axes = plt.gca() ymin, ymax = axes.get_ylim() plt.axvline(x=freqs[freq_idx], linewidth=1, color='r') plt.savefig(os.path.join(fig_dir,fig_name)) plt.close() for midx in range(nrois): fig_name = 'zoom_spectrum_mask%04d.png' %(midx) fig=plt.figure() plt.plot(freqs[0:top_freq_idx],mag_data[midx,0:top_freq_idx]) plt.xlabel('Frequency (Hz)',fontsize=16) plt.ylabel('Magnitude',fontsize=16) axes = plt.gca() ymin, ymax = axes.get_ylim() plt.axvline(x=freqs[freq_idx], linewidth=1, color='r') plt.savefig(os.path.join(fig_dir,fig_name)) plt.close() fig_dir = os.path.join(fig_out_dir, 'File%03d_%s' % (fid, curr_slice),'timecourse') if not os.path.exists(fig_dir): os.makedirs(fig_dir) stimperiod_t=np.true_divide(1,stimfreq) stimperiod_frames=stimperiod_t*frame_rate periodstartframes=np.round(np.arange(0,len(frametimes),stimperiod_frames))[:] periodstartframes = periodstartframes.astype('int') for midx in range(nrois): fig_name = 'timecourse_fit_mask%04d.png' %(midx) fig=plt.figure() plt.plot(frametimes,roi_trace[midx,:],'b') plt.plot(frametimes,signal_fit[midx,:],'r') plt.xlabel('Time (s)',fontsize=16) plt.ylabel('Pixel Value',fontsize=16) axes = plt.gca() ymin, ymax = axes.get_ylim() for f in periodstartframes: plt.axvline(x=frametimes[f], linewidth=1, color='k') axes.set_xlim([frametimes[0],frametimes[-1]]) plt.savefig(os.path.join(fig_dir,fig_name)) plt.close() # #Read in average image (for viuslization) masks_fn = os.path.join(file_out_dir,'masks.hdf5') mask_file = h5py.File(masks_fn, 'r') im0 = np.array(mask_file[curr_slice]['meanImg']) mask_file.close() szx,szy = im0.shape im1 = np.uint8(np.true_divide(im0,np.max(im0))*255) im2 = np.dstack((im1,im1,im1)) #set phase map range for visualization phase_array_disp=np.copy(phase_array) phase_array_disp[phase_array<0]=-phase_array[phase_array<0] phase_array_disp[phase_array>0]=(2*np.pi)-phase_array[phase_array>0] #mark rois magratio_roi = np.empty((szy,szx)) magratio_roi[:] = np.NAN mag_roi = np.copy(magratio_roi) varexp_roi = np.copy(magratio_roi) phase_roi = np.copy(magratio_roi) for midx in range(nrois): if iscell[midx]: maskpix = np.where(np.squeeze(mask_array[midx,:,:])) #print(len(maskpix)) magratio_roi[maskpix]=mag_ratio_array[midx] mag_roi[maskpix]=mag_array[midx] varexp_roi[maskpix]=varexp_array[midx] phase_roi[maskpix]=phase_array_disp[midx] fig_dir = fig_out_dir data_str = 'File%03d_%s'%(fid,curr_slice) sns.set_style("darkgrid", {'axes.grid' : True}) fig_name = 'phase_mag_ratio_joint_%s.png' % data_str p = sns.jointplot(phase_array_disp,mag_ratio_array,xlim = (0,2*np.pi)) p.set_axis_labels(xlabel='Phase', ylabel='Mag Ratio',fontsize = 15) p.savefig(os.path.join(fig_dir,fig_name)) plt.close() sns.set_style("whitegrid", {'axes.grid' : False}) fig_name = 'phase_info_%s.png' % data_str fig=plt.figure() plt.imshow(im2,'gray') plt.imshow(phase_roi,'nipy_spectral',alpha = 0.5,vmin=0,vmax=2*np.pi) plt.colorbar() plt.savefig(os.path.join(fig_dir,fig_name)) plt.close() fig_name = 'mag_info_%s.png' % data_str fig=plt.figure() plt.imshow(im2,'gray') plt.imshow(mag_roi, alpha = 0.5) plt.colorbar() plt.savefig(os.path.join(fig_dir,fig_name)) plt.close() fig_name = 'mag_ratio_info_%s.png' % data_str fig=plt.figure() plt.imshow(im2,'gray') plt.imshow(magratio_roi, alpha = 0.5) plt.colorbar() plt.savefig(os.path.join(fig_dir,fig_name)) plt.close() fig_name = 'varexp_info_%s.png' % data_str fig=plt.figure() plt.imshow(im2,'gray') plt.imshow(varexp_roi, alpha = 0.5) plt.colorbar() plt.savefig(os.path.join(fig_dir,fig_name)) plt.close() fig_name = 'phase_nice_%s.png' % data_str #curr_file #(tiff_fn[:-4]) dpi = 80 szY,szX = im1.shape # What size does the figure need to be in inches to fit the image? figsize = szX / float(dpi), szY / float(dpi) # Create a figure of the right size with one axes that takes up the full figure fig = plt.figure(figsize=figsize) ax = fig.add_axes([0, 0, 1, 1]) # Hide spines, ticks, etc. ax.axis('off') ax.imshow(im2,'gray') ax.imshow(phase_roi,'nipy_spectral',alpha = 0.5,vmin=0,vmax=2*np.pi) fig.savefig(os.path.join(fig_dir,fig_name), dpi=dpi, transparent=True) plt.close() std_thresh = .008 phase_roi_thresh = np.copy(phase_roi) phase_roi_thresh[magratio_roi<std_thresh] = np.nan fig_name = 'phase_nice_thresh_%s.png' % data_str #curr_file #(tiff_fn[:-4]) dpi = 80 szY,szX = im1.shape # What size does the figure need to be in inches to fit the image? figsize = szX / float(dpi), szY / float(dpi) # Create a figure of the right size with one axes that takes up the full figure fig = plt.figure(figsize=figsize) ax = fig.add_axes([0, 0, 1, 1]) # Hide spines, ticks, etc. ax.axis('off') ax.imshow(im2,'gray') ax.imshow(phase_roi_thresh,'nipy_spectral',alpha = 0.5,vmin=0,vmax=2*np.pi) fig.savefig(os.path.join(fig_dir,fig_name), dpi=dpi, transparent=True) plt.close() #correct orientation phase_roi_corr = np.transpose(phase_roi) phase_roi_corr = np.flip(phase_roi_corr,0) phase_roi_corr = np.flip(phase_roi_corr,1) im1_corr = np.transpose(im1) im1_corr = np.flip(im1_corr,0) im1_corr = np.flip(im1_corr,1) im2_corr = np.dstack((im1_corr,im1_corr,im1_corr)) fig_name = 'phase_nice_corrected_%s.png' % data_str #curr_file #(tiff_fn[:-4]) dpi = 80 szY,szX = im1.shape # What size does the figure need to be in inches to fit the image? figsize = szX / float(dpi), szY / float(dpi) # Create a figure of the right size with one axes that takes up the full figure fig = plt.figure(figsize=figsize) ax = fig.add_axes([0, 0, 1, 1]) # Hide spines, ticks, etc. ax.axis('off') ax.imshow(im2_corr,'gray') ax.imshow(phase_roi_corr,'nipy_spectral',alpha = 0.5,vmin=0,vmax=2*np.pi) fig.savefig(os.path.join(fig_dir,fig_name), dpi=dpi, transparent=True) plt.close() phase_roi_corr_thresh = np.transpose(phase_roi_thresh) phase_roi_corr_thresh = np.flip(phase_roi_corr_thresh,0) phase_roi_corr_thresh = np.flip(phase_roi_corr_thresh,1) fig_name = 'phase_nice_corrected_thresh_%s.png' % data_str #curr_file #(tiff_fn[:-4]) dpi = 80 szY,szX = im1.shape # What size does the figure need to be in inches to fit the image? figsize = szX / float(dpi), szY / float(dpi) # Create a figure of the right size with one axes that takes up the full figure fig = plt.figure(figsize=figsize) ax = fig.add_axes([0, 0, 1, 1]) # Hide spines, ticks, etc. ax.axis('off') ax.imshow(im2_corr,'gray') ax.imshow(phase_roi_corr_thresh,'nipy_spectral',alpha = 0.5,vmin=0,vmax=2*np.pi) fig.savefig(os.path.join(fig_dir,fig_name), dpi=dpi, transparent=True) plt.close()
def nnff(nn, x, y): #NNFF performs a feed forward pass # nn = nnff(nn,x,y) returns a neural network structure with updated layer activations, # error and loss (nn.a,nn.e,nn.L) n = nn.N m = x.shape[0] temp = np.ones([m, x.shape[1] + 1], dtype=np.float64) temp[:, 1:] = x #temp[:,1:] = np.copy(x) # Allocate space for activation, error and loss # Since matlab allows structs to have cells/arrays etc on the fly, # This is not possible with python. We declare members and then use them nn.A[0] = np.copy(temp) # Feedforward pass for i in range(1, n - 1): if nn.ActivationFunction == 'sigm': #Caluclate the unit's outputs (including the units bias term) r = np.dot(nn.A[i - 1], nn.W[i - 1].T) nn.A[i] = sigm.sigm(r) elif nn.ActivationFunction == 'tanh_opt': r = np.dot(nn.A[i - 1], nn.W[i - 1].T) nn.A[i] = tanh_opt.tanh_opt(r) # Droput if nn.DropoutFraction > 0: if nn.Testing == True: nn.A[i] = np.multiply(nn.A[i], (1 - nn.DropoutFraction)) else: nn.DropOutMask[i] = np.random.uniform(1, 0, nn.A[i].shape) # Get the indices and convert them to floats indices = [nn.DropOutMask[i] > nn.DropoutFraction] tempMask = np.where(indices, 1, 0).astype( np.float64) # convert binary to float nn.DropOutMask[i] = tempMask.squeeze( ) # Remove the extra unwanted dimension nn.A[i] = np.multiply(nn.A[i], nn.DropOutMask[i]) nn.A[i][ nn.A[i] == 0.] = 0 # Remove the negative zeros that propagate to cause large errors in testing phase # Calculate running exponential activations for use with sparsity if nn.NonSparsityPenalty > 0: pass #Add Bias term biasTerm = np.ones(shape=(nn.A[i].shape[0], nn.A[i].shape[1] + 1.0), dtype=np.float64) biasTerm[:, 1:] = nn.A[i] nn.A[i] = biasTerm if nn.Output == 'sigm': nn.A[n - 1] = sigm.sigm(np.dot(nn.A[n - 2], nn.W[n - 2].T)) elif nn.Output == 'linear': nn.A[n - 1] = np.dot(nn.A[n - 2], nn.W[n - 2].T) elif nn.Output == 'softmax': nn.A[n - 1] = np.dot(nn.A[n - 2], nn.W[n - 2].T) maxVector = nn.A[n - 1].argmax(axis=1) # Returns a flattened array maxVector = maxVector.reshape(maxVector.shape[0], 1) # Convert to coloumn vector Mx1 nn.A[n - 1] = np.exp(np.subtract(nn.A[n - 1], maxVector)) sumVector = np.sum(nn.A[n - 1], 1) sumVector = sumVector.reshape(sumVector.shape[0], 1) # Convert to coloumn vector Mx1 nn.A[n - 1] = np.true_divide(nn.A[n - 1], sumVector) # Error and Loss nn.E = y - nn.A[n - 1] if nn.Output == 'sigm' or nn.Output == 'linear': nn.L = 0.5 * np.sum(np.power(nn.E, 2)) / m elif nn.Output == "softmax": # Try removing values that are too small. This helps countering the invalid value encountered in log warning eps = 1e-50 nn.A[n - 1][nn.A[n - 1] < eps] = eps nn.L = -1.0 * np.sum(np.multiply(y, np.log(nn.A[n - 1]))) / m return nn
def gini_index(X, y): """ This function implements the gini index feature selection. Input ---------- X: {numpy array}, shape (n_samples, n_features) input data y: {numpy array}, shape (n_samples,) input class labels Output ---------- gini: {numpy array}, shape (n_features, ) gini index value of each feature """ n_samples, n_features = X.shape # initialize gini_index for all features to be 0.5 gini = np.ones(n_features) * 0.5 # For i-th feature we define fi = x[:,i] ,v include all unique values in fi for i in range(n_features): v = np.unique(X[:, i]) for j in range(len(v)): # left_y contains labels of instances whose i-th feature value is less than or equal to v[j] print('X', X[:, i]) print('V', v[j]) left_y = y[X[:, i] <= v[j]] # right_y contains labels of instances whose i-th feature value is larger than v[j] right_y = y[X[:, i] > v[j]] # gini_left is sum of square of probability of occurrence of v[i] in left_y # gini_right is sum of square of probability of occurrence of v[i] in right_y gini_left = 0 gini_right = 0 for k in range(np.min(y), np.max(y) + 1): if len(left_y) != 0: # t1_left is probability of occurrence of k in left_y t1_left = np.true_divide(len(left_y[left_y == k]), len(left_y)) t2_left = np.power(t1_left, 2) gini_left += t2_left if len(right_y) != 0: # t1_right is probability of occurrence of k in left_y t1_right = np.true_divide(len(right_y[right_y == k]), len(right_y)) t2_right = np.power(t1_right, 2) gini_right += t2_right gini_left = 1 - gini_left gini_right = 1 - gini_right # weighted average of len(left_y) and len(right_y) t1_gini = (len(left_y) * gini_left + len(right_y) * gini_right) # compute the gini_index for the i-th feature value = np.true_divide(t1_gini, len(y)) if value < gini[i]: gini[i] = value return gini
def percentile(a, q, *args, **kwargs): q = np.true_divide(q, 100) return quantile(a, q, *args, **kwargs)
def interpolate(w, beta): return ( (1 - beta) * (np.true_divide(np.linalg.norm(w, ord=1), w.shape[1]))) + (beta * w)
def flatten_normal_map(normal_map, visualize=False): sigma = 64.0 normal_map_mask = np.isnan(normal_map) normal_map_smoothed = normal_map.copy() normal_map_smoothed[normal_map_mask] = 0.0 normal_map_smoothed = skimage.filters.gaussian(normal_map_smoothed, sigma=sigma, multichannel=True) normal_map_smoothed_nans = skimage.filters.gaussian((~normal_map_mask).astype(np.float), sigma=sigma, multichannel=True) normal_map_smoothed = np.true_divide( normal_map_smoothed, normal_map_smoothed_nans, out=np.zeros_like(normal_map_smoothed), where=normal_map_smoothed_nans != 0.0 ) norms = np.zeros((normal_map_smoothed.shape[0], normal_map_smoothed.shape[1], 3)) norms[:,:,0] = np.linalg.norm(normal_map_smoothed, axis=2) norms[:,:,1] = norms[:,:,0] norms[:,:,2] = norms[:,:,0] normal_map_smoothed = np.true_divide( normal_map_smoothed, norms, out=np.full_like(normal_map_smoothed, np.nan), where=norms != 0.0 ) flat_normal_map = normal_map + (np.array([0.0, 0.0, 1.0]) - normal_map_smoothed) norms = np.zeros((flat_normal_map.shape[0], flat_normal_map.shape[1], 3)) norms[:,:,0] = np.linalg.norm(flat_normal_map, axis=2) norms[:,:,1] = norms[:,:,0] norms[:,:,2] = norms[:,:,0] flat_normal_map = np.true_divide( flat_normal_map, norms, out=np.zeros_like(flat_normal_map), where=norms != 0.0 ) nm_min = -1.0 nm_max = 1.0 normal_map_scaled = (normal_map - nm_min) / (nm_max - nm_min) normal_map_smoothed_scaled = (normal_map_smoothed - nm_min) / (nm_max - nm_min) flat_normal_map_scaled = (flat_normal_map - nm_min) / (nm_max - nm_min) if visualize: fig = plt.figure(figsize=(16,16)) ax = fig.add_subplot(2, 2, 1) ax.grid(False) plt.imshow(normal_map_scaled, origin='upper') plt.title('Normal Map') ax = fig.add_subplot(2, 2, 2) ax.grid(False) plt.imshow(normal_map_smoothed_scaled, origin='upper') plt.title('Normal Map Smoothed') ax = fig.add_subplot(2, 2, 3) ax.grid(False) plt.imshow(flat_normal_map_scaled, origin='upper') plt.title('Flat Normal Map') plt.show() return flat_normal_map
##print len(four[0]),my_data[:,0][:len(four)] #print len(my_data) import locale additionlist = [] for ff in range(0, 12): tempolist = [0] * 71 list71 = [] for i in range(1, len(my_data)): templistt = [] for j in range(0, len(vallist[i])): #if my_data[:,1][i]>0: if int(my_data[:, ff][i] > 0): if (my_data[:, ff][i] * np.true_divide(vallist[i][j], sum(vallist[i]))): templistt.append( my_data[:, ff][i] * np.true_divide(vallist[i][j], sum(vallist[i]))) else: templistt.append(0) else: templistt.append(0) #print templistt # list71.append((templistt)) #print len(list32),'**' #print list32 #time.sleep(1) # for i in range(0,len(my_data)): #31 for j in range(0, 42):
def run_single(dist_type, gap, mu_style, hyp_style, pi1, no_arms, num_hyp, sigma, epsilon, top_arms, alpha0, trunctimerange, FDR, NUMRUN, mu_max, alg_num=0, punif=0, cauchyn=0, verbose=0, precision=1e-8): alg_name = alg_list[alg_num] numtrunc = len(trunctimerange) # Protocol-y results time_str = datetime.today().strftime("%m%d%y_%H%M") if not os.path.exists('./results'): os.makedirs('./results') res_filename = './results/output_%s.dat' % time_str result_file = open(res_filename, 'w') # Load mu_mat with ready to go mu (mu_mat, Hypo) = parse_mu.get_mu(dist_type, gap, mu_style, hyp_style, pi1, no_arms, num_hyp, sigma, epsilon, top_arms, mu_max) #ipdb.set_trace() num_alt = sum(Hypo) if dist_type == 1: bound_type = 'SubGaussian_LIL' elif dist_type == 0: bound_type = 'Bernoulli_LIL' # in fact can get rid of numtrunc, just when you want to debug can check pval at different times pval_mat = np.zeros(shape=(numtrunc, num_hyp, NUMRUN)) rej_mat = np.zeros(shape=(numtrunc, num_hyp, NUMRUN)) samples_mat = np.zeros(shape=(numtrunc, num_hyp, NUMRUN)) alpha_mat = np.zeros(shape=(numtrunc, num_hyp, NUMRUN)) wealth_mat = np.zeros(shape=(numtrunc, num_hyp, NUMRUN)) FDR_tsr = np.zeros(shape=(numtrunc, num_hyp, NUMRUN)) falrej_vec = np.zeros([numtrunc, NUMRUN]) correj_vec = np.zeros([numtrunc, NUMRUN]) samples_vec = np.zeros([numtrunc, NUMRUN]) totrej_vec = np.zeros([numtrunc, NUMRUN]) rightarm_vec = np.zeros([numtrunc, NUMRUN]) pval_vec = np.zeros(num_hyp) for l in range(NUMRUN): # Initialize FDR procedures, all first values are to be tossed (non-used, including alpha) if FDR == 0: proc = GAIPlus.GAI_proc(alpha0) elif FDR == 1: proc = Lord.LORD_proc(alpha0) elif FDR == 2: proc = GAI_MW.GAI_MW_proc(alpha0) elif FDR == 3: proc = wrongFDR.wrongFDR_proc(alpha0) elif FDR == 4: proc = AlphaInvest.ALPHA_proc(alpha0) # dummy wrong FDR, always giving same alpha0 or some other constant elif FDR == 5: proc = Bonferroni.BONF_proc(alpha0) tic = time.time() for i in range(num_hyp): # Get means of experiment mu_list = mu_mat[i] this_exp = rowexp_new.rowexp(Hypo[i], no_arms, 1, mu_list) if verbose: result_file.write("Run: %d\n" % l) #result_file.write(mu_list) # Draw exp if possibly alg-dependent exp this_alpha = proc.alpha[-1] if (punif == 1) & (Hypo[i] == 0): # Skip experiment rightarm_b = 0 bestarm_idx = -1 else: #if this_alpha == 0: # ipdb.set_trace() this_exp.multi_ab(this_alpha, trunctimerange, epsilon, bound_type, alg_name, 1, cauchyn, punif, verbose=verbose, precision=precision) rightarm_b = this_exp.rightarm bestarm_idx = this_exp.bestarm['index'] # Compute values and all for different truncation times for q, trunctime in enumerate(trunctimerange): #ipdb.set_trace() # Get P values if (punif == 1) & (Hypo[i] == 0): pval_mat[q][i][l] = np.random.rand( ) # Uniform if null hypothesis to get some FDR total_samples = 1 else: #pval_vec[i] = this_exp.pval[q] # Take the min of all p-values you have seen pval_mat[q][i][l] = this_exp.pval[q] #pval_mat[q][i][l] = min(this_exp.pvals) total_samples = this_exp.total_queries[q] samples_mat[q][i][l] = total_samples # If wealth still positive for that procedure if (proc.wealth_vec[-1] >= 0): # Reject rej_mat[q][i][l] = (pval_mat[q][i][l] <= this_alpha + precision) # Total measures falrej_vec[q][l] = falrej_vec[q][l] + rej_mat[q][i][l] * ( 1 - Hypo[i]) correj_vec[q][ l] = correj_vec[q][l] + rej_mat[q][i][l] * Hypo[i] rightarm_vec[q][l] = rightarm_vec[q][l] + ( Hypo[i]) * rightarm_b * rej_mat[q][i][l] samples_vec[q][l] = samples_vec[q][l] + total_samples totrej_vec[q][l] = falrej_vec[q][l] + correj_vec[q][l] FDR_tsr[q][i][l] = np.true_divide(falrej_vec[q][l], max(totrej_vec[q][l], 1)) #ipdb.set_trace() if verbose: result_file.write( "true best: %d, found best: %d, queries: %d \n" % (argmax(mu_list), bestarm_idx, total_samples)) result_file.write( "alpha_j: %f, p_j: %f, rej: %d \n" % (this_alpha, pval_mat[q][i][l], rej_mat[q][i][l])) if (proc.wealth_vec[-1] >= 0): # Get next alpha (and wealth) from FDR if theres a next hypothesis to test if i < num_hyp - 1: wealth_mat[q][i][l] = proc.wealth_vec[-1] alpha_mat[q][i][l] = proc.next_alpha( rej_mat[q][i][l]) # use last rejection #ipdb.set_trace() if verbose: result_file.write( "Time for one complete experiment with %d hypotheses was %f" % (num_hyp, time.time() - tic)) #ipdb.set_trace() # Save data dir_name = './dat' for q, trunctime in enumerate(trunctimerange): #ipdb.set_trace() FDR_vec = np.true_divide( falrej_vec[q], [max(totrej_vec[q][l], 1) for l in range(len(totrej_vec[q]))]) TDR_vec = np.true_divide(correj_vec[q], num_alt) BDR_vec = np.true_divide(rightarm_vec[q], num_alt) FDR_mat = FDR_tsr[q] pr_filename = 'PR_D%d_MS%d_AG%d_G%.1f_MM%.1f_E%.1f_Si%.1f_TA%d_HS%d_P%.1f_AL%.1f_FDR%d_NH%d_NA%d_TT%d_PU%d_CN%d_NR%d_%s' % ( dist_type, mu_style, alg_num, gap, mu_max, epsilon, sigma, top_arms, hyp_style, pi1, alpha0, FDR, num_hyp, no_arms, trunctime, punif, cauchyn, NUMRUN, time_str) ad_filename = 'AD_D%d_MS%d_AG%d_G%.1f_MM%.1f_E%.1f_Si%.1f_TA%d_HS%d_P%.1f_AL%.1f_FDR%d_NH%d_NA%d_TT%d_PU%d_CN%d_NR%d_%s' % ( dist_type, mu_style, alg_num, gap, mu_max, epsilon, sigma, top_arms, hyp_style, pi1, alpha0, FDR, num_hyp, no_arms, trunctime, punif, cauchyn, NUMRUN, time_str) td_filename = 'TD_D%d_MS%d_AG%d_G%.1f_MM%.1f_E%.1f_Si%.1f_TA%d_HS%d_P%.1f_AL%.1f_FDR%d_NH%d_NA%d_TT%d_PU%d_CN%d_NR%d_%s' % ( dist_type, mu_style, alg_num, gap, mu_max, epsilon, sigma, top_arms, hyp_style, pi1, alpha0, FDR, num_hyp, no_arms, trunctime, punif, cauchyn, NUMRUN, time_str) # Save data saveres(dir_name, td_filename, FDR_mat) saveres(dir_name, ad_filename, [ BDR_vec, TDR_vec, FDR_vec, samples_vec[q], falrej_vec[q], totrej_vec[q] ]) saveres( dir_name, pr_filename, np.r_[rej_mat[q], pval_mat[q], alpha_mat[q], wealth_mat[q], samples_mat[q]]) result_file.close()
statements = full_data['Statement'] speakers = full_data['Speaker'] ratings = ratings.replace(0, 1) ratings = ratings.replace(2, 1) ratings = ratings.replace(5, 0) ratings = ratings.replace(4, 0) ratings = ratings.replace(3, 0) encoder = LabelEncoder() labels = encoder.fit_transform(ratings) speakers = encoder.fit_transform(speakers) speakers = np.array(speakers) speakers = np.true_divide(speakers, speakers.argmax()) statements = preprocess_data(statements, _regular_expressions=True, _remove_capitals=True, _remove_punctuation=True, _remove_stopwords=True, _stemming=True) all_words = [] for statement in statements: words = word_tokenize(statement) for w in words: all_words.append(w) all_words = nltk.FreqDist(all_words) most_common = all_words.most_common(WORDS_NUMBER) ######################################################################################################
validValues = [66, 68, 130, 132] elif product == "LS8_OLI_LASRC": validValues = [322, 386, 834, 898, 1346, 324, 388, 836, 900, 1348] cloud_mask = isin(nbar["pixel_qa"].values, validValues) for band in bands: datos = np.where( np.logical_and(nbar.data_vars[band] != nodata, cloud_mask), nbar.data_vars[band], np.nan) allNan = ~np.isnan(datos) if normalized: m = np.nanmean(datos.reshape((datos.shape[0], -1)), axis=1) st = np.nanstd(datos.reshape((datos.shape[0], -1)), axis=1) datos = np.true_divide( (datos - m[:, np.newaxis, np.newaxis]), st[:, np.newaxis, np.newaxis]) * np.nanmean(st) + np.nanmean(m) medians[band] = np.nanmedian(datos, 0) medians[band][np.sum(allNan, 0) < minValid] = np.nan del datos period_green = medians["green"] period_nir = medians["swir1"] del medians mask_nan = np.logical_or(np.isnan(period_green), np.isnan(period_nir)) period_ndwi = np.true_divide(np.subtract(period_green, period_nir), np.add(period_green, period_nir)) period_ndwi[mask_nan] = np.nan #Hace un clip para evitar valores extremos. period_ndwi[period_ndwi > 1] = 1.0 period_ndwi[period_ndwi < -1] = np.nan import xarray as xr
tan = utils.copy_docstring( 'tf.math.tan', lambda x, name=None: np.tan(x)) tanh = utils.copy_docstring( 'tf.math.tanh', lambda x, name=None: np.tanh(x)) top_k = utils.copy_docstring( 'tf.math.top_k', _top_k) truediv = utils.copy_docstring( 'tf.math.truediv', lambda x, y, name=None: np.true_divide(x, y)) # unsorted_segment_max = utils.copy_docstring( # 'tf.math.unsorted_segment_max', # lambda data, segment_ids, num_segments, name=None: ( # np.unsorted_segment_max)) # unsorted_segment_mean = utils.copy_docstring( # 'tf.math.unsorted_segment_mean', # lambda data, segment_ids, num_segments, name=None: ( # np.unsorted_segment_mean)) # unsorted_segment_min = utils.copy_docstring( # 'tf.math.unsorted_segment_min', # lambda data, segment_ids, num_segments, name=None: ( # np.unsorted_segment_min))
# ------------------------------------------------------------------------- # *** Calculate average within run print('------Calculating average within run') # In order to reduce memory demands (in case of a large number of runs), we # calculate the average time series within runs in a first step, and form # the overall average at the end. First, calculate the sum over the fifth # dimension (which represents the block number), producing a four- # dimensional array: aryTmp = np.sum(aryTmpBlcks, axis=0, keepdims=False) # Divide by number of blocks: aryTmp = np.true_divide(aryTmp, varTmpNumBlck) # Append array to list: aryRunsAvrgs[index_02, :, :, :, :] = np.copy(aryTmp).astype(np.float32) # ----------------------------------------------------------------------------- # *** Calculate average across runs print('---Calculating average across runs') # Create the sum over the dimension of the array that represents run number, # producing a four-dimensional array: aryAvrg = np.sum(aryRunsAvrgs, axis=0, keepdims=False)
y = X y_hat = decoder saver = tf.train.Saver() session = tf.Session() saver.restore(session, './mnist-autoencoder-99') prediction = session.run([decoder], feed_dict={X: im}) prediction = np.asarray(prediction).reshape((28, 28)) * 255. im = im.reshape((28, 28)) * 255. image = Image.fromarray( np.concatenate((im, prediction), axis=0).astype('uint8'), 'L') image.save('/Users/mesuterhanunal/Desktop/images/%d.png' % random.randint(0, 100000)) if __name__ == '__main__': X_train, _, X_test, Y_test = mnist.load() X_train = np.true_divide(X_train, 255.0) X_test = np.true_divide(X_test, 255.0) autoencoder = AutoEncoder( X_train, hidden_sizes=[64, 64, 2], num_layers=3, activations=['tanh', 'tanh', 'tanh', 'tanh', 'tanh', 'relu']) # autoencoder.train(lr=0.00002, epoch=100, batch_size=64) for i in range(10): idx = np.random.choice(np.argwhere(Y_test == i).reshape(-1), 1) autoencoder.test(X_test[idx].reshape((1, 784)))
def receiver(data): data = list(data) chirp = CHIRP #Find how much to shift to reach the first chirp //Synchronisation shifts = shift_finder(chirp, data, SAMPLE_FREQUENCY, window=0) shift = shifts[0] + 1 # Remove Stuff before and after data and split into frames # 1) Remove everything up to the beginning of the first chirp # 2) Split into frames (unkown number), remove last frame if it's not a full frame data = data[shift - CHIRP_BLOCKS_PER_FRAME * PREFIXED_SYMBOL_LENGTH:] data = [ data[i:i + DATA_PER_FRAME] for i in range(0, len(data), DATA_PER_FRAME) ] if len(data[-1]) != DATA_PER_FRAME: del data[-1] # Remove the chirp data = [ frame[CHIRP_BLOCKS_PER_FRAME * PREFIXED_SYMBOL_LENGTH:] for frame in data ] # Channel Estimation frame = data[0] # Split into symbols frame = [ frame[i:i + PREFIXED_SYMBOL_LENGTH] for i in range(0, len(frame), PREFIXED_SYMBOL_LENGTH) ] # Isolate Estimation Symbols estimation_symbols = frame[0:KNOWN_DATA_BLOCKS_PER_FRAME] estimation_symbols = [norm(symbol) for symbol in estimation_symbols] known_symbol = get_known_data() known_symbol = norm(known_symbol) channel_response = channel_estimation(estimation_symbols, known_symbol) # plt.figure() # plt.plot(channel_response.real,color='r') # plt.plot(channel_response.imag,color='b') # plt.show() # plt.figure() # plt.scatter(channel_response.real,channel_response.imag) # Isolate data symbols data = [ frame[KNOWN_DATA_BLOCKS_PER_FRAME * PREFIXED_SYMBOL_LENGTH:-PREFIXED_SYMBOL_LENGTH * KNOWN_DATA_BLOCKS_PER_FRAME] for frame in data ] #Remove chirp and take blocks of 4800 for i, frame in enumerate(data): frame = [ frame[i:i + PREFIXED_SYMBOL_LENGTH][CP:] for i in range(0, len(frame), PREFIXED_SYMBOL_LENGTH) ] data[i] = frame data = [symbol for frame in data for symbol in frame] #Power Checking symbol_powers = np.array( [np.sqrt(np.mean(np.square(symbol))) for symbol in data]) symbol_powers -= np.min(symbol_powers) symbol_powers = norm(symbol_powers) for i, power in enumerate(symbol_powers): if power < 0.1 * 32767: data[i] = None data = [symbol for symbol in data if symbol] #check_typing(data) # plt.figure() # plt.plot(symbol_powers) # plt.show() # FFT the symbols data = [np.fft.fft(symbol, N) for symbol in data] # plt.figure() # plt.scatter(np.array(data).real, np.array(data).imag) # plt.show() # Divide each symbol by channel response data = [ np.true_divide(symbol, channel_response).tolist() for symbol in data ] # Discard second half of all symbols and keep only symbols in bins 100-1500 data = [ symbol[L_PADDING + 1:1 + L_PADDING + CONSTELLATION_VALUES_PER_BLOCK] for symbol in data ] # Flatten into single list of symbols data = [value for symbol in data for value in symbol] # plt.figure() # plt.title("Symbols before demapping") # plt.scatter(np.array(data).real, np.array(data).imag) # plt.show() # Map each symbol to constellation values for i, value in enumerate(data): # Get distance to all symbols in constellation distances = { abs(value - const_value): key for key, const_value in CONSTELLATION.items() } # Get minimum distance minimum_distance = min(distances.keys()) # Find symbol matching minimum distance and append data[i] = distances[minimum_distance] # Make into one big string data = "".join(["".join(symbol) for symbol in data]) return data
def take_snapshot(camera_position, camera_lookat, is_layer_view) -> typing.Optional[QImage]: """ Take a snapshot of the current scene. :param camera_position: The position of the camera to take the snapshot with. :param camera_lookat: The position of the focal point of the camera. :param is_layer_view: Whether we're looking at layer view or the model itself. :return: A screenshot of the current scene. """ application = cura.CuraApplication.CuraApplication.getInstance() plugin_registry = application.getPluginRegistry() # Set the camera to the desired position. We'll use the actual camera for the snapshot just because it looks cool while it's busy. camera = application.getController().getScene().getActiveCamera() camera.setPosition( UM.Math.Vector.Vector(camera_position[0], camera_position[2], camera_position[1]) ) # Note that these are OpenGL coordinates, swapping Y and Z. if not camera_lookat: bounding_box = UM.Math.AxisAlignedBox.AxisAlignedBox() for node in UM.Scene.Iterator.DepthFirstIterator.DepthFirstIterator( cura.CuraApplication.CuraApplication.getInstance( ).getController().getScene().getRoot()): if node.isSelectable(): bounding_box = bounding_box + node.getBoundingBox() camera_lookat = bounding_box.center else: camera_lookat = UM.Math.Vector.Vector(camera_lookat[0], camera_lookat[2], camera_lookat[1]) camera.lookAt(camera_lookat) if abs(camera.getPosition().x - camera_lookat.x) < 0.01 and abs( camera.getPosition().z - camera_lookat.z) < 0.01: # Looking straight up or straight down. # Make sure the yaw of the camera is consistent regardless of previous position. if camera.getPosition().y > camera_lookat.y: camera.setOrientation(UM.Math.Quaternion.Quaternion(-2, 0, 0, 2)) else: camera.setOrientation(UM.Math.Quaternion.Quaternion(2, 0, 0, 2)) time.sleep( 2 ) # Some time to update the scene nodes. Don't know if this is necessary but it feels safer. # Use a transparent background. gl_bindings = UM.View.GL.OpenGL.OpenGL.getInstance().getBindingsObject() gl_bindings.glClearColor(0.0, 0.0, 0.0, 0.0) gl_bindings.glClear(gl_bindings.GL_COLOR_BUFFER_BIT | gl_bindings.GL_DEPTH_BUFFER_BIT) try: # In Qt6 it's an enum, in Qt5 it's a field of QImage. colour_format = QImage.Format.Format_ARGB32 # For Qt6. except AttributeError: colour_format = QImage.Format_ARGB32 # For Qt5. if is_layer_view: # Remove any nozzle node. It can get in the way of what we want to see and influence cropping of the image badly. simulation_view_plugin = plugin_registry.getPluginObject( "SimulationView") for node in UM.Scene.Iterator.DepthFirstIterator.DepthFirstIterator( application.getController().getScene().getRoot()): if hasattr( node, "_createNozzleMesh" ): # This node is a NozzleNode (the actual class is not exposed to us outside the plug-in). node.getParent().removeChild(node) render_pass = simulation_view_plugin.getSimulationPass() render_pass.render() time.sleep(1.2) screenshot = render_pass.getOutput() print("---- screenshot size:", screenshot.width(), "x", screenshot.height()) if screenshot.width() != render_width or screenshot.height( ) != render_height: print( "---- render output not correct size! Resizing window to compensate." ) main_window = application.getMainWindow() delta_width = render_width - screenshot.width() delta_height = render_height - screenshot.height() main_window.setWidth(main_window.width() + delta_width) main_window.setHeight(main_window.height() + delta_height) return None # Failed to render. Try again after waiting outside of Qt thread. # Remove alpha channel from this picture. We don't want the semi-transparent support since we don't draw the object outline here. # Sadly, QImage.convertToFormat has only 2 formats with boolean alpha and they both premultiply. So we'll go the hard way: Through Numpy. pixel_bits = screenshot.bits().asarray(screenshot.sizeInBytes()) pixels = numpy.frombuffer(pixel_bits, dtype=numpy.uint8).reshape( [screenshot.height(), screenshot.width(), 4]) opaque = numpy.nonzero(pixels[:, :, 0]) pixels[opaque[0], opaque[1], 3] = 255 return QImage( pixels.data, pixels.shape[1], pixels.shape[0], colour_format ).copy( ) # Make a copy because the pixel data will go out of scope for Numpy, so that would be invalid memory. else: # Render the objects themselves! Going to be quite complex here since the render is highly specialised in what it shows and what it doesn't. view = plugin_registry.getPluginObject("SolidView") view._checkSetup() renderer = view.getRenderer() support_angle = application.getGlobalContainerStack().getProperty( "support_angle", "value") view._enabled_shader.setUniformValue( "u_overhangAngle", math.cos( math.radians(90 - support_angle))) # Correct overhang angle. view._enabled_shader.setUniformValue( "u_lowestPrintableHeight", -1.0) # Don't show initial layer height. object_batch = renderer.createRenderBatch(shader=view._enabled_shader) renderer.addRenderBatch(object_batch) for node in UM.Scene.Iterator.DepthFirstIterator.DepthFirstIterator( application.getController().getScene().getRoot()): if not node.getMeshData() or not node.isSelectable(): continue uniforms = {} # Get the object's colour. extruder_index = int( node.callDecoration("getActiveExtruderPosition")) material_color = application.getExtrudersModel().getItem( extruder_index)["color"] uniforms["diffuse_color"] = [ int(material_color[1:3], 16) / 255, int(material_color[3:5], 16) / 255, int(material_color[5:7], 16) / 255, 1.0 ] # Render with special shaders for special types of meshes, or otherwise in the normal batch. if node.callDecoration("isNonPrintingMesh") and ( node.callDecoration("isInfillMesh") or node.callDecoration("isCuttingMesh")): renderer.queueNode(node, shader=view._non_printing_shader, uniforms=uniforms, transparent=True) elif node.callDecoration("isSupportMesh"): uniforms["diffuse_color_2"] = [ uniforms["diffuse_color"][0] * 0.6, uniforms["diffuse_color"][1] * 0.6, uniforms["diffuse_color"][2] * 0.6, 1.0 ] renderer.queueNode(node, shader=view._support_mesh_shader, uniforms=uniforms) else: object_batch.addItem( node.getWorldTransformation(copy=False), node.getMeshData(), uniforms=uniforms, normal_transformation=node.getCachedNormalMatrix()) default_pass = renderer.getRenderPass("default") default_pass.render() time.sleep(1.2) normal_shading = default_pass.getOutput() xray_pass = renderer.getRenderPass("xray") renderer.addRenderPass(xray_pass) xray_pass.render() time.sleep(1.2) xray_shading = xray_pass.getOutput() # Manually composite these shadings. Because the composite shader also adds a background colour. normal_data = normal_shading.bits().asarray( normal_shading.sizeInBytes()) composite_pixels = numpy.frombuffer(normal_data, dtype=numpy.uint8).reshape([ normal_shading.height(), normal_shading.width(), 4 ]) # Start from the normal image. colours = numpy.true_divide(composite_pixels[:, :, 0:3], 255) # Scaled to [0, 1]. alpha = numpy.true_divide(composite_pixels[:, :, 3], 255) xray_data = xray_shading.bits().asarray(xray_shading.sizeInBytes()) xray_pixels = numpy.frombuffer(xray_data, dtype=numpy.uint8).reshape( [xray_shading.height(), xray_shading.width(), 4]) xray_pixels = numpy.mod( xray_pixels[:, :, 0:3], 10 ) // 5 # The X-ray shader creates increments of 5 for some reason. If there are an odd number of increments (not divisible by 10) then it must be highlighted. hue_shift = ((alpha - 0.333) * 6.2831853) cos_shift = numpy.repeat(numpy.expand_dims(numpy.cos(-hue_shift), axis=2), 3, axis=2) sin_shift = numpy.repeat(numpy.expand_dims(numpy.sin(-hue_shift), axis=2), 3, axis=2) k = numpy.array( [0.57735, 0.57735, 0.57735] ) # 1/sqrt(3), resulting in a diagonal unit vector around which we rotate the channels. cross_colour = numpy.cross(colours, k) * -1 dot_colour = numpy.repeat(numpy.expand_dims(numpy.dot(colours, k), axis=2), 3, axis=2) rotated_hue = colours * cos_shift + cross_colour * sin_shift + ( cos_shift * -1 + 1.0) * dot_colour * k # Rodrigues' rotation formula! rotated_hue = rotated_hue * 255 composite_pixels[:, :, 0: 3] -= composite_pixels[:, :, 0: 3] * xray_pixels # Don't use the normal colour for x-rayed pixels. composite_pixels[:, :, 0:3] += (rotated_hue * xray_pixels).astype( "uint8") # Use the rotated colour instead. composite_pixels[:, :, 3][alpha > 0.1] = 255 return QImage( composite_pixels.data, composite_pixels.shape[1], composite_pixels.shape[0], colour_format ).copy( ) # Make a copy because the pixel data will go out of scope for Numpy, so that would be invalid memory.
def time_quantity_np_truediv(self): np.true_divide(self.data, self.data2)
def channel_estimation(symbols, known_block): first_symbol = np.fft.fft(symbols[0], N) last_symbol = np.fft.fft(symbols[-1], N) # known_block = known_block[CP:] # symbols = [symbol[CP:] for symbol in symbols] # Take average value of H determined for each block symbols = np.average(symbols, axis=0) symbols_freq = np.fft.fft(symbols, N) #This should not print out anything and yet it does print([block for block in known_block if np.abs(block) < 0.00001]) known_block_freq = np.fft.fft(known_block, N) channel_response_freq = np.true_divide( symbols_freq, known_block_freq, out=np.zeros_like(symbols_freq), where=np.abs(known_block_freq) > 0.01, ) # Remove DC value channel_response_freq[0] = 0 channel_response_freq[int(N / 2)] = 0 #Might be needed later to avoid decoding issues # channel_response = np.fft.ifft(channel_response_freq, N)[:10] # plt.figure() # plt.title("Channel Responce in DFT domain") # plt.plot(channel_response) # plt.show() #####Linear phase stuff first_symbol_resp = np.true_divide(first_symbol, known_block_freq, out=np.zeros_like(first_symbol), where=known_block_freq != 0) last_symbol_resp = np.true_divide(last_symbol, known_block_freq, out=np.zeros_like(last_symbol), where=known_block_freq != 0) phase_shift_start = np.angle(first_symbol_resp, deg=True) phase_shift_end = np.angle(last_symbol_resp, deg=True) phase_shift_start = np.unwrap(phase_shift_start) phase_shift_end = np.unwrap(phase_shift_end) phase_shift = np.subtract(phase_shift_end, phase_shift_start) x = np.linspace(0, N, N) # plt.figure() # plt.plot(first_symbol_resp, color='r') # plt.plot(last_symbol_resp) # plt.show() # plt.figure() # plt.plot(phase_shift,label="sub") lin_phase_shift = np.polyfit(x, phase_shift, deg=1) # print(lin_phase_shift) lin_phase_shift = [i * lin_phase_shift[0] + lin_phase_shift[1] for i in x] # plt.plot(lin_phase_shift, label='lin') # plt.legend() # plt.show() #channel_response_freq = np.fft.fft(channel_response,N) return channel_response_freq
def truncated_division(a, b): ints = [np.int8, np.int16, np.int32, np.int64] if b.dtype in ints: return np.trunc(np.true_divide(a, b)).astype(b.dtype) else: return np.true_divide(a, b).astype(b.dtype)
def load_data_train(root, img_size=64): ''' loads the data from files and returns a torch dataset ''' data_folders = [os.path.join(root, 'Imagenet64_train_part1'), os.path.join(root, 'Imagenet64_train_part2')] x_data = [] img_size2 = img_size * img_size tot_mean = None for data_folder in data_folders: for filename in os.listdir(data_folder): # read the data as numpy arrays with open(os.path.join(data_folder, filename), 'rb') as f: print("processing file: ", filename) print("unpickling") datadict = pickle.load(f, encoding='latin1') x = np.array(datadict['data']) # y = np.array(datadict['labels']) mean_image = datadict['mean'] print("finished unpickling") print("preprocessing data") # performing normalization i.e. x /= np.float32(255) np.true_divide(x, np.float32(255), out=x, casting='unsafe') np.true_divide(mean_image, np.float32(255), out=mean_image, casting='unsafe') # Labels are indexed from 1, shift it so that indexes start at 0 # y = [i-1 for i in y] np.subtract(x, mean_image, out=x, casting='unsafe') if tot_mean is None: tot_mean = mean_image else: tot_mean += mean_image x = np.dstack((x[:, :img_size2], x[:, img_size2:2*img_size2], x[:, 2*img_size2:])) x = x.reshape((x.shape[0], img_size, img_size, 3)).transpose(0, 3, 1, 2) x_data.append(x) # y_data.append(y) print("completed preprocessing\n") # X_train, Y_train = np.concatenate(x_data), np.concatenate(y_data) X_train = np.concatenate(x_data) tot_mean /= 10 # print("done processing everything!! Creating mirrored versions now") # create mirrored images # X_train = x[0:data_size, :, :, :] # Y_train = y[0:data_size] # X_train_flip = X_train[:, :, :, ::-1] # Y_train_flip = Y_train # X_train = np.concatenate((X_train, X_train_flip), axis=0) # Y_train = np.concatenate((Y_train, Y_train_flip), axis=0) print("returning data now") return (ImageNet64Data(X_train, None), # CIFAR10Data(X_val, y_val), # CIFAR10Data(X_test, y_test), mean_image)
m_new = m st_new = st for axis in l: # If axis is 0 it is equivalent to x[np.newaxis,:] # If axis is 1 it is equivalent to x[:,np.newaxis] # And so on m_new = np.expand_dims(m_new, axis=axis) st_new = np.expand_dims(st_new, axis=axis) print('Time axis', time_axis) print('New axis', l) print('m', m.shape) print('st', st.shape) print('st_new', st_new.shape) print('m_new', m_new.shape) datos = np.true_divide( (datos - m_new), st_new) * np.nanmean(st) + np.nanmean(m) medians[band] = np.nanmedian(datos, time_axis) medians[band][np.sum(allNan, time_axis) < minValid] = -9999 medians["ndvi"] = np.true_divide(medians["nir"] - medians["red"], medians["nir"] + medians["red"]) medians["nbr"] = np.true_divide(medians["nir"] - medians["swir1"], medians["nir"] + medians["swir1"]) medians["nbr2"] = np.true_divide(medians["swir1"] - medians["swir2"], medians["swir1"] + medians["swir2"]) medians["ndmi"] = np.true_divide(medians["nir"] - medians["swir1"], medians["nir"] + medians["swir1"]) #medians["gndvi"]=np.true_divide(medians["nir"]-medians["green"],medians["nir"]+medians["green"]) medians["rvi"] = np.true_divide(medians["nir"], medians["red"])
def truncated_mod(a, b): return a-b*np.trunc(np.true_divide(a, b)).astype(b.dtype)
def prep_data(data, covariates, data_start, train=True): #print("train: ", train) time_len = data.shape[0] #print("time_len: ", time_len) input_size = window_size - stride_size windows_per_series = np.full((num_series), (time_len - input_size) // stride_size) #print("windows pre: ", windows_per_series.shape) if train: windows_per_series -= (data_start + stride_size - 1) // stride_size #print("data_start: ", data_start.shape) #print(data_start) #print("windows: ", windows_per_series.shape) #print(windows_per_series) total_windows = np.sum(windows_per_series) x_input = np.zeros((total_windows, window_size, 1 + num_covariates + 1), dtype='float32') label = np.zeros((total_windows, window_size), dtype='float32') v_input = np.zeros((total_windows, 2), dtype='float32') #cov = 3: ground truth + age + day_of_week + hour_of_day + num_series #cov = 4: ground truth + age + day_of_week + hour_of_day + month_of_year + num_series count = 0 if not train: covariates = covariates[-time_len:] for series in trange(num_series): cov_age = stats.zscore(np.arange(total_time - data_start[series])) if train: covariates[data_start[series]:time_len, 0] = cov_age[:time_len - data_start[series]] else: covariates[:, 0] = cov_age[-time_len:] for i in range(windows_per_series[series]): if train: window_start = stride_size * i + data_start[series] else: window_start = stride_size * i window_end = window_start + window_size ''' print("x: ", x_input[count, 1:, 0].shape) print("window start: ", window_start) print("window end: ", window_end) print("data: ", data.shape) print("d: ", data[window_start:window_end-1, series].shape) ''' x_input[count, 1:, 0] = data[window_start:window_end - 1, series] x_input[count, :, 1:1 + num_covariates] = covariates[window_start:window_end, :] x_input[count, :, -1] = series label[count, :] = data[window_start:window_end, series] nonzero_sum = (x_input[count, 1:input_size, 0] != 0).sum() if nonzero_sum == 0: v_input[count, 0] = 0 else: v_input[count, 0] = np.true_divide( x_input[count, 1:input_size, 0].sum(), nonzero_sum) + 1 x_input[count, :, 0] = x_input[count, :, 0] / v_input[count, 0] if train: label[count, :] = label[count, :] / v_input[count, 0] count += 1 prefix = os.path.join(save_path, 'train_' if train else 'test_') np.save(prefix + 'data_' + save_name, x_input) np.save(prefix + 'v_' + save_name, v_input) np.save(prefix + 'label_' + save_name, label)
def normalizeQueryState(data): if data is None: return data[0] = data[0].astype(float) data[1] = data[1].astype(float) data[2] = data[2].astype(float) data[0][0] = np.true_divide(data[0][0],20000) data[0][1] = np.true_divide(data[0][1],5000) data[0][2] = np.true_divide(data[0][2],5000) data[0][3] = np.true_divide(data[0][3],200) data[0][4] = np.true_divide(data[0][4],200) data[0][5] = np.true_divide(data[0][5],200) data[0][6] = np.true_divide(data[0][6],200) data[1][0] = np.true_divide(data[1][0],4) data[1][1] = np.true_divide(data[1][1],2) data[2][0] = np.true_divide(data[2][0],3) data[2][1] = np.true_divide(data[2][1],2) data[2][3] = np.true_divide(data[2][3],3) data[2][4] = np.true_divide(data[2][4],2000) data[2][5] = np.true_divide(data[2][5],255) return data
def test_true_divide_2(self): from numpy import arange, array, true_divide assert (true_divide(arange(3), array([2, 2, 2])) == array([0, 0.5, 1])).all()
def load_data(project_name, pickle_name): ''' Description: data loader for two different projects: -'mnist' -'celebA' This function is taylored specifically for the data organization of these projects and needs to be adapted in order to work with other projects. Inputs: -'project_name' (str) = the name of the project from which we will load data. -'pickle_name' (str) = the name of the pickle file containing the data. Return: -'data_dict' (data_dict) = for the 'mnist' project, a data dictionary with form data_dict[key_1][key_2] with key_1 in ['train', 'testa', 'testb'] and key_2 in ['inputs', 'target']. OR -'images_array' (np.ndarray) = a stacked 4-D array of shape 'num_images' x 64 x 64 x 3. -'features_images_dict' (dict) = a dictionary with structure {'face features': [list of indices of images having that face feature]}. ''' # Data loading for the 'mnist' project. Expected file structure is # root # |--mnist # |--const.DIR_DATA # |--'pickle_name'-tf-train.pkl # |--'pickle_name'-tf-testa.pkl # |--'pickle_name'-tf-testb.pkl if project_name == 'mnist': data_dict = {} for key in const.LIST_TRAIN_TESTA_TESTB_KEYS: data_dict[key] = {} path = os.path.join(const.PATH_ROOT, project_name, const.DIR_DATA, pickle_name + '-' + key + '.pkl') with open(path, 'rb') as file_handle: data_tuple = pickle.load(file_handle) data_dict[key]['inputs'] = np.stack(data_tuple[0] / 255) data_dict[key]['inputs'] = np.expand_dims(np.copy( data_dict[key]['inputs']), axis=3) data_dict[key]['target'] = np.stack(data_tuple[1]) return data_dict # Data loading for the 'celebA' project. Expected file structure is # root # |--celebA # |--const.DIR_DATA # |--'pickle_name'.pkl # |--'pickle_name'_details.pkl (optional, not used for training) elif project_name == 'celebA': # Get face image data: path = os.path.join(const.PATH_ROOT, project_name, const.DIR_DATA, pickle_name + '.pkl') with open(path, 'rb') as file_handle: images_array = pickle.load(file_handle) images_array = np.true_divide(images_array, 255.0) # Looks for face attribute labels too: path = os.path.join(const.PATH_ROOT, project_name, const.DIR_DATA, pickle_name + '_details.pkl') try: with open(path, 'rb') as file_handle: features_images_dict = pickle.load(file_handle) except: features_images_dict = {} return images_array, features_images_dict
import numpy as np a = np.array([5, 5, -5, -5]) b = np.array([2, -2, 2, -2]) print(a, b) c = np.true_divide(a, b) d = np.divide(a, b) e = a / b print(c, d, e) f = np.floor_divide(a, b) g = a // b print(f, g) h = np.ceil(a / b).astype(int) print(h) i = np.trunc(a / b).astype(int) j = (a / b).astype(int) print(i, j) k = np.remainder(a, b) l = np.mod(a, b) m = a % b print(k, l, m) n = np.fmod(a, b) print(n)
satype = 'C[%d:]=A[%d:]%sscalar' % ( zoff, xoff, op) y = y[0] elif scalararraytype == 2: satype = 'C[%d:]=scalar%sA[%d:]' % ( zoff, op, yoff) #y = x y=y[0] x = x[0] CPA_min = runBench( np_func, z, x, y, internalCount=internalCount) CPE_min = np.true_divide(CPA_min, n) CPEs[zi][xi][yi] = CPE_min if args.verbose: print(args.prefix, '% 7s' % impl, satype, np_type.__name__, '% 6d' % internalCount, '% 7d' % n, '% 4.2f' % CPE_min, sep=', ', flush=True) if scalararraytype == 0: satype = ' array%sarray' % op elif scalararraytype == 1: satype = 'array%sscalar' % op
def get_crawled_term_analysis_page(shared_state): st.header("Crawled term analysis") crawled_terms_df = shared_state.crawled_terms_df df_counts_by_hour = shared_state.df_counts_by_hour df_most_common_hashtags = shared_state.df_most_common_hashtags df_most_common_tokens = shared_state.df_most_common_tokens df_cooccurrence = shared_state.df_cooccurrence st.subheader("List of Crawled Terms (since 3rd of November)") st.dataframe(crawled_terms_df) st.subheader("Co-occurrence matrix (for terms with count > 5000)") st.dataframe(df_cooccurrence) co_occurrence_diagonal = np.diagonal(df_cooccurrence) with np.errstate(divide="ignore", invalid="ignore"): co_occurrence_fraction = np.nan_to_num( np.true_divide(df_cooccurrence, co_occurrence_diagonal[:, None]) ) fig = plt.figure() st.subheader("Co-occurence heatmap (inverted log-scaling)") st.markdown("Closer to 0 means higher correlation") with st.echo(): co_occurrence_heatmap_df = pd.DataFrame( np.log(co_occurrence_fraction), index=df_cooccurrence.index, columns=df_cooccurrence.columns, ) np.fill_diagonal(co_occurrence_heatmap_df.values, np.nan) sns.heatmap( co_occurrence_heatmap_df, cbar=False, square=True, annot=True, vmin=-5, vmax=0, center=-2, cmap="PuBu", linecolor="black", ) plt.tick_params( axis="both", which="major", labelsize=10, labelbottom=False, bottom=False, top=False, labeltop=True, ) plt.xticks(rotation=45) st.pyplot(fig) selected_crawled_term = st.selectbox("Select term", crawled_terms_df["term"].values) st.plotly_chart( plotly_hourly_coverage( df_counts_by_hour, selected_crawled_term, selected_crawled_term ) ) col1, col2 = st.columns(2) col1.subheader("Top hashtags for '{}'".format(selected_crawled_term)) col1.dataframe(get_most_common(df_most_common_hashtags, selected_crawled_term, 15)) col2.subheader("Top tokens for '{}'".format(selected_crawled_term)) col2.dataframe(get_most_common(df_most_common_tokens, selected_crawled_term, 15)) st.subheader("--Temporarily Disabled--") st.subheader("10 randomly sampled tweets from '{}'".format(selected_crawled_term)) # term_stats = ( # recent_tweet_df[recent_tweet_df[selected_crawled_term] == 1][ # ["retweet_count", "quote_count"] # ] # .fillna(0) # .astype(int) # ) # top_retweeted = term_stats.nlargest(10, "retweet_count").sort_values( # "retweet_count", ascending=False # ) # top_quoted = term_stats.nlargest(10, "quote_count").sort_values( # "quote_count", ascending=False # ) # st.table( # pd.DataFrame( # map( # lambda t: [t["text"], t["quote_count"], t["retweet_count"]], # lookup_parsed_tweet_data( # filtered_by_crawled_term.sample(n=10).index.values # ), # ), # columns=["text", "quote_count", "retweet_count"], # ) # ) st.subheader("10 most retweeted tweets for '{}'".format(selected_crawled_term)) # st.table( # pd.DataFrame( # map( # lambda t: [t["text"], t["quote_count"], t["retweet_count"]], # lookup_parsed_tweet_data(top_retweeted.index.values), # ), # columns=["text", "quote_count", "retweet_count"], # ) # ) st.subheader("10 most quoted tweets for '{}'".format(selected_crawled_term))
def process( input_CD_image, input_OF_image, params, model_params, num_of_heatmaps=27, # 26 + background --> 27 num_of_OFFs=52, # 26 pairs: 52 layers in total num_of_OFFs_normal=27): # number of pairs (26) + 1 --> 27 print(input_CD_image) print(input_OF_image) oriImgCD = cv2.imread(input_CD_image) # B,G,R order oriImgOF = cv2.imread(input_OF_image) # B,G,R order rawDepth = read_pgm(input_CD_image.replace("mc_blob.png", "depth.pgm"), byteorder='>') heatmap_avg = np.zeros( (oriImgCD.shape[0], oriImgCD.shape[1], num_of_heatmaps)) off_avg = np.zeros((oriImgOF.shape[0], oriImgOF.shape[1], num_of_OFFs)) for m in range(len(multiplier)): scale = multiplier[m] image_CD_ToTest = cv2.resize(oriImgCD, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC) image_CD_ToTest_padded, pad = util.padRightDownCorner( image_CD_ToTest, model_params['stride'], model_params['padValue']) input_img_CD = np.transpose( np.float32(image_CD_ToTest_padded[:, :, :, np.newaxis]), (3, 0, 1, 2)) # required shape (1, width, height, channels) image_OF_ToTest = cv2.resize(oriImgCD, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC) image_OF_ToTest_padded, pad = util.padRightDownCorner( image_OF_ToTest, model_params['stride'], model_params['padValue']) input_img_OF = np.transpose( np.float32(image_OF_ToTest_padded[:, :, :, np.newaxis]), (3, 0, 1, 2)) # required shape (1, width, height, channels) output_blobs = model.predict([input_img_OF, input_img_CD]) # extract outputs, resize, and remove padding # The CD input is used for having the required parameters since they are the same for both inputs heatmap = np.squeeze(output_blobs[1]) # output 1 is heatmaps heatmap = cv2.resize(heatmap, (0, 0), fx=model_params['stride'], fy=model_params['stride'], interpolation=cv2.INTER_CUBIC) heatmap = heatmap[:image_CD_ToTest_padded.shape[0] - pad[2], :image_CD_ToTest_padded.shape[1] - pad[3], :] heatmap = cv2.resize(heatmap, (oriImgCD.shape[1], oriImgCD.shape[0]), interpolation=cv2.INTER_CUBIC) off = np.squeeze(output_blobs[0]) # output 0 is OFFs off = cv2.resize(off, (0, 0), fx=model_params['stride'], fy=model_params['stride'], interpolation=cv2.INTER_CUBIC) off = off[:image_CD_ToTest_padded.shape[0] - pad[2], :image_CD_ToTest_padded.shape[1] - pad[3], :] off = cv2.resize(off, (oriImgCD.shape[1], oriImgCD.shape[0]), interpolation=cv2.INTER_CUBIC) heatmap_avg = heatmap_avg + heatmap / len(multiplier) off_avg = off_avg + off / len(multiplier) all_peaks = [] peak_counter = 0 for part in range(num_of_heatmaps): map_ori = heatmap_avg[:, :, part] map = gaussian_filter(map_ori, sigma=3) map_left = np.zeros(map.shape) map_left[1:, :] = map[:-1, :] map_right = np.zeros(map.shape) map_right[:-1, :] = map[1:, :] map_up = np.zeros(map.shape) map_up[:, 1:] = map[:, :-1] map_down = np.zeros(map.shape) map_down[:, :-1] = map[:, 1:] peaks_binary = np.logical_and.reduce( (map >= map_left, map >= map_right, map >= map_up, map >= map_down, map > params['thre1'])) peaks = list( zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0])) # note reverse peaks_with_score = [x + (map_ori[x[1], x[0]], ) for x in peaks] id = range(peak_counter, peak_counter + len(peaks)) peaks_with_score_and_id = [ peaks_with_score[i] + (id[i], ) for i in range(len(id)) ] all_peaks.append(peaks_with_score_and_id) peak_counter += len(peaks) connection_all = [] special_k = [] mid_num = 4 for k in range(len(mapIdx)): score_mid = off_avg[:, :, [x - num_of_OFFs_normal for x in mapIdx[k]]] candA = all_peaks[limbSeq[k][0] - 1] candB = all_peaks[limbSeq[k][1] - 1] nA = len(candA) nB = len(candB) indexA, indexB = limbSeq[k] if (nA != 0 and nB != 0): connection_candidate = [] for i in range(nA): for j in range(nB): vec = np.subtract(candB[j][:2], candA[i][:2]) norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1]) # failure case when 2 body parts overlaps if norm == 0: continue vec = np.divide(vec, norm) startend = list(zip(np.linspace(candA[i][0], candB[j][0], num=mid_num), \ np.linspace(candA[i][1], candB[j][1], num=mid_num))) vec_x = np.array( [score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] \ for I in range(len(startend))]) vec_y = np.array( [score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] \ for I in range(len(startend))]) score_midpts = np.multiply(vec_x, vec[0]) + np.multiply( vec_y, vec[1]) score_with_dist_prior = sum( score_midpts) / len(score_midpts) + min( 0.5 * oriImgCD.shape[0] / norm - 1, 0) criterion1 = len( np.nonzero(score_midpts > params['thre2']) [0]) > 0.8 * len(score_midpts) criterion2 = score_with_dist_prior > 0 if criterion1 and criterion2: connection_candidate.append([ i, j, 3 * score_with_dist_prior, 3 * score_with_dist_prior + candA[i][2] + candB[j][2] ]) connection_candidate = sorted(connection_candidate, key=lambda x: x[2], reverse=True) connection = np.zeros((0, 5)) for c in range(len(connection_candidate)): i, j, s = connection_candidate[c][0:3] if (i not in connection[:, 3] and j not in connection[:, 4]): connection = np.vstack( [connection, [candA[i][3], candB[j][3], s, i, j]]) if (len(connection) >= min(nA, nB)): break connection_all.append(connection) else: special_k.append(k) connection_all.append([]) # last number in each row is the total parts number of that person # the second last number in each row is the score of the overall configuration subset = -1 * np.ones((0, num_of_OFFs_normal + 1)) candidate = np.array([item for sublist in all_peaks for item in sublist]) for k in range(len(mapIdx)): if k not in special_k: partAs = connection_all[k][:, 0] partBs = connection_all[k][:, 1] indexA, indexB = np.array(limbSeq[k]) - 1 for i in range(len(connection_all[k])): # = 1:size(temp,1) found = 0 subset_idx = [-1, -1] for j in range(len(subset)): # 1:size(subset,1): if subset[j][indexA] == partAs[i] or subset[j][ indexB] == partBs[i]: subset_idx[found] = j found += 1 if found == 1: j = subset_idx[0] if (subset[j][indexB] != partBs[i]): subset[j][indexB] = partBs[i] subset[j][-1] += 1 subset[j][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2] elif found == 2: # if found 2 and disjoint, merge them j1, j2 = subset_idx membership = ((subset[j1] >= 0).astype(int) + (subset[j2] >= 0).astype(int))[:-2] if len(np.nonzero(membership == 2)[0]) == 0: # merge subset[j1][:-2] += (subset[j2][:-2] + 1) subset[j1][-2:] += subset[j2][-2:] subset[j1][-2] += connection_all[k][i][2] subset = np.delete(subset, j2, 0) else: # as like found == 1 subset[j1][indexB] = partBs[i] subset[j1][-1] += 1 subset[j1][-2] += candidate[ partBs[i].astype(int), 2] + connection_all[k][i][2] # if find no partA in the subset, create a new subset elif not found and k < num_of_heatmaps - 1: row = -1 * np.ones(num_of_OFFs_normal + 1) row[indexA] = partAs[i] row[indexB] = partBs[i] row[-1] = 2 row[-2] = sum(candidate[connection_all[k][i, :2].astype(int), 2]) + \ connection_all[k][i][2] subset = np.vstack([subset, row]) # delete some rows of subset which has few parts occur deleteIdx = [] for i in range(len(subset)): if subset[i][-1] < 4 or subset[i][-2] / subset[i][-1] < 0.4: deleteIdx.append(i) # subset = np.delete(subset, deleteIdx, axis=0) canvas = oriImgCD # B,G,R order all_peaks_max_index = np.zeros(num_of_heatmaps - 1, dtype=int) for i in range(num_of_heatmaps - 1): if len(all_peaks[i]) > 0: max_value = 0 for j in range(len(all_peaks[i])): if max_value < all_peaks[i][j][2]: max_value = all_peaks[i][j][2] all_peaks_max_index[i] = j deleteIdReflector = [] for i in range(num_of_heatmaps - 1): if len(all_peaks[i]) > 0: for j in range(num_of_heatmaps - 1): if i != j and len(all_peaks[j]) > 0: vec = np.subtract(all_peaks[i][all_peaks_max_index[i]][:2], all_peaks[j][all_peaks_max_index[j]][:2]) norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1]) if norm < 6: if (all_peaks[i][all_peaks_max_index[i]][2] > all_peaks[j][all_peaks_max_index[j]][2]): deleteIdReflector.append(j) else: deleteIdReflector.append(i) for i in range(len(deleteIdReflector)): all_peaks[deleteIdReflector[i]] = [] file_3d.write(str(frameIndex) + '\n') file_3d.write('NONE { }\n') detected_contour_depth_values = [] detected_contour_coordinates = [] detected_rectangles = [] detected_ids = [] merged_sets = [] for i in range(num_of_heatmaps - 1): if len(all_peaks[i]) > 0 and all_peaks[i] != []: # cv2.circle(canvas, all_peaks[i][all_peaks_max_index[i]][0:2], 4, colors[i], thickness=4) # Copy the thresholded image. im_floodfill = canvas.copy() # Mask used to flood filling. # Notice the size needs to be 2 pixels than the image. h, w = canvas.shape[:2] mask = np.zeros((h + 2, w + 2), np.uint8) # Floodfill from point (0, 0) flood_return = cv2.floodFill( im_floodfill, mask, all_peaks[i][all_peaks_max_index[i]][0:2], [255, 255, 255]) for j in range(len(detected_rectangles)): if (detected_ids[j] != i): if isMergedRegion(detected_rectangles[j], flood_return[3]): # if () # del detected_rectangles[j] # break merged_sets.append([i, detected_ids[j]]) # cv2.imshow("image", flood_return[1]) # cv2.waitKey(0) detected_ids.append(i) detected_rectangles.append(flood_return[3]) # Invert floodfilled image im_floodfill_inv = cv2.bitwise_not(im_floodfill) # Combine the two images to get the foreground. fill_image = canvas | im_floodfill_inv mask_gray = cv2.cvtColor(fill_image, cv2.COLOR_BGR2GRAY) # mask_gray = cv2.normalize(src=mask_gray, dst=None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_8UC1) im2, contours, hierarchy = cv2.findContours( mask_gray, cv2.RETR_CCOMP, cv2.CHAIN_APPROX_NONE) if len(contours) > 1: values = np.zeros((contours[1].shape[0]), dtype=float) for p in range(contours[1].shape[0]): depth_value = rawDepth[contours[1][p][0][1]][contours[1][p] [0][0]] # print(str(i) + " " + str(depth_value)) if depth_value > 1000 and depth_value < 2500 and contours[ 1][p][0][1] > 10 and contours[1][p][0][ 1] < 400 and contours[1][p][0][ 0] > 40 and contours[1][p][0][0] < 400: values[p] = depth_value else: values[p] = 0 if (values[p] > 0): detected_contour_coordinates.append([ contours[1][p][0][1], contours[1][p][0][0], values[p] ]) detected_contour_coordinates.sort(key=lambda x: x[ 2]) # = np.sort(detected_contour_coordinates, axis=2) values[::-1].sort() values = [x for x in values if x > 0] if np.median(values) != np.nan and np.median(values) > 0: detected_contour_depth_values.append(values) else: del detected_ids[-1] del detected_rectangles[-1] else: del detected_ids[-1] del detected_rectangles[-1] ## Clustering temp_detected = [x for x in detected_ids] for i in range(len(merged_sets)): if (merged_sets[i][0] in temp_detected) and (merged_sets[i][1] in temp_detected): kmeans = KMeans(n_clusters=2, random_state=0).fit(detected_contour_coordinates) detected_contour_depth_values[detected_ids.index( merged_sets[i][0])] = [ x[2] for x in detected_contour_coordinates if kmeans.labels_[detected_contour_coordinates.index(x)] == 0 ] detected_contour_depth_values[detected_ids.index( merged_sets[i][1])] = [ x[2] for x in detected_contour_coordinates if kmeans.labels_[detected_contour_coordinates.index(x)] == 1 ] temp_detected.remove(merged_sets[i][0]) temp_detected.remove(merged_sets[i][1]) # spatial mapping from depthmap to 3D world using the intrinsic and extrinsic camera matrices # the extracted 3D points are stored in text files detected_index = 0 for i in range(num_of_heatmaps - 1): if (i in detected_ids): depth = np.median(detected_contour_depth_values[detected_index]) if "D4" in input_image: vec3 = [ KRT4_x[all_peaks[i][all_peaks_max_index[i]][0]][int( all_peaks[i][all_peaks_max_index[i]][1])] * depth, KRT4_y[all_peaks[i][all_peaks_max_index[i]][0]][int( all_peaks[i][all_peaks_max_index[i]][1])] * depth, depth, 1000.0 ] vec3 = np.true_divide(vec3, 1000.0) final_vec3 = np.matmul(Ext4, vec3, out=None) file_3d.write(reflectors[i + 1] + ' { ' + str(final_vec3[0]) + ' ' + str(final_vec3[1]) + ' ' + str(final_vec3[2]) + ' ' + str(final_vec3[0]) + ' ' + str(final_vec3[1]) + ' ' + str(final_vec3[2]) + ' }\n') elif "D6" in input_image: vec3 = [ KRT6_x[all_peaks[i][all_peaks_max_index[i]][0]][int( all_peaks[i][all_peaks_max_index[i]][1])] * depth, KRT6_y[all_peaks[i][all_peaks_max_index[i]][0]][int( all_peaks[i][all_peaks_max_index[i]][1])] * depth, depth, 1000.0 ] vec3 = np.true_divide(vec3, 1000.0) final_vec3 = np.matmul(Ext6, vec3, out=None) file_3d.write(reflectors[i + 1] + ' { ' + str(final_vec3[0]) + ' ' + str(final_vec3[1]) + ' ' + str(final_vec3[2]) + ' ' + str(final_vec3[0]) + ' ' + str(final_vec3[1]) + ' ' + str(final_vec3[2]) + ' }\n') elif "D8" in input_image: vec3 = [ KRT8_x[all_peaks[i][all_peaks_max_index[i]][0]][int( all_peaks[i][all_peaks_max_index[i]][1])] * depth, KRT8_y[all_peaks[i][all_peaks_max_index[i]][0]][int( all_peaks[i][all_peaks_max_index[i]][1])] * depth, depth, 1000.0 ] vec3 = np.true_divide(vec3, 1000.0) final_vec3 = np.matmul(Ext8, vec3, out=None) file_3d.write(reflectors[i + 1] + ' { ' + str(final_vec3[0]) + ' ' + str(final_vec3[1]) + ' ' + str(final_vec3[2]) + ' ' + str(final_vec3[0]) + ' ' + str(final_vec3[1]) + ' ' + str(final_vec3[2]) + ' }\n') detected_index += 1 else: file_3d.write(reflectors[i + 1] + ' { }\n') else: file_3d.write(reflectors[i + 1] + ' { }\n') file_3d.write('NOSE { }\n') stickwidth = 4 # for i in range(num_of_heatmaps): # # for n in range(len(subset)): # # index = subset[n][np.array(limbSeq[i]) - 1] # # if -1 in index: # # continue # for n in range(len(connection_all)): # if len(connection_all[n]): # partAs = connection_all[n][:, 0] # partBs = connection_all[n][:, 1] # indexA, indexB = np.array(limbSeq[n]) - 1 # cur_canvas = canvas.copy() # Y = candidate[indexA, 0] # X = candidate[indexB, 1] # mX = np.mean(X) # mY = np.mean(Y) # length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5 # angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1])) # polygon = cv2.ellipse2Poly((int(mY), int(mX)), (int(length / 2), stickwidth), int(angle), 0, # 360, 1) # cv2.fillConvexPoly(cur_canvas, polygon, colors[i]) # canvas = cv2.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0) #### PAPER FIGURE # at this stage, the estimates are overlayed on the depth images - the depth images occur by grayscaling the colorized images *NOT the raw depth canvas = cv2.cvtColor(canvas, cv2.COLOR_BGR2GRAY) canvas = cv2.cvtColor(canvas, cv2.COLOR_GRAY2BGR) for i in range(len(limbSeq)): if len(all_peaks[limbSeq[i][0] - 1]) > 0 and len( all_peaks[limbSeq[i][1] - 1]) > 0: cur_canvas = canvas.copy() Y = all_peaks[limbSeq[i][0] - 1][all_peaks_max_index[limbSeq[i][0] - 1]] X = all_peaks[limbSeq[i][1] - 1][all_peaks_max_index[limbSeq[i][1] - 1]] mX = (X[1] + Y[1]) / 2 mY = (X[0] + Y[0]) / 2 length = ((X[0] - Y[0])**2 + (X[1] - Y[1])**2)**0.5 angle = math.degrees(math.atan2(X[1] - Y[1], X[0] - Y[0])) polygon = cv2.ellipse2Poly( (int(mY), int(mX)), (int(length / 2), stickwidth), int(angle), 0, 360, 1) cv2.fillConvexPoly(cur_canvas, polygon, colors[limbSeq[i][0] - 1]) canvas = cv2.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0) for i in range(num_of_heatmaps - 1): if len(all_peaks[i]) > 0: cv2.putText(canvas, str(i + 1), all_peaks[i][all_peaks_max_index[i]][0:2], cv2.FONT_HERSHEY_SIMPLEX, 1.0, colors[i], thickness=2, lineType=cv2.LINE_AA) cv2.imwrite(input_image_CD.replace(".png", "_processed.jpg"), canvas) return canvas