def test_sampling(): Itemset.set_items(range(4)) Itemset.clear_db() db0 = set([Transaction([0]), ]) db1 = set([Transaction([1,2]), Transaction([1,3]), Transaction([1]), ]) Itemset.set_db(0, db0) Itemset.set_db(1, db1) n = 3000 nsamp, samp = sample(n, [db0], db1, {}, mode=3) print(nsamp) for s,cnt in samp.items(): if frozenset([1]) == s: assert (3/7-0.05)*nsamp < cnt < (3/7+0.05)*nsamp if frozenset([1,2]) == s: assert (1/7-0.05)*nsamp < cnt < (1/7+0.05)*nsamp if frozenset([1,3]) == s: assert (1/7-0.05)*nsamp < cnt < (1/7+0.05)*nsamp if frozenset([2]) == s: assert (1/7-0.05)*nsamp < cnt < (1/7+0.05)*nsamp if frozenset([3]) == s: assert (1/7-0.05)*nsamp < cnt < (1/7+0.05)*nsamp sel = [Rule([1,3], 1)] covered = set([t for r in sel for t in r.trans()]) nsamp, samp = sample(n, [db0], db1, covered, mode=3) print(nsamp) for s,cnt in samp.items(): if frozenset([1,2]) == s: assert (0.5-0.1)*nsamp < cnt < (0.5+0.1)*nsamp if frozenset([2]) == s: assert (0.5-0.1)*nsamp < cnt < (0.5+0.1)*nsamp nsamp, samp = sample(n, [db0], db1, covered, mode=2) print(nsamp) for s,cnt in samp.items(): if frozenset([1]) == s: assert (1/3-0.1)*nsamp < cnt < (1/3+0.1)*nsamp if frozenset([1,2]) == s: assert (1/3-0.1)*nsamp < cnt < (1/3+0.1)*nsamp if frozenset([2]) == s: assert (1/3-0.1)*nsamp < cnt < (1/3+0.1)*nsamp sel = [Rule([1], 1), Rule([1,3], 1)] covered = [t for r in sel for t in r.trans()] assert len(covered) == 4 covered = set([t for r in sel for t in r.trans()]) assert len(covered) == 3
def generate_sample(input_model, weights, quantized_model, scaling_mode, calibration_algo, conv_algo, iterations=10, enable_1st_conv=False, sampling_single=False): (blobs, params, top_blobs_map, bottom_blobs_map, conv_top_blob_layer_map, conv_bottom_blob_layer_map, winograd_bottoms, winograd_convolutions) = sampling.sample(input_model, weights, conv_algo, iterations, enable_1st_conv, sampling_single) (inputs_max, outputs_max, inputs_min) = sampling.calibrate_activations( blobs, conv_top_blob_layer_map, conv_bottom_blob_layer_map, winograd_bottoms, calibration_algo, "SINGLE", conv_algo) params_max = sampling.calibrate_parameters(params, winograd_convolutions, "DIRECT", scaling_mode.upper(), conv_algo) generate_sample_impl(input_model, quantized_model, inputs_max, outputs_max, inputs_min, params_max, enable_1st_conv) return (top_blobs_map, bottom_blobs_map)
def buildImageDataset(data_folder_out, modality, seed): x_train_filenames = [] filenames = [None] * len(modality) nums = np.zeros(len(modality)) for i, m in enumerate(modality): filenames[i], _ = getTrainNLabelNames(data_folder_out[i], m, ext='*.tfrecords') nums[i] = len(filenames[i]) x_train_filenames += filenames[i] #shuffle random.shuffle(x_train_filenames) print("Number of images obtained for training and validation: " + str(nums)) """ Sample dataset """ np.random.seed(seed) nums = np.max(nums) - nums for i, _ in enumerate(modality): index = sample(list(range(len(filenames[i]))), nums[i]) x_train_filenames += [filenames[i][j] for j in index] #x_train_filenames = bootstrapping(x_train_filenames) return x_train_filenames
def get_train_data(size=10, start=-5, stop=5): xs, ys = sample(gauss_kernel, start=start, stop=stop, n=size, num_sample=1, random_x=True) return xs, ys[0]
def result(self, rowclicked, colclicked): node_type = self.project_detail.iloc[rowclicked]['模块类型'] node_name = self.project_detail.iloc[rowclicked]['模块名字'] node_save_path = self.project_detail.iloc[rowclicked]['保存地址'] try: fr = open(node_save_path, 'rb') node_info = pickle.load(fr) fr.close() flag_error = 0 except Exception as e: flag_error = 1 tk.messagebox.showwarning('错误', e) if flag_error != 1: try: if self.root2.state() == 'normal': tk.messagebox.showwarning('错误', "请先处理当前打开窗口") except: self.root2 = Toplevel(self.root) self.root2.title(node_name) if node_type == 'SPLIT': new_node = spliting(self.root2, self.project_detail) new_node.load_node(node_data=node_info, ac='result') elif node_type == 'DATA': new_node = inputdata(self.root2, self.project_detail) new_node.load(node_info) new_node.variable_seting_ui() elif node_type == 'SAMPLE': new_node = sample(self.root2, self.project_detail) new_node.load_node(node_data=node_info, ac='result') elif node_type == 'IGN': new_node = IGN(self.root2, self.project_detail) new_node.load_node(node_info, ac='result') new_node.result() elif node_type == 'SCR': new_node = model(self.root2, self.project_detail) new_node.import_node(node_info, ac='result') new_node.reult_show_only(self.root2) elif node_type == 'Scoring': new_node = scoreing(self.root2, self.project_detail) new_node.load_node(node_info, ac='result') self.root.wait_window(self.root2) try: tt = [{ '模块类型': new_node.node_setting['node_type'], '模块名字': new_node.node_setting['node_name'], '引用模块': new_node.node_setting['use_node'], '保存地址': new_node.node_setting['node_save_path'], '创建时间': new_node.node_setting['time'], '状态': 'Good' }] mm = pd.DataFrame(tt) print(mm) self.project_detail = self.project_detail[ self.project_detail['模块名字'] != node_name] self.project_detail = self.project_detail.append(mm) self.refresh_df(self.root, self.project_detail) except: pass
def test_api(self, k): """Test basic API for sample function.""" data = list(range(100)) data_in = data.copy() sampled_elements = sample(data) assert data == data_in # Function does not change the data assert isinstance(sampled_elements, list) assert len(sampled_elements) == 1 assert set(sampled_elements).issubset(set(data))
def sample_from_each_label(self, labels_samp, nsamp, covered, mode): # Sample rules from each label samps = [] for label in labels_samp: db0 = [db for l, db in Itemset.dbs.items() if l != label] nsamp_, samp = sample(nsamp, db0, Itemset.dbs[label], covered, mode=mode) if nsamp_ == 0: continue samp = [Rule(s, label) for s in list(samp)] # Very time-consuming samps = samps + samp return set(samps)
def gen_sentence(chain): start_list = [] for pair in list(chain.keys()): if pair[0]=="[START]": start_list.append(pair) word_tuple = start_list[randint(0,len(start_list)-1)] return_sentence = '' for word in word_tuple[1:]: return_sentence += ' ' + word while word_tuple[-1]!="[STOP]": new_word = sampling.sample(chain[word_tuple]) return_sentence += ' ' + new_word word_list = list(word_tuple[1:]) word_list.append(new_word) word_tuple = tuple(word_list) return return_sentence[:-7] + '.'
def create_probabilistic_graph(data, num_samples, k): nodes, polygons = sample(data, num_samples) print("Creating KDTree ...") t0 = time.time() g = nx.Graph() tree = KDTree(nodes) for n1 in nodes: # for each node connect try to connect to k nearest nodes p1 = Point(n1) idxs = tree.query([n1], k, return_distance=False)[0] for idx in idxs: n2 = nodes[idx] p2 = Point(n2) if p2 == p1: continue if can_connect(polygons, n1, n2): g.add_edge((n1[0], n1[1]), (n2[0], n2[1]), weight=1) print("Time taken {0} seconds ...".format(time.time() - t0)) return g
3] # The alpha parameter for dirichlet prior of hidden word ### Start the generator part # First, generate the starting position of the magic word for all sequences uniformly position = [0] * K for i in xrange(K): position[i] = randint(0, N - w + 1) # Generate the background letters for all sequences cat_b = dirichlet(alpha_b) sequences = [] for i in xrange(K): seq = [] for j in xrange(N): seq += [sample(alphabet, cat_b)] sequences += [seq] # Generate the magic words theta = dirichlet(alpha_w, w) for i in xrange(K): start_pos = position[i] for j in xrange(w): sequences[i][start_pos + j] = sample(alphabet, theta[j]) if WRITE_TO_FILE: ### Store the generated sequence to file to be read later filename = 'a.seq' f = open(filename, 'w')
from torchvision.utils import save_image from tqdm import tqdm from model import Generator from sampling import sample, get_cycling_grid_same, n_classes device = 'cuda' model = Generator(n_feat=36, codes_dim=24, n_classes=n_classes).to(device) model.load_state_dict(torch.load('generator.pth')) height = 8 width = 15 cycles = 8 frames_per_half_cycle = 24*4 all_zs = get_cycling_grid_same(height, width, cycles, frames_per_half_cycle) for f_idx in tqdm(range(all_zs.size(2))): z_batch = all_zs[:, :, f_idx] z_batch = z_batch.view(-1, 120) aux_labels = np.arange(120) images = sample(model, z_batch, aux_labels=aux_labels) frame = torch.empty((3, height * 64, width * 64), device=device) for h in range(height): for w in range(width): frame[:, h * 64:(h + 1) * 64, w * 64:(w + 1) * 64] = images[h * width + w] save_image(frame, 'breed_frames/{}.png'.format(f_idx))
def generate_sample(input_model, weights, quantized_model, scaling_mode, calibration_algo, conv_algo, iterations=10, enable_1st_conv=False): (blobs, params, top_blobs_map, bottom_blobs_map, conv_top_blob_layer_map, conv_bottom_blob_layer_map, winograd_bottoms, winograd_convolutions) = sampling.sample(input_model, weights, conv_algo, iterations, enable_1st_conv) (inputs_max, outputs_max, inputs_min) = sampling.calibrate_activations(blobs, conv_top_blob_layer_map, conv_bottom_blob_layer_map, winograd_bottoms, calibration_algo, "SINGLE", conv_algo) params_max = sampling.calibrate_parameters(params, winograd_convolutions, "DIRECT", scaling_mode.upper(), conv_algo) generate_sample_impl(input_model, quantized_model, inputs_max, outputs_max, inputs_min, params_max, enable_1st_conv) return (top_blobs_map, bottom_blobs_map)
def sample(): result = sampling.sample(list(distributions.values())) return json.dumps({ 'restaurant': result['restaurant'].__dict__, 'probabilities': result['probabilities'] })
pcol = pcol * pm A[i] = pback * pcol return A # First, initialize the start position randomly pos = [randint(0, N - w + 1) for x in range(K)] orig_pos = pos[:] MAX_ITER = 100 p = [0] * (N - w) b = [0] * MAX_ITER for it in range(MAX_ITER): for i in range(K): p = full_conditional(sequences, pos, i) total = sum(p) p = [x / total for x in p] # Sample new position #print 'p', p pos[i] = sample(list(range(N - w)), p) b[it] = max(p) # Happy printing print('start pos', orig_pos) print('last pos', pos) print('true pos', position) plt.plot(b)
def func_menu(self, func): try: if self.root2.state() == 'normal': tk.messagebox.showwarning('错误', "请先处理当前打开窗口") except: self.root2 = Toplevel(self.root) if func == 'importdata': self.root2.title('导入数据集') new_node = inputdata(self.root2, self.project_detail) new_node.newdatainput() tip = '导入数据集' elif func == 'split': self.root2.title('数据集分区') new_node = spliting(self.root2, self.project_detail) new_node.ui_start() tip = '数据集分区' elif func == 'sampling': self.root2.title('数据集抽样') new_node = sample(self.root2, self.project_detail) new_node.ui_start() tip = '数据集抽样' elif func == 'IGN': self.root2.title('交互式分组') new_node = IGN(self.root2, self.project_detail) new_node.Start_UI() new_node.adjustsetting() tip = '交互式分组' elif func == 'model': self.root2.title('评分卡模型') new_node = model(self.root2, self.project_detail) new_node.Start_UI() new_node.adjustsetting() tip = '评分卡模型' elif func == 'Scoring': self.root2.title('数据集打分') new_node = scoreing(self.root2, self.project_detail) new_node.Start_UI() new_node.adjustsetting() tip = '数据集打分' elif func == 'load_node': new_node = import_node(self.root2, self.project_detail) tip = '导入模块' self.root.wait_window(self.root2) if new_node.save != 'Y': tk.messagebox.showwarning('错误', "%s未完成" % tip) else: try: print(new_node.save) tt = [{ '模块类型': new_node.node_setting['node_type'], '模块名字': new_node.node_setting['node_name'], '引用模块': new_node.node_setting['use_node'], '保存地址': new_node.node_setting['node_save_path'], '创建时间': new_node.node_setting['time'], '状态': 'Good' }] mm = pd.DataFrame(tt) print(mm) self.project_detail = self.project_detail.append(mm) # del new_node self.refresh_df(self.root, self.project_detail) except Exception as e: tk.messagebox.showwarning('错误', "%s未完成%s" % (tip, e))
for i in xrange(K): # We sample the next position of magic word in this sequence # Therefore, we exclude this sequence from model calculation seq_minus = sequences[:] del seq_minus[i] pos_minus = pos[:] del pos_minus[i] q, p = compute_model(seq_minus, pos_minus, alphabet, w) # We try for every possible position of magic word in sequence i and # calculate the probability of it being as background or magic word # The probability for magic word is calculated by multiplying the probability # for each character in each position qx = [1] * (N - w) px = [1] * (N - w) for j in xrange(N - w): for k in xrange(w): c = sequences[i][j + k] qx[j] = qx[j] * q[c][k] px[j] = px[j] * p[c] # Compute the ratio between word and background, the pythonic way Aj = [x / y for (x, y) in zip(qx, px)] norm_c = sum(Aj) Aj = map(lambda x: x / norm_c, Aj) # Sampling new position with regards to probability distribution Aj pos[i] = sample(range(N - w), Aj) # Happy printing print 'new pos', pos
# We pick the sequence, well, in sequence starting from index 0 for i in xrange(K): # We sample the next position of magic word in this sequence # Therefore, we exclude this sequence from model calculation seq_minus = sequences[:]; del seq_minus[i] pos_minus = pos[:]; del pos_minus[i] q, p = compute_model(seq_minus, pos_minus, alphabet, w) # We try for every possible position of magic word in sequence i and # calculate the probability of it being as background or magic word # The probability for magic word is calculated by multiplying the probability # for each character in each position qx = [1]*(N-w) px = [1]*(N-w) for j in xrange(N-w): for k in xrange(w): c = sequences[i][j+k] qx[j] = qx[j] * q[c][k] px[j] = px[j] * p[c] # Compute the ratio between word and background, the pythonic way Aj = [x/y for (x,y) in zip(qx, px)] norm_c = sum(Aj) Aj = map(lambda x: x/norm_c, Aj) # Sampling new position with regards to probability distribution Aj pos[i] = sample(range(N-w), Aj) # Happy printing print 'new pos', pos
from model import Generator from sampling import sample, get_cycling_grid, n_classes device = 'cuda' class_id = 0 model = Generator(n_feat=36, codes_dim=24, n_classes=n_classes).to(device) model.load_state_dict(torch.load('generator.pth')) height = 8 width = 15 cycles = 8 frames_per_half_cycle = 24 * 4 all_zs = get_cycling_grid(height, width, cycles, frames_per_half_cycle) for f_idx in tqdm(range(all_zs.size(2))): z_batch = all_zs[:, :, f_idx] z_batch = z_batch.view(-1, 120) aux_labels = np.arange(120) images = sample(model, z_batch, class_id=class_id) frame = torch.empty((3, height * 64, width * 64), device=device) for h in range(height): for w in range(width): frame[:, h * 64:(h + 1) * 64, w * 64:(w + 1) * 64] = images[h * width + w] save_image(frame, 'single_frames/{}.png'.format(f_idx))
### Start the generator part # First, generate the starting position of the magic word for all sequences uniformly position = [0]*K for i in xrange(K): position[i] = randint(0, N-w+1) # Generate the background letters for all sequences cat_b = dirichlet(alpha_b) sequences = [] for i in xrange(K): seq = [] for j in xrange(N): seq += [sample(alphabet, cat_b)] sequences += [seq] # Generate the magic words theta = dirichlet(alpha_w, w) for i in xrange(K): start_pos = position[i] for j in xrange(w): sequences[i][start_pos+j] = sample(alphabet, theta[j]) if WRITE_TO_FILE: ### Store the generated sequence to file to be read later filename = 'a.seq' f = open(filename, 'w')