def test_sampling():
Itemset.set_items(range(4))
Itemset.clear_db()
db0 = set([Transaction([0]),
])
db1 = set([Transaction([1,2]),
Transaction([1,3]),
Transaction([1]),
])
Itemset.set_db(0, db0)
Itemset.set_db(1, db1)
n = 3000
nsamp, samp = sample(n, [db0], db1, {}, mode=3)
print(nsamp)
for s,cnt in samp.items():
if frozenset([1]) == s:
assert (3/7-0.05)*nsamp < cnt < (3/7+0.05)*nsamp
if frozenset([1,2]) == s:
assert (1/7-0.05)*nsamp < cnt < (1/7+0.05)*nsamp
if frozenset([1,3]) == s:
assert (1/7-0.05)*nsamp < cnt < (1/7+0.05)*nsamp
if frozenset([2]) == s:
assert (1/7-0.05)*nsamp < cnt < (1/7+0.05)*nsamp
if frozenset([3]) == s:
assert (1/7-0.05)*nsamp < cnt < (1/7+0.05)*nsamp
sel = [Rule([1,3], 1)]
covered = set([t for r in sel for t in r.trans()])
nsamp, samp = sample(n, [db0], db1, covered, mode=3)
print(nsamp)
for s,cnt in samp.items():
if frozenset([1,2]) == s:
assert (0.5-0.1)*nsamp < cnt < (0.5+0.1)*nsamp
if frozenset([2]) == s:
assert (0.5-0.1)*nsamp < cnt < (0.5+0.1)*nsamp
nsamp, samp = sample(n, [db0], db1, covered, mode=2)
print(nsamp)
for s,cnt in samp.items():
if frozenset([1]) == s:
assert (1/3-0.1)*nsamp < cnt < (1/3+0.1)*nsamp
if frozenset([1,2]) == s:
assert (1/3-0.1)*nsamp < cnt < (1/3+0.1)*nsamp
if frozenset([2]) == s:
assert (1/3-0.1)*nsamp < cnt < (1/3+0.1)*nsamp
sel = [Rule([1], 1), Rule([1,3], 1)]
covered = [t for r in sel for t in r.trans()]
assert len(covered) == 4
covered = set([t for r in sel for t in r.trans()])
assert len(covered) == 3
def generate_sample(input_model,
weights,
quantized_model,
scaling_mode,
calibration_algo,
conv_algo,
iterations=10,
enable_1st_conv=False,
sampling_single=False):
(blobs, params, top_blobs_map, bottom_blobs_map, conv_top_blob_layer_map,
conv_bottom_blob_layer_map, winograd_bottoms,
winograd_convolutions) = sampling.sample(input_model, weights, conv_algo,
iterations, enable_1st_conv,
sampling_single)
(inputs_max, outputs_max, inputs_min) = sampling.calibrate_activations(
blobs, conv_top_blob_layer_map, conv_bottom_blob_layer_map,
winograd_bottoms, calibration_algo, "SINGLE", conv_algo)
params_max = sampling.calibrate_parameters(params,
winograd_convolutions, "DIRECT",
scaling_mode.upper(), conv_algo)
generate_sample_impl(input_model, quantized_model, inputs_max, outputs_max,
inputs_min, params_max, enable_1st_conv)
return (top_blobs_map, bottom_blobs_map)
def buildImageDataset(data_folder_out, modality, seed):
x_train_filenames = []
filenames = [None] * len(modality)
nums = np.zeros(len(modality))
for i, m in enumerate(modality):
filenames[i], _ = getTrainNLabelNames(data_folder_out[i],
m,
ext='*.tfrecords')
nums[i] = len(filenames[i])
x_train_filenames += filenames[i]
#shuffle
random.shuffle(x_train_filenames)
print("Number of images obtained for training and validation: " +
str(nums))
""" Sample dataset """
np.random.seed(seed)
nums = np.max(nums) - nums
for i, _ in enumerate(modality):
index = sample(list(range(len(filenames[i]))), nums[i])
x_train_filenames += [filenames[i][j] for j in index]
#x_train_filenames = bootstrapping(x_train_filenames)
return x_train_filenames
def get_train_data(size=10, start=-5, stop=5):
xs, ys = sample(gauss_kernel,
start=start,
stop=stop,
n=size,
num_sample=1,
random_x=True)
return xs, ys[0]
def result(self, rowclicked, colclicked):
node_type = self.project_detail.iloc[rowclicked]['模块类型']
node_name = self.project_detail.iloc[rowclicked]['模块名字']
node_save_path = self.project_detail.iloc[rowclicked]['保存地址']
try:
fr = open(node_save_path, 'rb')
node_info = pickle.load(fr)
fr.close()
flag_error = 0
except Exception as e:
flag_error = 1
tk.messagebox.showwarning('错误', e)
if flag_error != 1:
try:
if self.root2.state() == 'normal':
tk.messagebox.showwarning('错误', "请先处理当前打开窗口")
except:
self.root2 = Toplevel(self.root)
self.root2.title(node_name)
if node_type == 'SPLIT':
new_node = spliting(self.root2, self.project_detail)
new_node.load_node(node_data=node_info, ac='result')
elif node_type == 'DATA':
new_node = inputdata(self.root2, self.project_detail)
new_node.load(node_info)
new_node.variable_seting_ui()
elif node_type == 'SAMPLE':
new_node = sample(self.root2, self.project_detail)
new_node.load_node(node_data=node_info, ac='result')
elif node_type == 'IGN':
new_node = IGN(self.root2, self.project_detail)
new_node.load_node(node_info, ac='result')
new_node.result()
elif node_type == 'SCR':
new_node = model(self.root2, self.project_detail)
new_node.import_node(node_info, ac='result')
new_node.reult_show_only(self.root2)
elif node_type == 'Scoring':
new_node = scoreing(self.root2, self.project_detail)
new_node.load_node(node_info, ac='result')
self.root.wait_window(self.root2)
try:
tt = [{
'模块类型': new_node.node_setting['node_type'],
'模块名字': new_node.node_setting['node_name'],
'引用模块': new_node.node_setting['use_node'],
'保存地址': new_node.node_setting['node_save_path'],
'创建时间': new_node.node_setting['time'],
'状态': 'Good'
}]
mm = pd.DataFrame(tt)
print(mm)
self.project_detail = self.project_detail[
self.project_detail['模块名字'] != node_name]
self.project_detail = self.project_detail.append(mm)
self.refresh_df(self.root, self.project_detail)
except:
pass
def test_api(self, k):
"""Test basic API for sample function."""
data = list(range(100))
data_in = data.copy()
sampled_elements = sample(data)
assert data == data_in # Function does not change the data
assert isinstance(sampled_elements, list)
assert len(sampled_elements) == 1
assert set(sampled_elements).issubset(set(data))
def sample_from_each_label(self, labels_samp, nsamp, covered, mode):
# Sample rules from each label
samps = []
for label in labels_samp:
db0 = [db for l, db in Itemset.dbs.items() if l != label]
nsamp_, samp = sample(nsamp,
db0,
Itemset.dbs[label],
covered,
mode=mode)
if nsamp_ == 0:
continue
samp = [Rule(s, label) for s in list(samp)] # Very time-consuming
samps = samps + samp
return set(samps)
def gen_sentence(chain):
start_list = []
for pair in list(chain.keys()):
if pair[0]=="[START]":
start_list.append(pair)
word_tuple = start_list[randint(0,len(start_list)-1)]
return_sentence = ''
for word in word_tuple[1:]:
return_sentence += ' ' + word
while word_tuple[-1]!="[STOP]":
new_word = sampling.sample(chain[word_tuple])
return_sentence += ' ' + new_word
word_list = list(word_tuple[1:])
word_list.append(new_word)
word_tuple = tuple(word_list)
return return_sentence[:-7] + '.'
def create_probabilistic_graph(data, num_samples, k):
nodes, polygons = sample(data, num_samples)
print("Creating KDTree ...")
t0 = time.time()
g = nx.Graph()
tree = KDTree(nodes)
for n1 in nodes:
# for each node connect try to connect to k nearest nodes
p1 = Point(n1)
idxs = tree.query([n1], k, return_distance=False)[0]
for idx in idxs:
n2 = nodes[idx]
p2 = Point(n2)
if p2 == p1:
continue
if can_connect(polygons, n1, n2):
g.add_edge((n1[0], n1[1]), (n2[0], n2[1]), weight=1)
print("Time taken {0} seconds ...".format(time.time() - t0))
return g
3] # The alpha parameter for dirichlet prior of hidden word
### Start the generator part
# First, generate the starting position of the magic word for all sequences uniformly
position = [0] * K
for i in xrange(K):
position[i] = randint(0, N - w + 1)
# Generate the background letters for all sequences
cat_b = dirichlet(alpha_b)
sequences = []
for i in xrange(K):
seq = []
for j in xrange(N):
seq += [sample(alphabet, cat_b)]
sequences += [seq]
# Generate the magic words
theta = dirichlet(alpha_w, w)
for i in xrange(K):
start_pos = position[i]
for j in xrange(w):
sequences[i][start_pos + j] = sample(alphabet, theta[j])
if WRITE_TO_FILE:
### Store the generated sequence to file to be read later
filename = 'a.seq'
f = open(filename, 'w')
from torchvision.utils import save_image
from tqdm import tqdm
from model import Generator
from sampling import sample, get_cycling_grid_same, n_classes
device = 'cuda'
model = Generator(n_feat=36, codes_dim=24, n_classes=n_classes).to(device)
model.load_state_dict(torch.load('generator.pth'))
height = 8
width = 15
cycles = 8
frames_per_half_cycle = 24*4
all_zs = get_cycling_grid_same(height, width, cycles, frames_per_half_cycle)
for f_idx in tqdm(range(all_zs.size(2))):
z_batch = all_zs[:, :, f_idx]
z_batch = z_batch.view(-1, 120)
aux_labels = np.arange(120)
images = sample(model, z_batch, aux_labels=aux_labels)
frame = torch.empty((3, height * 64, width * 64), device=device)
for h in range(height):
for w in range(width):
frame[:, h * 64:(h + 1) * 64, w * 64:(w + 1) * 64] = images[h * width + w]
save_image(frame, 'breed_frames/{}.png'.format(f_idx))
def generate_sample(input_model, weights, quantized_model, scaling_mode, calibration_algo, conv_algo, iterations=10, enable_1st_conv=False):
(blobs, params, top_blobs_map, bottom_blobs_map, conv_top_blob_layer_map, conv_bottom_blob_layer_map, winograd_bottoms, winograd_convolutions) = sampling.sample(input_model, weights, conv_algo, iterations, enable_1st_conv)
(inputs_max, outputs_max, inputs_min) = sampling.calibrate_activations(blobs, conv_top_blob_layer_map, conv_bottom_blob_layer_map, winograd_bottoms, calibration_algo, "SINGLE", conv_algo)
params_max = sampling.calibrate_parameters(params, winograd_convolutions, "DIRECT", scaling_mode.upper(), conv_algo)
generate_sample_impl(input_model, quantized_model, inputs_max, outputs_max, inputs_min, params_max, enable_1st_conv)
return (top_blobs_map, bottom_blobs_map)
def sample():
result = sampling.sample(list(distributions.values()))
return json.dumps({
'restaurant': result['restaurant'].__dict__,
'probabilities': result['probabilities']
})
pcol = pcol * pm
A[i] = pback * pcol
return A
# First, initialize the start position randomly
pos = [randint(0, N - w + 1) for x in range(K)]
orig_pos = pos[:]
MAX_ITER = 100
p = [0] * (N - w)
b = [0] * MAX_ITER
for it in range(MAX_ITER):
for i in range(K):
p = full_conditional(sequences, pos, i)
total = sum(p)
p = [x / total for x in p]
# Sample new position
#print 'p', p
pos[i] = sample(list(range(N - w)), p)
b[it] = max(p)
# Happy printing
print('start pos', orig_pos)
print('last pos', pos)
print('true pos', position)
plt.plot(b)
def func_menu(self, func):
try:
if self.root2.state() == 'normal':
tk.messagebox.showwarning('错误', "请先处理当前打开窗口")
except:
self.root2 = Toplevel(self.root)
if func == 'importdata':
self.root2.title('导入数据集')
new_node = inputdata(self.root2, self.project_detail)
new_node.newdatainput()
tip = '导入数据集'
elif func == 'split':
self.root2.title('数据集分区')
new_node = spliting(self.root2, self.project_detail)
new_node.ui_start()
tip = '数据集分区'
elif func == 'sampling':
self.root2.title('数据集抽样')
new_node = sample(self.root2, self.project_detail)
new_node.ui_start()
tip = '数据集抽样'
elif func == 'IGN':
self.root2.title('交互式分组')
new_node = IGN(self.root2, self.project_detail)
new_node.Start_UI()
new_node.adjustsetting()
tip = '交互式分组'
elif func == 'model':
self.root2.title('评分卡模型')
new_node = model(self.root2, self.project_detail)
new_node.Start_UI()
new_node.adjustsetting()
tip = '评分卡模型'
elif func == 'Scoring':
self.root2.title('数据集打分')
new_node = scoreing(self.root2, self.project_detail)
new_node.Start_UI()
new_node.adjustsetting()
tip = '数据集打分'
elif func == 'load_node':
new_node = import_node(self.root2, self.project_detail)
tip = '导入模块'
self.root.wait_window(self.root2)
if new_node.save != 'Y':
tk.messagebox.showwarning('错误', "%s未完成" % tip)
else:
try:
print(new_node.save)
tt = [{
'模块类型': new_node.node_setting['node_type'],
'模块名字': new_node.node_setting['node_name'],
'引用模块': new_node.node_setting['use_node'],
'保存地址': new_node.node_setting['node_save_path'],
'创建时间': new_node.node_setting['time'],
'状态': 'Good'
}]
mm = pd.DataFrame(tt)
print(mm)
self.project_detail = self.project_detail.append(mm)
# del new_node
self.refresh_df(self.root, self.project_detail)
except Exception as e:
tk.messagebox.showwarning('错误', "%s未完成%s" % (tip, e))
for i in xrange(K):
# We sample the next position of magic word in this sequence
# Therefore, we exclude this sequence from model calculation
seq_minus = sequences[:]
del seq_minus[i]
pos_minus = pos[:]
del pos_minus[i]
q, p = compute_model(seq_minus, pos_minus, alphabet, w)
# We try for every possible position of magic word in sequence i and
# calculate the probability of it being as background or magic word
# The probability for magic word is calculated by multiplying the probability
# for each character in each position
qx = [1] * (N - w)
px = [1] * (N - w)
for j in xrange(N - w):
for k in xrange(w):
c = sequences[i][j + k]
qx[j] = qx[j] * q[c][k]
px[j] = px[j] * p[c]
# Compute the ratio between word and background, the pythonic way
Aj = [x / y for (x, y) in zip(qx, px)]
norm_c = sum(Aj)
Aj = map(lambda x: x / norm_c, Aj)
# Sampling new position with regards to probability distribution Aj
pos[i] = sample(range(N - w), Aj)
# Happy printing
print 'new pos', pos
# We pick the sequence, well, in sequence starting from index 0
for i in xrange(K):
# We sample the next position of magic word in this sequence
# Therefore, we exclude this sequence from model calculation
seq_minus = sequences[:]; del seq_minus[i]
pos_minus = pos[:]; del pos_minus[i]
q, p = compute_model(seq_minus, pos_minus, alphabet, w)
# We try for every possible position of magic word in sequence i and
# calculate the probability of it being as background or magic word
# The probability for magic word is calculated by multiplying the probability
# for each character in each position
qx = [1]*(N-w)
px = [1]*(N-w)
for j in xrange(N-w):
for k in xrange(w):
c = sequences[i][j+k]
qx[j] = qx[j] * q[c][k]
px[j] = px[j] * p[c]
# Compute the ratio between word and background, the pythonic way
Aj = [x/y for (x,y) in zip(qx, px)]
norm_c = sum(Aj)
Aj = map(lambda x: x/norm_c, Aj)
# Sampling new position with regards to probability distribution Aj
pos[i] = sample(range(N-w), Aj)
# Happy printing
print 'new pos', pos
from model import Generator
from sampling import sample, get_cycling_grid, n_classes
device = 'cuda'
class_id = 0
model = Generator(n_feat=36, codes_dim=24, n_classes=n_classes).to(device)
model.load_state_dict(torch.load('generator.pth'))
height = 8
width = 15
cycles = 8
frames_per_half_cycle = 24 * 4
all_zs = get_cycling_grid(height, width, cycles, frames_per_half_cycle)
for f_idx in tqdm(range(all_zs.size(2))):
z_batch = all_zs[:, :, f_idx]
z_batch = z_batch.view(-1, 120)
aux_labels = np.arange(120)
images = sample(model, z_batch, class_id=class_id)
frame = torch.empty((3, height * 64, width * 64), device=device)
for h in range(height):
for w in range(width):
frame[:, h * 64:(h + 1) * 64,
w * 64:(w + 1) * 64] = images[h * width + w]
save_image(frame, 'single_frames/{}.png'.format(f_idx))
### Start the generator part
# First, generate the starting position of the magic word for all sequences uniformly
position = [0]*K
for i in xrange(K):
position[i] = randint(0, N-w+1)
# Generate the background letters for all sequences
cat_b = dirichlet(alpha_b)
sequences = []
for i in xrange(K):
seq = []
for j in xrange(N):
seq += [sample(alphabet, cat_b)]
sequences += [seq]
# Generate the magic words
theta = dirichlet(alpha_w, w)
for i in xrange(K):
start_pos = position[i]
for j in xrange(w):
sequences[i][start_pos+j] = sample(alphabet, theta[j])
if WRITE_TO_FILE:
### Store the generated sequence to file to be read later
filename = 'a.seq'
f = open(filename, 'w')
