def get_evolver_and_sketch(source_image,
output_folder,
num_triangles,
save_frequency,
color_type,
start_from=None,
continue_run=False,
randomized=False):
im = Image.open(source_image)
im = im.convert(mode=color_type)
assert not (start_from and continue_run
), "you should only use start_from or continue_run, not both"
sketch = None
e = None
if continue_run:
# find the latest file in the folder
file_names = glob.glob(
os.path.join(output_folder, "intermediate_???.txt"))
best = -1
for filename in file_names:
num = int(filename[-7:-4])
if num > best:
best = num
if best >= 0:
filename = os.path.join(output_folder,
"intermediate_%03d.txt" % best)
module_logger.info(
"Restarting evolution based on found file 'intermediate_%03d.txt'",
best)
sketch = Sketch.read(filename)
e = Evolver(im,
output_folder,
num_triangles,
save_frequency,
color_type=color_type,
save_index=best + 1)
# Preferred is to load from the auto-save, but in case it died while saving the above will still work
filename = os.path.join(output_folder, "auto_save.txt")
if os.path.exists(filename):
sketch = Sketch.read(filename)
if sketch is None:
utils.clean_dir(output_folder)
if start_from:
sketch = Sketch.read(start_from)
else:
if randomized:
seeder = RandomSeeder(im, output_folder, num_triangles,
color_type)
else:
seeder = Seeder(im, output_folder, num_triangles, color_type)
sketch = seeder.run()
e = Evolver(im,
output_folder,
num_triangles,
save_frequency,
color_type=color_type)
return e, sketch
def POST(self, name):
code = web.data().decode('utf-8')
write_file(name, code)
try:
s = Sketch()
s.render(code)
return s.tostring()
except Exception as e:
web.ctx.status = "403 Bad Input"
web.header("content-type", "text/plain")
return str(e)
def two_pass(self):
X = square_tensor_gen(self.n,
self.rank,
dim=self.dim,
typ=self.gen_typ,
noise_level=self.noise_level)
start_time = time.time()
sketch = Sketch(X, self.k, random_seed=None)
sketchs, _, = sketch.get_sketchs()
sketch_time = time.time() - start_time
start_time = time.time()
sketch_two_pass = SketchTwoPassRecover(X, sketchs, self.rank)
X_hat, _, _ = sketch_two_pass.recover()
recover_time = time.time() - start_time
rerr = eval_mse(X, X_hat)
return (sketch_time, recover_time), rerr
def get_sketch_for_posterity(self, num_bits):
"""Posterize the image using the number of bits, then
find the areas of contiguous color and make triangles out of them,
returning the resulting sketch
Args:
num_bits: int
"""
if self.color_type == RGB:
im = self.source_image.convert("L")
im = ImageOps.posterize(im, num_bits)
im = im.convert("RGB")
else:
im = ImageOps.posterize(self.source_image, num_bits)
areas = areas_finder.get_areas(im, 1000)
triangles = []
for area in areas:
triangles.extend(self.get_triangles_for_area(area))
triangles = self.filter_and_sort_triangles(triangles)
num_background_triangles = min(self.num_triangles, max(50, self.num_triangles // 10))
background_triangles = self.cover_background(num_background_triangles)
if len(triangles) > (self.num_triangles - num_background_triangles):
over = len(triangles) - (self.num_triangles - num_background_triangles)
triangles = triangles[over:]
triangles = background_triangles + triangles
return Sketch(self.size, self.color_type, triangles)
def get_evolver_and_sketch(
source_image,
output_folder,
num_triangles,
save_frequency,
color_type,
start_from=None,
continue_run=False,
randomized=False,
):
im = Image.open(source_image)
im = im.convert(mode=color_type)
assert not (start_from and continue_run), "you should only use start_from or continue_run, not both"
sketch = None
e = None
if continue_run:
# find the latest file in the folder
file_names = glob.glob(os.path.join(output_folder, "intermediate_???.txt"))
best = -1
for filename in file_names:
num = int(filename[-7:-4])
if num > best:
best = num
if best >= 0:
filename = os.path.join(output_folder, "intermediate_%03d.txt" % best)
module_logger.info("Restarting evolution based on found file 'intermediate_%03d.txt'", best)
sketch = Sketch.read(filename)
e = Evolver(im, output_folder, num_triangles, save_frequency, color_type=color_type, save_index=best + 1)
# Preferred is to load from the auto-save, but in case it died while saving the above will still work
filename = os.path.join(output_folder, "auto_save.txt")
if os.path.exists(filename):
sketch = Sketch.read(filename)
if sketch is None:
utils.clean_dir(output_folder)
if start_from:
sketch = Sketch.read(start_from)
else:
if randomized:
seeder = RandomSeeder(im, output_folder, num_triangles, color_type)
else:
seeder = Seeder(im, output_folder, num_triangles, color_type)
sketch = seeder.run()
e = Evolver(im, output_folder, num_triangles, save_frequency, color_type=color_type)
return e, sketch
def one_pass(self):
X = square_tensor_gen(self.n,
self.rank,
dim=self.dim,
typ=self.gen_typ,
noise_level=self.noise_level)
start_time = time.time()
sketch = Sketch(X, self.k, s=self.s, random_seed=self.random_seed)
sketchs, core_sketch, = sketch.get_sketchs()
sketch_time = time.time() - start_time
start_time = time.time()
sketch_one_pass = SketchOnePassRecover(sketchs, core_sketch,
self.Tinfo_bucket,
self.Rinfo_bucket)
X_hat, _, _ = sketch_one_pass.recover()
recover_time = time.time() - start_time
rerr = eval_mse(X, X_hat)
return (sketch_time, recover_time), rerr
def test_shifted_sketch(self):
sketch = Sketch()
var1 = sketch.add_variable_point()
var2 = sketch.add_point((2.0, 3.0, 4.0))
sketch.add_alignment_constraint(var1, var2, 0, 2.0)
sketch.add_alignment_constraint(var1, var2, 1, 3.0)
sketch.add_alignment_constraint(var1, var2, 2, 4.0)
self.assertTrue(sketch.is_constrained)
def test_simple_sketch(self):
sketch = Sketch()
var1 = sketch.add_variable_point()
var2 = sketch.add_point((2.0, 3.0, 4.0))
# make the points align
sketch.add_alignment_constraint(var1, var2, 0, 0.0)
sketch.add_alignment_constraint(var1, var2, 1, 0.0)
sketch.add_alignment_constraint(var1, var2, 2, 0.0)
self.assertTrue(sketch.is_constrained)
def test_wrong_constraints(self):
sketch = Sketch()
var1 = sketch.add_point((2.0, 4.1, 5.1))
var2 = sketch.add_point((2.0, 3.0, 4.0))
sketch.add_alignment_constraint(var1, var2, 0, 0.0)
with self.assertRaises(InvalidConstraint):
sketch.add_alignment_constraint(var1, var2, 1, 0.0)
import pygame
from pygame.locals import *
from sys import exit
from sketch import Sketch
pygame.init()
sketch = Sketch()
sketch.setup()
clock = pygame.time.Clock()
while True:
clock.tick(20)
for event in pygame.event.get():
if event.type == QUIT:
pygame.quit()
exit()
sketch.draw()
pad = Paddle()
paddle = pad.paddle()
# Ball of circle shape
Ball = Ball()
# Global part
#Sets key bindings to the right thing.
bindings = Bindings(screen, snakehead, paddle)
bindings.Keyboard_bindings()
#movement of the snakehead
move = Move()
move.move(snakehead)
tail = Tail()
semgents = tail.add_tail(segments)
# Displays the score
sk = Sketch()
sketch = sk.Setup()
# Main game loop
while True:
screen.update()
#updates the scoreboard
sk.write(score)
Ball.movementBall()
# sets the start tail to three
tail.start_tail(segments)
# Move the end segments first in reverse order
for index in range(len(segments) - 1, 0, -1):
x = segments[index - 1].xcor()
def run(self):
""" Main routine of the selfplay experiment. """
start = time.time()
# If there is a closed list on file, read it
print('search_algo = ', self.search_algo)
try:
if isinstance(self.search_algo, IteratedWidth):
with open('iw_k' + str(self.search_algo.initial_state.k), mode='rb') as file:
closed_list = pickle.load(file)
print('entrei aqui - iw')
else:
with open('bfs', mode='rb') as file:
closed_list = pickle.load(file)
print('entrei aqui - bfs')
print('tamanho total bfs = ', len(closed_list))
closed_list = closed_list[:n_states]
print('tamanho bfs dps = ', len(closed_list))
except Exception as e:
# Otherwise, calculate it
print('entrei no except - ', str(e))
closed_list = self.search_algo.run()
# Tag the nodes in order for SA to not mutate them later
for state in closed_list:
state.set_fixed_nodes()
# Save closed list to file
with open(self.folder + 'closed_list_' + str(len(closed_list)) + 'states', 'wb') as file:
pickle.dump(closed_list, file)
with open(self.folder + 'closed_list_' + str(len(closed_list)) + 'states_program.txt', 'a') as f:
for i, state in enumerate(closed_list):
print('State', i, 'from closed list - Novelty = ', state.novelty, ' - Depth = ', state.depth, file=f)
print(state.generate_program(), file=f)
print(file=f)
with open(self.folder + 'closed_list_' + str(len(closed_list)) + 'states_tree.txt', 'a') as f:
for i, state in enumerate(closed_list):
to_print = 'State ' + str(i) + ' from closed list - Novelty = ' + str(state.novelty) + ' - Depth = ' + str(state.depth)
state.print_tree_to_file(self.folder + 'closed_list_' + str(len(closed_list)) + 'states_tree.txt', to_print)
print(file=f)
# Player initially to be beaten
player_2 = RandomGlennPlayer()
# Pass a copy of closed_list to not lose the information of which
# unfinished program birthed the better program
copied_closed_list = pickle.loads(pickle.dumps(closed_list, -1))
# Apply a local search to find the best program in closed_list
start_LS = time.time()
best_player_state, best_player_index = self.local_search(copied_closed_list, player_2)
#best_player_state = closed_list[20]
#best_player_index = 20
print('estado escolhido - arvore')
best_player_state.print_tree()
print('estado escolhido - programa')
print(best_player_state.generate_program())
elapsed_LS = time.time() - start_LS
# Finish it randomly because MC simulations can return unfinished programs
best_player_state.finish_tree_randomly()
best_player = Sketch(best_player_state.generate_program()).get_object()
# Save to file - Script chosen before local search
Sketch(closed_list[best_player_index].generate_program()).save_file_custom(self.folder, self.suffix + '_before_LS')
# Save to file - Script chosen after local search
Sketch(best_player_state.generate_program()).save_file_custom(self.folder, self.suffix + '_after_LS')
# Save to file - Script state
with open(self.folder + 'state_chosen_LS', 'wb') as file:
pickle.dump(best_player_state, file)
with open(self.filename, 'a') as f:
print('Best Script before SA - before transformation', file=f)
print(closed_list[best_player_index].generate_program(), file=f)
print('Best Script before SA - after transformation', file=f)
print(best_player_state.generate_program(), file=f)
print('Time elapsed for local search =', elapsed_LS, file=f)
with open(self.filename, 'a') as f:
print('\n\n\n\n\n\n\n\n Final SA will begin \n\n\n\n\n\n\n\n', file=f)
# Apply SA to the best player
start_SA = time.time()
SA_state, _ = self.outer_SA.run(best_player_state, best_player)
elapsed_SA = time.time() - start_SA
with open(self.filename, 'a') as f:
print('Best Script after SA', file=f)
print(SA_state.generate_program(), file=f)
print('Time elapsed for last SA =', elapsed_SA, file=f)
# Save to file - Final script
Sketch(SA_state.generate_program()).save_file_custom(self.folder, self.suffix + '_after_SA')
elapsed = time.time() - start
with open(self.filename, 'a') as f:
print('Elapsed time = ', elapsed, file=f)
return SA_state
def local_search(self, closed_list, player_2):
""" Apply a local search to all unfinished programs. """
# Finish the unfinished programs randomly and evaluate them against
# player_2
if self.LS_type == 0:
scores = []
for i, state in enumerate(closed_list):
start_LS_ite = time.time()
if not state.is_finished():
state.finish_tree_randomly()
player_1 = Sketch(state.generate_program()).get_object()
try:
vic, loss, draw = self.play_batch_games(player_1, player_2)
print('deu certo - i = ', i, 'v/l/d = ', vic, loss, draw)
except:
print('deu ruim - i = ', i)
vic = 0
loss = 0
draw = 0
scores.append(vic)
elapsed_LS_ite = time.time() - start_LS_ite
with open(self.filename, 'a') as f:
print('Finished looping through state', i, 'from closed_list in local search. V/L/D = ', vic, loss, draw, 'Time =', elapsed_LS_ite, file=f)
with open(self.filename, 'a') as f:
print('Score vector = ', scores, file=f)
best_player_index = np.argmax(scores)
return closed_list[best_player_index], best_player_index
# Finish the unfinished programs randomly, apply SA to each of them
# and evaluate them against player_2
elif self.LS_type == 1:
scores = []
for i, state in enumerate(closed_list):
start_LS_ite = time.time()
inner_log = self.folder + '/short_log.txt'
with open(inner_log, 'a') as f:
print('SA beginning - State', i, file=f)
# Finish randomly all unfinished programs
if not state.is_finished():
state.finish_tree_randomly()
# Apply SA
SA_state, _ = self.inner_SA.run(state, player_2)
closed_list[i] = SA_state
# Evaluate it against player_2
player_1 = Sketch(SA_state.generate_program()).get_object()
try:
vic, loss, draw = self.play_batch_games(player_1, player_2)
print('deu certo - i = ', i, 'v/l/d = ', vic, loss, draw)
except:
vic = 0
loss = 0
draw = 0
print('deu ruim - i = ', i)
scores.append(vic)
elapsed_LS_ite = time.time() - start_LS_ite
with open(inner_log, 'a') as f:
print('SA end - Time:', elapsed_LS_ite, '\n\n', file=f)
with open(self.filename, 'a') as f:
print('Finished looping through state', i, 'from closed_list in local search. V/L/D = ', vic, loss, draw, 'Time =', elapsed_LS_ite, file=f)
with open(self.filename, 'a') as f:
print('Score vector = ', scores, file=f)
best_player_index = np.argmax(scores)
return closed_list[best_player_index], best_player_index
# Use Monte Carlo simulations for each of the states while evaluating
# against player_2
elif self.LS_type == 2:
scores = []
for i, state in enumerate(closed_list):
start_LS_ite = time.time()
local_score = []
for _ in range(self.n_MC_simulations):
state_aux = pickle.loads(pickle.dumps(state, -1))
if not state_aux.is_finished():
state_aux.finish_tree_randomly()
player_1 = NewSketch(state_aux.generate_program()).get_object()
try:
vic, loss, draw = self.play_batch_games(player_1, player_2)
print('deu certo - i = ', i, 'v/l/d = ', vic, loss, draw)
except:
vic = 0
loss = 0
draw = 0
print('deu ruim - i = ', i)
local_score.append(vic)
scores.append(sum(local_score)/ len(local_score))
elapsed_LS_ite = time.time() - start_LS_ite
with open(self.filename, 'a') as f:
print('Finished looping through state', i, 'from closed_list in local search. V/L/D = ', vic, loss, draw,'Time =', elapsed_LS_ite, file=f)
with open(self.filename, 'a') as f:
print('Score vector (victories avg) = ', scores, file=f)
# Return the script that won more in the MC simulations
best_player_index = np.argmax(scores)
return closed_list[best_player_index], best_player_index
def run(self, initial_state, player_to_be_beaten):
"""
Main routine of Simulated Annealing. SA will start mutating from
initial_state and the first player to be beaten is player_to_be_beaten.
If this player is beaten, the player that beat it will be the one to be
beaten in the next iteration.
"""
# Original program
curr_tree = pickle.loads(pickle.dumps(initial_state, -1))
curr_player = Sketch(curr_tree.generate_program()).get_object()
# Program to be beaten is given from selfplay
best_tree = None
best_player = player_to_be_beaten
# Evaluate the starting program against the program from selfplay
try:
victories, _, _ = self.evaluate(curr_player, best_player)
# If the program gives an error during evaluation, this program should
# be 'discarded/bad' -> set victory to 0.
except Exception as e:
victories = 0
best_score = victories
# Initial program
with open(self.filename, 'a') as f:
print('Initial SA program below', file=f)
print(initial_state.generate_program(), file=f)
print(file=f)
# Number of "successful" iterations
successful = 0
curr_temp = self.init_temp
for i in range(2, self.n_SA_iterations + 2):
with open(self.filename, 'a') as f:
print('Beginning iteration ',i, file=f)
start = time.time()
#print('sa ite', i)
# Make a copy of curr_tree
mutated_tree = pickle.loads(pickle.dumps(curr_tree, -1))
# Mutate it
mutated_tree.mutate_tree()
# Get the object of the mutated program
mutated_player = Sketch(mutated_tree.generate_program()).get_object()
# Evaluates the mutated program against best_program
try:
victories_mut, losses_mut, draws_mut = self.evaluate(mutated_player, curr_player)
except Exception as e:
victories_mut = 0
losses_mut = 0
draws_mut = 0
new_score = victories_mut
# Update score given the temperature parameters
updated_score_best, updated_score_mutated = self.update_score(
best_score,
new_score,
curr_temp
)
if self.to_log:
inner_folder = self.folder + '/Successful_SA/'
inner_log = self.folder + '/Successful_SA/short_log.txt'
if not os.path.exists(inner_folder):
os.makedirs(inner_folder)
with open(inner_log, 'a') as f:
print('SA - Iteration = ',i, '- V/L/D =', victories_mut, losses_mut, draws_mut, file=f)
else:
inner_log = self.folder + '/short_log.txt'
with open(inner_log, 'a') as f:
print('SA - Iteration = ',i, '- V/L/D =', victories_mut, losses_mut, draws_mut, file=f)
if updated_score_mutated > updated_score_best:
successful += 1
if self.to_log:
# Save to file
inner_folder = self.folder + '/Successful_SA/'
if not os.path.exists(inner_folder):
os.makedirs(inner_folder)
Sketch(mutated_tree.generate_program()).save_file_custom(inner_folder, self.suffix + '_SA_ite' + str(i))
with open(self.filename, 'a') as f:
elapsed = time.time() - start
print('SA - Iteration = ',i, '- Mutated player was better - Victories =', victories_mut, file=f)
print('Mutated program below - Time of this iteration =', elapsed, 'Successful ite. so far = ', successful, file=f)
print(mutated_tree.generate_program(), file=f)
print(file=f)
best_score = new_score
# Copy the trees
best_tree = mutated_tree
# Copy the script
best_player = mutated_player
# update temperature according to schedule
else:
with open(self.filename, 'a') as f:
elapsed = time.time() - start
print('SA - Iteration = ',i, '- Mutated player was worse - Victories =', victories_mut, file=f)
print('Mutated program below - Time of this iteration =', elapsed, 'Successful ite. so far = ', successful, file=f)
print(mutated_tree.generate_program(), file=f)
print(file=f)
curr_temp = self.temperature_schedule(i)
# It can happen that the mutated state is never better than
# player_to_be_beaten, therefore we set best_tree and best_player to the
# original initial_state.
if best_tree is None:
best_tree = curr_tree
best_player = curr_player
with open(self.filename, 'a') as f:
print('Successful iterations of this SA = ', successful, 'out of', self.n_SA_iterations, 'iterations.', file=f)
return best_tree, best_player
median = input()
if (median == 'end'):
break
if (median not in names):
print('输入的文件名有误，仅支持输入当前目录下的文件，请重新输入：')
median = None
else:
filenames.append(median)
k = 5 #k-grams.量化宽度
d = 10000 #文档摘要维度
sketches = [0 for i in filenames]
for i in range(len(filenames)):
with open(filenames[i], 'r', encoding='UTF-8') as f:
text = f.read()
sketches[i] = Sketch(text, k, d)
# 输出结果标题
with open('clone.xls', 'w') as c:
c.write('查重结果\n')
c.write(' ' * 27)
c.write('\t')
print(' ' * 26, end='')
for filename in filenames:
print('{: <21}'.format(filename), end=' ')
c.write(format(filename))
c.write('\t')
c.write('\n')
print()
# 输出结果比较明细
def test_sketch_one_point_constant(self):
sketch = Sketch()
sketch.add_point((0.0, 0.0, 0.0))
self.assertTrue(sketch.is_constrained)
def run(self):
return Sketch(self.size, self.color_type)
# # s_new = db.insert_host_unknown_hashes(domain_unknown_hashes, SKETCH_WINDOW) # db.get_host_unknown_hashes(domain_unknown_hashes) # # domain_total_hashes = Sketch(SKETCH_DELTA, SKETCH_EPSILON, SKETCH_WINDOW, 0) # # s_new = db.insert_host_total_hashes(domain_total_hashes, SKETCH_WINDOW) # db.get_host_total_hashes(domain_total_hashes) # # # domain_unknown_hash_ratio = Sketch(SKETCH_DELTA, SKETCH_EPSILON, SKETCH_WINDOW, 1) # # s_new = db.insert_host_unknown_hash_ratio(domain_unknown_hash_ratio) # db.get_host_unknown_hash_ratio(domain_unknown_hash_ratio) server_total_downloads = Sketch(SKETCH_DELTA, SKETCH_EPSILON, SKETCH_WINDOW, 0) s_new = db.insert_server_total(server_total_downloads, SKETCH_WINDOW) db.get_server_total(server_total_downloads) # # server_malware_downloads = Sketch(SKETCH_DELTA, SKETCH_EPSILON, SKETCH_WINDOW, 0) # # s_new = db.insert_server_malware(server_malware_downloads, SKETCH_WINDOW) # db.get_server_malware(server_malware_downloads) # # server_suspicious_downloads = Sketch(SKETCH_DELTA, SKETCH_EPSILON, SKETCH_WINDOW, 0) # # s_new = db.insert_server_suspicious(server_suspicious_downloads, SKETCH_WINDOW) # db.get_server_suspicious(server_suspicious_downloads) # # server_benign_downloads = Sketch(SKETCH_DELTA, SKETCH_EPSILON, SKETCH_WINDOW, 0)
def test_sketch_one_point_variable(self):
sketch = Sketch()
sketch.add_point((0.0, 0.0, 0.0))
sketch.add_variable_point()
self.assertFalse(sketch.is_constrained)