def start_profiling(func, filepath, statistical=True):
import pprofile, signal
if statistical:
prof = pprofile.StatisticalProfile()
else:
prof = pprofile.Profile()
def stop_profiling(prof, filepath):
print('Writing profiling data: %s' % filepath, file=sys.stderr)
print('You can use kcachegrind to analyze it.', file=sys.stderr)
with open(filepath, 'w') as f:
prof.callgrind(f)
# This makes the `finally` block work as expected if we're terminated by
# SIGTERM, which happens by default when running `timeout 30 stig ...`.
# https://stackoverflow.com/a/42200623
# https://mail.python.org/pipermail/python-ideas/2016-February/038474.html
import signal
class SigTerm(SystemExit):
pass
def sigterm(sig, frame):
raise SigTerm
signal.signal(15, sigterm)
try:
with prof():
func()
finally:
stop_profiling(prof, filepath)
def extract_syscalls(fname):
profiler = pprofile.Profile()
# clear the function cache if it is not empty, otherwise we get random syscalls based on the order of dependencies being processed
if (len(function_cache) > 0):
print("Clearing function cache")
function_cache.clear()
start_time()
try:
cfg = get_cfg(fname)
except:
print("[-] angr could not extract the CFG from %s" % fname)
return None
stop_time("Getting CFG")
insn = syscalls.init(fname)
stop_time("Syscall Init")
addrs = syscalls.find_syscall_locations(insn)
stop_time("Syscall locations")
sys_addrs = [x[1] for x in addrs]
syslist = function_calling_syscalls(cfg, sys_addrs)
stop_time("Syslist")
callees = get_call_targets(cfg)
stop_time("Callee list")
syscaller = syscalls_per_function(cfg, callees, syslist)
stop_time("Extract syscalls per function")
whitelist = {}
try:
with open("function_whitelist.json") as wl:
whitelist = json.loads(wl.read())
except:
pass
insn_to_syscall = {}
used_syscalls = {}
for fnc in syscaller:
insn_list = []
for sysc in syscaller[fnc]:
for addr in addrs:
if addr[1] == sysc:
if addr[0] not in insn_to_syscall:
insn_to_syscall[addr[0]] = syscalls.find_syscall_nr(
insn, addr[0])
insn_list.append(insn_to_syscall[addr[0]])
if fnc in whitelist:
print("Found %s in whitelist, adding %d syscall(s)" %
(fnc, len(whitelist[fnc])))
insn_list += whitelist[fnc]
used_syscalls[fnc] = sorted(list(set(insn_list))).copy()
stop_time("Find syscall numbers")
all_syscalls = set()
for f in used_syscalls:
all_syscalls.update(set(used_syscalls[f]))
used_syscalls[":all"] = sorted(list(all_syscalls))
return used_syscalls
def profiler(request):
profile = pprofile.Profile()
yield profile
filename = request.config.getoption('--profile')
with filename.open('w', encoding='utf-8') as out:
profile.annotate(out)
with filename.with_suffix('.callgrind').open('w') as out:
profile.callgrind(out)
def __init__(self, sample_mode=False, thread_mode=False, period=0.01):
self._sample_mode = sample_mode
if sample_mode:
self._profiler = pprofile.StatisticThread(single=not thread_mode,
period=period)
elif thread_mode:
self._profiler = pprofile.ThreadProfile()
else:
self._profiler = pprofile.Profile()
def profile_algorithm(algorithm_name, filename):
dataset = _numeric_columns_from_csv(filename)
dataset.dropna(axis=0, how='any', inplace=True)
prof = pprofile.Profile()
with prof:
_benchmark_algorithm(algorithm_name, dataset)
prof.dump_stats(filename="./calc/benchmarks/profiling.log")
with open("./calc/benchmarks/profiling.log", 'r') as prof_file:
with open("./calc/benchmarks/profiling_stripped.log",
'w') as stripped_file:
for line in prof_file:
line = re.sub(r'^.*\|.*0\.00%\|.*$\n', r'', line)
stripped_file.write(line)
def inner_wrapper(*args, **kwargs):
r.incr(key)
call_num = int(r.get(key))
if call_num > max_call_num or (call_num - 1) % step != 0:
return func(*args, **kwargs)
print_title = (' ' * 30 + f"-*-pprofile_print_stats-*-|{call_num}")
prof = pprofile.Profile()
with prof():
result = func(*args, **kwargs)
print('-' * 100)
print(print_title)
print('-' * 100)
print('')
_pprofile_dump(prof,
f"{PROFILE_ROOT_PATH}/cachegrind.out.{call_num}")
return result
def inner_wrapper(*args, **kwargs):
r.incr(key)
call_num = int(r.get(key))
if call_num > max_call_num or (call_num - 1) % step != 0:
return func(*args, **kwargs)
print_title = (' ' * 30 + f'-*-pprofile_print_stats-*-|{call_num}')
prof = pprofile.Profile()
with prof():
result = func(*args, **kwargs)
print('-' * 100)
print(print_title)
print('-' * 100)
print('')
_pprofile_dump(
prof, f"/tmp/{PROJECT_NAME}/pp/cachegrind.out.{call_num}",
call_num == 1)
return result
def line_profiler():
profiler = None
try:
if settings.PROFILER["line-profiler"]:
import pprofile
if settings.PROFILER["line-profiler-type"] == "deterministic":
profiler = pprofile.Profile()
elif settings.PROFILER["line-profiler-type"] == "statistic":
prof = pprofile.StatisticalProfile()
profiler = prof(
period=0.001, # Sample every 1ms
single=True, # Only sample current thread
)
except ImportError:
print("Unable to create line_profiler : ImportError")
except Exception as e:
print("Unable to create line_profiler : " + str(e))
return profiler
def harvest(self):
import pprofile
profiler = pprofile.Profile()
profiler.enable()
objects = []
div = self.soup.findAll('div', {'id': "mw-content-text"})[0]
pattern = re.compile("\w*, \w*")
for tag in div.findAll('a'):
name = re.findall(pattern, tag.text)
if name:
objects.append({
'name': name[0],
})
profiler.disable()
with open('names_harvest.prof', 'w+') as out:
profiler.callgrind(out)
return objects
def run_all_tests_cases():
"""
Run all the test cases
:return:
"""
for edp in test_cases():
if profiling:
profiler = pprofile.Profile()
with profiler:
run_a_test_case(edp)
# profiler.print_stats()
with open("profile." + edp.name, 'w') as fich:
profiler.callgrind(fich)
# cProfile.run("edp.run_test_case(graphs)", "profile." + edp.name)
# p = pstats.Stats("profile." + edp.name)
# p.sort_stats('time').print_stats(10)
# p.sort_stats('cumulative').print_stats(10)
# os.remove("profile." + edp.name)
else:
run_a_test_case(edp)
def benchmark_dataloader(
loader: DataLoader,
max_iterations: int = -1,
profile: bool = False,
profile_callgrind: Path = None,
) -> None:
dataloader_iter = iter(loader)
def run_dataloader():
end_of_iter_time = time()
total_iterations = (
len(dataloader_iter)
if max_iterations < 0
else min(len(dataloader_iter), max_iterations)
)
for i, batch in enumerate(dataloader_iter):
if 0 < max_iterations <= i:
break
start_of_iter_time = time()
dataloader_duration_s = start_of_iter_time - end_of_iter_time
examples_per_second = loader.batch_size / dataloader_duration_s
print(
"batch[{}/{}] {:.2f} examples/s".format(
i + 1, total_iterations, examples_per_second
)
)
end_of_iter_time = start_of_iter_time
if profile:
prof = pprofile.Profile()
with prof():
run_dataloader()
if profile_callgrind is not None:
with open(str(profile_callgrind), "w", encoding="utf8") as f:
prof.callgrind(f)
print("Wrote callgrind profile log to {}".format(profile_callgrind))
else:
prof.print_stats()
else:
run_dataloader()
def profile(self, func, suffix='', args=(), kw=None):
"""Profile `func(*args, **kw)` with selected profiler,
and dump output in a file called `func.__name__ + suffix`
"""
if not kw:
kw = {}
if PROFILER == 'pprofile':
import pprofile
prof = pprofile.Profile()
else:
from cProfile import Profile
prof = Profile()
prof_file = '%s%s' % (func.__name__, suffix)
try:
os.unlink(prof_file)
except OSError:
pass
prof.runcall(func, *args, **kw)
prof.dump_stats(prof_file)
async def execute(self, message, profile=False):
if message.author.id == self.user.id:
return
if message.content.startswith(self.profile.prefix):
raw = message.content[len(self.profile.prefix):].split()
else:
raw = message.content.split(" ", 2)[1:]
cmd = raw[0].lower()
args = " ".join(raw[1:])
args = re.compile(r'''((?:[^\s"']|"[^"]*"|'[^']*')+)''').split(
args)[1::2]
new = []
for arg in args:
new.append(arg.strip("\""))
args = new
for ext in self.extensions:
for c in ext.commands:
if c.name == cmd:
if bot.in_role_list(message.author, c.roles):
if profile:
import pprofile
profiler = pprofile.Profile()
with profiler():
await self.command(
c, bot.Context(self, ext, message), args)
out = io.StringIO()
with redirect_stdout(out):
profiler.print_stats()
resp = out.getvalue().split("\n")
await bot.send_stats(c, self.profile.prefix, resp,
raw, message.channel)
else:
await self.command(c,
bot.Context(self, ext, message),
args)
else:
await message.channel.send(
"You are not allowed to run that command.")
import random, math, time
import pprofile
profiler = pprofile.Profile()
class MonteCarlo():
def __init__(self, mode=0, runAmount=900):
self.mode = mode
self.runAmount = runAmount
self.timeRand = []
self.congruentialRand = None
def getRandom(self, min=0, max=1000):
rand = 1
rand = random.randint(min, max)
return rand
def getCongruentialRand(self, a, b, m):
if self.congruentialRand == None:
self.congruentialRand = int(m / 2) # r_0
self.congruentialRand = (a * self.congruentialRand + b) % m # r_n++
return self.congruentialRand
# Function to check if a random point is inside the circle quarter
def checkIfInside(self):
x = 0
y = 0
m = 1000
def __profile_file(self):
"""Method used to profile the given file line by line."""
self.line_profiler = pprofile.Profile()
self.line_profiler.runfile(open(self.pyfile.path, 'r'), {},
self.pyfile.path)
def execute_wfc(
filename: Optional[str] = None,
tile_size: int = 1,
pattern_width: int = 2,
rotations: int = 8,
output_size: Tuple[int, int] = (48, 48),
ground: Optional[int] = None,
attempt_limit: int = 10,
output_periodic: bool = True,
input_periodic: bool = True,
loc_heuristic: Literal["lexical", "hilbert", "spiral", "entropy", "anti-entropy", "simple", "random"] = "entropy",
choice_heuristic: Literal["lexical", "rarest", "weighted", "random"] = "weighted",
visualize: bool = False,
global_constraint: Literal[False, "allpatterns"] = False,
backtracking: bool = False,
log_filename: str = "log",
logging: bool = False,
global_constraints: None = None,
log_stats_to_output: Optional[Callable[[Dict[str, Any], str], None]] = None,
*,
image: Optional[NDArray[np.integer]] = None,
) -> NDArray[np.integer]:
timecode = datetime.datetime.now().isoformat().replace(":", ".")
time_begin = time.perf_counter()
output_destination = r"./output/"
input_folder = r"./images/samples/"
rotations -= 1 # change to zero-based
input_stats = {
"filename": str(filename),
"tile_size": tile_size,
"pattern_width": pattern_width,
"rotations": rotations,
"output_size": output_size,
"ground": ground,
"attempt_limit": attempt_limit,
"output_periodic": output_periodic,
"input_periodic": input_periodic,
"location heuristic": loc_heuristic,
"choice heuristic": choice_heuristic,
"global constraint": global_constraint,
"backtracking": backtracking,
}
# Load the image
if filename:
if image is not None:
raise TypeError("Only filename or image can be provided, not both.")
image = imageio.imread(input_folder + filename + ".png")[:, :, :3] # TODO: handle alpha channels
if image is None:
raise TypeError("An image must be given.")
# TODO: generalize this to more than the four cardinal directions
direction_offsets = list(enumerate([(0, -1), (1, 0), (0, 1), (-1, 0)]))
tile_catalog, tile_grid, _code_list, _unique_tiles = make_tile_catalog(image, tile_size)
(
pattern_catalog,
pattern_weights,
pattern_list,
pattern_grid,
) = make_pattern_catalog_with_rotations(
tile_grid, pattern_width, input_is_periodic=input_periodic, rotations=rotations
)
logger.debug("pattern catalog")
# visualize_tiles(unique_tiles, tile_catalog, tile_grid)
# visualize_patterns(pattern_catalog, tile_catalog, pattern_weights, pattern_width)
# figure_list_of_tiles(unique_tiles, tile_catalog, output_filename=f"visualization/tilelist_{filename}_{timecode}")
# figure_false_color_tile_grid(tile_grid, output_filename=f"visualization/tile_falsecolor_{filename}_{timecode}")
if visualize and filename:
figure_pattern_catalog(
pattern_catalog,
tile_catalog,
pattern_weights,
pattern_width,
output_filename=f"visualization/pattern_catalog_{filename}_{timecode}",
)
logger.debug("profiling adjacency relations")
if False:
import pprofile # type: ignore
profiler = pprofile.Profile()
with profiler:
adjacency_relations = adjacency_extraction(
pattern_grid,
pattern_catalog,
direction_offsets,
[pattern_width, pattern_width],
)
profiler.dump_stats(f"logs/profile_adj_{filename}_{timecode}.txt")
else:
adjacency_relations = adjacency_extraction(
pattern_grid,
pattern_catalog,
direction_offsets,
(pattern_width, pattern_width),
)
logger.debug("adjacency_relations")
if visualize:
figure_adjacencies(
adjacency_relations,
direction_offsets,
tile_catalog,
pattern_catalog,
pattern_width,
[tile_size, tile_size],
output_filename=f"visualization/adjacency_{filename}_{timecode}_A",
)
# figure_adjacencies(adjacency_relations, direction_offsets, tile_catalog, pattern_catalog, pattern_width, [tile_size, tile_size], output_filename=f"visualization/adjacency_{filename}_{timecode}_B", render_b_first=True)
logger.debug(f"output size: {output_size}\noutput periodic: {output_periodic}")
number_of_patterns = len(pattern_weights)
logger.debug(f"# patterns: {number_of_patterns}")
decode_patterns = dict(enumerate(pattern_list))
encode_patterns = {x: i for i, x in enumerate(pattern_list)}
_encode_directions = {j: i for i, j in direction_offsets}
adjacency_list: Dict[Tuple[int, int], List[Set[int]]] = {}
for _, adjacency in direction_offsets:
adjacency_list[adjacency] = [set() for _ in pattern_weights]
# logger.debug(adjacency_list)
for adjacency, pattern1, pattern2 in adjacency_relations:
# logger.debug(adjacency)
# logger.debug(decode_patterns[pattern1])
adjacency_list[adjacency][encode_patterns[pattern1]].add(encode_patterns[pattern2])
logger.debug(f"adjacency: {len(adjacency_list)}")
time_adjacency = time.perf_counter()
### Ground ###
ground_list: Optional[NDArray[np.int64]] = None
if ground:
ground_list = np.vectorize(lambda x: encode_patterns[x])(
pattern_grid.flat[(ground - 1) :]
)
if ground_list is None or ground_list.size == 0:
ground_list = None
if ground_list is not None:
ground_catalog = {
encode_patterns[k]: v
for k, v in pattern_catalog.items()
if encode_patterns[k] in ground_list
}
if visualize:
figure_pattern_catalog(
ground_catalog,
tile_catalog,
pattern_weights,
pattern_width,
output_filename=f"visualization/patterns_ground_{filename}_{timecode}",
)
wave = makeWave(
number_of_patterns, output_size[0], output_size[1], ground=ground_list
)
adjacency_matrix = makeAdj(adjacency_list)
### Heuristics ###
encoded_weights: NDArray[np.float64] = np.zeros((number_of_patterns), dtype=np.float64)
for w_id, w_val in pattern_weights.items():
encoded_weights[encode_patterns[w_id]] = w_val
choice_random_weighting: NDArray[np.float64] = np.random.random_sample(wave.shape[1:]) * 0.1
pattern_heuristic: Callable[[NDArray[np.bool_], NDArray[np.bool_]], int] = lexicalPatternHeuristic
if choice_heuristic == "rarest":
pattern_heuristic = makeRarestPatternHeuristic(encoded_weights)
if choice_heuristic == "weighted":
pattern_heuristic = makeWeightedPatternHeuristic(encoded_weights)
if choice_heuristic == "random":
pattern_heuristic = makeRandomPatternHeuristic(encoded_weights)
logger.debug(loc_heuristic)
location_heuristic: Callable[[NDArray[np.bool_]], Tuple[int, int]] = lexicalLocationHeuristic
if loc_heuristic == "anti-entropy":
location_heuristic = makeAntiEntropyLocationHeuristic(choice_random_weighting)
if loc_heuristic == "entropy":
location_heuristic = makeEntropyLocationHeuristic(choice_random_weighting)
if loc_heuristic == "random":
location_heuristic = makeRandomLocationHeuristic(choice_random_weighting)
if loc_heuristic == "simple":
location_heuristic = simpleLocationHeuristic
if loc_heuristic == "spiral":
location_heuristic = makeSpiralLocationHeuristic(choice_random_weighting)
if loc_heuristic == "hilbert":
location_heuristic = makeHilbertLocationHeuristic(choice_random_weighting)
### Visualization ###
(
visualize_choice,
visualize_wave,
visualize_backtracking,
visualize_propagate,
visualize_final,
visualize_after,
) = (None, None, None, None, None, None)
if filename and visualize:
(
visualize_choice,
visualize_wave,
visualize_backtracking,
visualize_propagate,
visualize_final,
visualize_after,
) = make_solver_visualizers(
f"{filename}_{timecode}",
wave,
decode_patterns=decode_patterns,
pattern_catalog=pattern_catalog,
tile_catalog=tile_catalog,
tile_size=[tile_size, tile_size],
)
if filename and logging:
(
visualize_choice,
visualize_wave,
visualize_backtracking,
visualize_propagate,
visualize_final,
visualize_after,
) = make_solver_loggers(f"{filename}_{timecode}", input_stats.copy())
if filename and logging and visualize:
vis = make_solver_visualizers(
f"{filename}_{timecode}",
wave,
decode_patterns=decode_patterns,
pattern_catalog=pattern_catalog,
tile_catalog=tile_catalog,
tile_size=[tile_size, tile_size],
)
log = make_solver_loggers(f"{filename}_{timecode}", input_stats.copy())
def visfunc(idx: int):
def vf(*args, **kwargs):
if vis[idx]:
vis[idx](*args, **kwargs)
if log[idx]:
return log[idx](*args, **kwargs)
return vf
(
visualize_choice,
visualize_wave,
visualize_backtracking,
visualize_propagate,
visualize_final,
visualize_after,
) = [visfunc(x) for x in range(len(vis))]
### Global Constraints ###
active_global_constraint = lambda wave: True
if global_constraint == "allpatterns":
active_global_constraint = make_global_use_all_patterns()
logger.debug(active_global_constraint)
combined_constraints = [active_global_constraint]
def combinedConstraints(wave: NDArray[np.bool_]) -> bool:
return all(fn(wave) for fn in combined_constraints)
### Solving ###
time_solve_start = None
time_solve_end = None
solution_tile_grid = None
logger.debug("solving...")
attempts = 0
while attempts < attempt_limit:
attempts += 1
time_solve_start = time.perf_counter()
stats = {}
# profiler = pprofile.Profile()
# with profiler:
# with PyCallGraph(output=GraphvizOutput(output_file=f"visualization/pycallgraph_{filename}_{timecode}.png")):
try:
solution = run(
wave.copy(),
adjacency_matrix,
locationHeuristic=location_heuristic,
patternHeuristic=pattern_heuristic,
periodic=output_periodic,
backtracking=backtracking,
onChoice=visualize_choice,
onBacktrack=visualize_backtracking,
onObserve=visualize_wave,
onPropagate=visualize_propagate,
onFinal=visualize_final,
checkFeasible=combinedConstraints,
)
if visualize_after:
stats = visualize_after()
# logger.debug(solution)
# logger.debug(stats)
solution_as_ids = np.vectorize(lambda x: decode_patterns[x])(solution)
solution_tile_grid = pattern_grid_to_tiles(
solution_as_ids, pattern_catalog
)
logger.debug("Solution:")
# logger.debug(solution_tile_grid)
if filename:
render_tiles_to_output(
solution_tile_grid,
tile_catalog,
(tile_size, tile_size),
output_destination + filename + "_" + timecode + ".png",
)
time_solve_end = time.perf_counter()
stats.update({"outcome": "success"})
except StopEarly:
logger.debug("Skipping...")
stats.update({"outcome": "skipped"})
raise
except TimedOut:
logger.debug("Timed Out")
if visualize_after:
stats = visualize_after()
stats.update({"outcome": "timed_out"})
except Contradiction as exc:
logger.warning(f"Contradiction: {exc}")
if visualize_after:
stats = visualize_after()
stats.update({"outcome": "contradiction"})
finally:
# profiler.dump_stats(f"logs/profile_{filename}_{timecode}.txt")
outstats = {}
outstats.update(input_stats)
solve_duration = time.perf_counter() - time_solve_start
if time_solve_end is not None:
solve_duration = time_solve_end - time_solve_start
adjacency_duration = time_solve_start - time_adjacency
outstats.update(
{
"attempts": attempts,
"time_start": time_begin,
"time_adjacency": time_adjacency,
"adjacency_duration": adjacency_duration,
"time solve start": time_solve_start,
"time solve end": time_solve_end,
"solve duration": solve_duration,
"pattern count": number_of_patterns,
}
)
outstats.update(stats)
if log_stats_to_output is not None:
log_stats_to_output(outstats, output_destination + log_filename + ".tsv")
if solution_tile_grid is not None:
return tile_grid_to_image(solution_tile_grid, tile_catalog, (tile_size, tile_size))
raise TimedOut("Attempt limit exceeded.")
def main(daemon_mode=True):
global prof
# parse option
parser = argparse.ArgumentParser()
parser.add_argument('--pid',
action='store',
dest='pid',
default=None,
help='pid filename')
parser.add_argument('--single',
action='store_true',
dest='singleMode',
default=False,
help='use single mode')
parser.add_argument(
'--hostname_file',
action='store',
dest='hostNameFile',
default=None,
help='to record the hostname where harvester is launched')
parser.add_argument('--rotate_log',
action='store_true',
dest='rotateLog',
default=False,
help='rollover log files before launching harvester')
parser.add_argument('--version',
action='store_true',
dest='showVersion',
default=False,
help='show version information and exit')
parser.add_argument('--profile_output',
action='store',
dest='profileOutput',
default=None,
help='filename to save the results of profiler')
parser.add_argument(
'--profile_mode',
action='store',
dest='profileMode',
default='s',
help=
'profile mode. s (statistic), d (deterministic), or t (thread-aware)')
parser.add_argument(
'--memory_logging',
action='store_true',
dest='memLogging',
default=False,
help='add information of memory usage in each logging message')
parser.add_argument('--foreground',
action='store_true',
dest='foreground',
default=False,
help='run in the foreground not to be daemonized')
options = parser.parse_args()
# show version information
if options.showVersion:
print("Version : {0}".format(panda_pkg_info.release_version))
print("Last commit : {0}".format(commit_timestamp.timestamp))
return
# check pid
if options.pid is not None and os.path.exists(options.pid):
print("ERROR: Cannot start since lock file {0} already exists".format(
options.pid))
return
# uid and gid
uid = pwd.getpwnam(harvester_config.master.uname).pw_uid
gid = grp.getgrnam(harvester_config.master.gname).gr_gid
# get umask
umask = os.umask(0)
os.umask(umask)
# memory logging
if options.memLogging:
core_utils.enable_memory_profiling()
# hostname
if options.hostNameFile is not None:
with open(options.hostNameFile, 'w') as f:
f.write(socket.getfqdn())
# rollover log files
if options.rotateLog:
core_utils.do_log_rollover()
if hasattr(_logger.handlers[0], 'doRollover'):
_logger.handlers[0].doRollover()
if daemon_mode and not options.foreground:
# redirect messages to stdout
stdoutHandler = logging.StreamHandler(sys.stdout)
stdoutHandler.setFormatter(_logger.handlers[0].formatter)
_logger.addHandler(stdoutHandler)
# collect streams not to be closed by daemon
files_preserve = []
for loggerName, loggerObj in iteritems(
logging.Logger.manager.loggerDict):
if loggerName.startswith('panda'):
for handler in loggerObj.handlers:
if hasattr(handler, 'stream'):
files_preserve.append(handler.stream)
sys.stderr = StdErrWrapper()
# make daemon context
dc = daemon.DaemonContext(stdout=sys.stdout,
stderr=sys.stderr,
uid=uid,
gid=gid,
umask=umask,
files_preserve=files_preserve,
pidfile=daemon.pidfile.PIDLockFile(
options.pid))
else:
dc = DummyContext()
with dc:
# remove pidfile to prevent child processes crashing in atexit
if not options.singleMode:
dc.pidfile = None
core_utils.set_file_permission(options.pid)
core_utils.set_file_permission(logger_config.daemon['logdir'])
_logger.info("start : version = {0}, last_commit = {1}".format(
panda_pkg_info.release_version, commit_timestamp.timestamp))
# stop event
stopEvent = threading.Event()
# profiler
prof = None
if options.profileOutput is not None:
# run with profiler
if options.profileMode == 'd':
# deterministic
prof = pprofile.Profile()
elif options.profileMode == 't':
# thread-aware
prof = pprofile.ThreadProfile()
else:
# statistic
prof = cProfile.Profile()
# post process for profiler
def disable_profiler():
global prof
if prof is not None:
# disable profiler
prof.disable()
# dump results
prof.dump_stats(options.profileOutput)
prof = None
# signal handlers
def catch_sigkill(sig, frame):
disable_profiler()
_logger.info('got signal={0}'.format(sig))
try:
os.remove(options.pid)
except:
pass
if os.getppid() == 1:
os.killpg(os.getpgrp(), signal.SIGKILL)
else:
os.kill(os.getpid(), signal.SIGKILL)
def catch_sigterm(sig, frame):
stopEvent.set()
try:
os.remove(options.pid)
except:
pass
# set handler
if daemon_mode:
signal.signal(signal.SIGINT, catch_sigkill)
signal.signal(signal.SIGHUP, catch_sigkill)
signal.signal(signal.SIGTERM, catch_sigkill)
signal.signal(signal.SIGUSR2, catch_sigterm)
# start master
master = Master(single_mode=options.singleMode,
stop_event=stopEvent,
daemon_mode=daemon_mode)
if master is None:
prof = None
else:
# enable profiler
if prof is not None:
prof.enable()
# run master
master.start()
# disable profiler
disable_profiler()
if daemon_mode:
_logger.info('terminated')
#!/usr/bin/env python
import threading
import pprofile
import time
import sys
def func():
# Busy loop, so context switches happen
end = time.time() + 1
while time.time() < end:
pass
# Single-treaded run
prof = pprofile.Profile()
with prof:
func()
prof.annotate(sys.stdout, __file__)
# Dual-threaded run
t1 = threading.Thread(target=func)
prof = pprofile.Profile()
with prof:
t1.start()
func()
t1.join()
prof.annotate(sys.stdout, __file__)
def wrapper(*args, **kwargs):
""" Define the input object decorator.
"""
# Get the function parameters
arg_spec = inspect.getargspec(obj)
defaults = [repr(item) for item in arg_spec.defaults or []]
optional = dict(zip(reversed(arg_spec.args or []), reversed(defaults)))
for name, value in kwargs.items():
if name in optional:
optional[name] = object_repr(value)
mandatory = []
self_parameter = None
for index in range(len(arg_spec.args) - len(optional)):
try:
if index < len(args):
value = args[index]
else:
value = kwargs[arg_spec.args[index]]
if arg_spec.args[index] == "self":
self_parameter = value
if arg_spec.args[index] == "cls":
args = args[1:]
value_repr = object_repr(value)
mandatory.append((arg_spec.args[index], value_repr))
except:
mandatory.append((arg_spec.args[index], None))
raise
# Create the function signature
params = ["{0}={1}".format(name, val) for name, val in mandatory]
params.extend(
["{0}={1}".format(name, val) for name, val in optional.items()])
signature = "{0}({1})".format(obj.__name__, ", ".join(params))
# Display a start call message
module = obj.__module__
package_name = module.split(".")[0]
if is_method:
obj_name = (module + "." + self_parameter.__class__.__name__ +
"." + obj.__name__)
else:
obj_name = module + "." + obj.__name__
print("{0}\n[{1}] Calling {2}...\n{3}".format(80 * "_", package_name,
obj_name, signature))
# A type signature if requested
_obj = obj
for decorator, registered_types in type_decorators:
_obj = decorator(*registered_types)(_obj)
# Call
start_time = time.time()
if use_profiler:
profiler = pprofile.Profile()
returncode = profiler.runcall(obj, *args, **kwargs)
else:
returncode = _obj(*args, **kwargs)
duration = time.time() - start_time
# Display execution profile
if use_profiler:
obj_code = inspect.getsourcelines(obj)[0]
annotate(profiler, sys.stdout,
inspect.getmodule(obj).__file__, obj_code)
# Display an end message
msg = "{0:.1f}s, {1:.1f}min".format(duration, duration / 60.)
print(max(0, (80 - len(msg))) * '_' + msg)
return returncode
def execute_wfc(filename,
tile_size=0,
pattern_width=2,
rotations=8,
output_size=[48, 48],
ground=None,
attempt_limit=10,
output_periodic=True,
input_periodic=True,
loc_heuristic="lexical",
choice_heuristic="lexical",
visualize=True,
global_constraint=False,
backtracking=False,
log_filename="log",
logging=True,
global_constraints=None,
log_stats_to_output=None):
timecode = f"{time.time()}"
time_begin = time.time()
output_destination = r"./output/"
input_folder = r"./images/samples/"
rotations -= 1 # change to zero-based
input_stats = {
"filename": filename,
"tile_size": tile_size,
"pattern_width": pattern_width,
"rotations": rotations,
"output_size": output_size,
"ground": ground,
"attempt_limit": attempt_limit,
"output_periodic": output_periodic,
"input_periodic": input_periodic,
"location heuristic": loc_heuristic,
"choice heuristic": choice_heuristic,
"global constraint": global_constraint,
"backtracking": backtracking
}
# Load the image
img = imageio.imread(input_folder + filename + ".png")
img = img[:, :, :3] # TODO: handle alpha channels
# TODO: generalize this to more than the four cardinal directions
direction_offsets = list(enumerate([(0, -1), (1, 0), (0, 1), (-1, 0)]))
tile_catalog, tile_grid, code_list, unique_tiles = make_tile_catalog(
img, tile_size)
pattern_catalog, pattern_weights, pattern_list, pattern_grid = make_pattern_catalog_with_rotations(
tile_grid,
pattern_width,
input_is_periodic=input_periodic,
rotations=rotations)
print("pattern catalog")
#visualize_tiles(unique_tiles, tile_catalog, tile_grid)
#visualize_patterns(pattern_catalog, tile_catalog, pattern_weights, pattern_width)
#figure_list_of_tiles(unique_tiles, tile_catalog, output_filename=f"visualization/tilelist_{filename}_{timecode}")
#figure_false_color_tile_grid(tile_grid, output_filename=f"visualization/tile_falsecolor_{filename}_{timecode}")
if visualize:
figure_pattern_catalog(
pattern_catalog,
tile_catalog,
pattern_weights,
pattern_width,
output_filename=
f"visualization/pattern_catalog_{filename}_{timecode}")
print("profiling adjacency relations")
adjacency_relations = None
if False:
profiler = pprofile.Profile()
with profiler:
adjacency_relations = adjacency_extraction(
pattern_grid, pattern_catalog, direction_offsets,
[pattern_width, pattern_width])
profiler.dump_stats(f"logs/profile_adj_{filename}_{timecode}.txt")
else:
adjacency_relations = adjacency_extraction(
pattern_grid, pattern_catalog, direction_offsets,
[pattern_width, pattern_width])
print("adjacency_relations")
if visualize:
figure_adjacencies(
adjacency_relations,
direction_offsets,
tile_catalog,
pattern_catalog,
pattern_width, [tile_size, tile_size],
output_filename=f"visualization/adjacency_{filename}_{timecode}_A")
#figure_adjacencies(adjacency_relations, direction_offsets, tile_catalog, pattern_catalog, pattern_width, [tile_size, tile_size], output_filename=f"visualization/adjacency_{filename}_{timecode}_B", render_b_first=True)
print(f"output size: {output_size}\noutput periodic: {output_periodic}")
number_of_patterns = len(pattern_weights)
print(f"# patterns: {number_of_patterns}")
decode_patterns = dict(enumerate(pattern_list))
encode_patterns = {x: i for i, x in enumerate(pattern_list)}
encode_directions = {j: i for i, j in direction_offsets}
adjacency_list = {}
for i, d in direction_offsets:
adjacency_list[d] = [set() for i in pattern_weights]
#print(adjacency_list)
for i in adjacency_relations:
#print(i)
#print(decode_patterns[i[1]])
adjacency_list[i[0]][encode_patterns[i[1]]].add(encode_patterns[i[2]])
print(f"adjacency: {len(adjacency_list)}")
time_adjacency = time.time()
### Ground ###
ground_list = []
if not (ground is 0):
ground_list = np.vectorize(lambda x: encode_patterns[x])(
pattern_grid.flat[(ground - 1):])
if len(ground_list) < 1:
ground_list = None
if not (ground_list is None):
ground_catalog = {
encode_patterns[k]: v
for k, v in pattern_catalog.items()
if encode_patterns[k] in ground_list
}
if visualize:
figure_pattern_catalog(
ground_catalog,
tile_catalog,
pattern_weights,
pattern_width,
output_filename=
f"visualization/patterns_ground_{filename}_{timecode}")
wave = makeWave(number_of_patterns,
output_size[0],
output_size[1],
ground=ground_list)
adjacency_matrix = makeAdj(adjacency_list)
### Heuristics ###
encoded_weights = np.zeros((number_of_patterns), dtype=np.float64)
for w_id, w_val in pattern_weights.items():
encoded_weights[encode_patterns[w_id]] = w_val
choice_random_weighting = np.random.random(wave.shape[1:]) * 0.1
pattern_heuristic = lexicalPatternHeuristic
if choice_heuristic == "weighted":
pattern_heuristic = makeWeightedPatternHeuristic(encoded_weights)
if choice_heuristic == "random":
pattern_heuristic = makeRandomPatternHeuristic(encoded_weights)
print(loc_heuristic)
location_heuristic = lexicalLocationHeuristic
if loc_heuristic == "anti-entropy":
location_heuristic = makeAntiEntropyLocationHeuristic(
choice_random_weighting)
if loc_heuristic == "entropy":
location_heuristic = makeEntropyLocationHeuristic(
choice_random_weighting)
if loc_heuristic == "random":
location_heuristic = makeRandomLocationHeuristic(
choice_random_weighting)
if loc_heuristic == "simple":
location_heuristic = simpleLocationHeuristic
if loc_heuristic == "spiral":
location_heuristic = makeSpiralLocationHeuristic(
choice_random_weighting)
if loc_heuristic == "hilbert":
location_heuristic = makeHilbertLocationHeuristic(
choice_random_weighting)
### Visualization ###
visualize_choice, visualize_wave, visualize_backtracking, visualize_propagate, visualize_final, visualize_after = None, None, None, None, None, None
if visualize:
visualize_choice, visualize_wave, visualize_backtracking, visualize_propagate, visualize_final, visualize_after = make_solver_visualizers(
f"{filename}_{timecode}",
wave,
decode_patterns=decode_patterns,
pattern_catalog=pattern_catalog,
tile_catalog=tile_catalog,
tile_size=[tile_size, tile_size])
if logging:
visualize_choice, visualize_wave, visualize_backtracking, visualize_propagate, visualize_final, visualize_after = make_solver_loggers(
f"{filename}_{timecode}", input_stats.copy())
if logging and visualize:
vis = make_solver_visualizers(f"{filename}_{timecode}",
wave,
decode_patterns=decode_patterns,
pattern_catalog=pattern_catalog,
tile_catalog=tile_catalog,
tile_size=[tile_size, tile_size])
log = make_solver_loggers(f"{filename}_{timecode}", input_stats.copy())
def visfunc(idx):
def vf(*args, **kwargs):
if vis[idx]:
vis[idx](*args, **kwargs)
if log[idx]:
return log[idx](*args, **kwargs)
return vf
visualize_choice, visualize_wave, visualize_backtracking, visualize_propagate, visualize_final, visualize_after = [
visfunc(x) for x in range(len(vis))
]
### Global Constraints ###
active_global_constraint = lambda wave: True
if global_constraint == "allpatterns":
active_global_constraint = make_global_use_all_patterns()
print(active_global_constraint)
### Search Depth Limit
def makeSearchLengthLimit(max_limit):
search_length_counter = 0
def searchLengthLimit(wave):
nonlocal search_length_counter
search_length_counter += 1
return search_length_counter <= max_limit
return searchLengthLimit
combined_constraints = [
active_global_constraint,
makeSearchLengthLimit(1200)
]
def combinedConstraints(wave):
print
return all([fn(wave) for fn in combined_constraints])
### Solving ###
time_solve_start = None
time_solve_end = None
solution_tile_grid = None
print("solving...")
attempts = 0
while attempts < attempt_limit:
attempts += 1
end_early = False
time_solve_start = time.time()
stats = {}
profiler = pprofile.Profile()
if True:
#with profiler:
#with PyCallGraph(output=GraphvizOutput(output_file=f"visualization/pycallgraph_{filename}_{timecode}.png")):
try:
solution = run(wave.copy(),
adjacency_matrix,
locationHeuristic=location_heuristic,
patternHeuristic=pattern_heuristic,
periodic=output_periodic,
backtracking=backtracking,
onChoice=visualize_choice,
onBacktrack=visualize_backtracking,
onObserve=visualize_wave,
onPropagate=visualize_propagate,
onFinal=visualize_final,
checkFeasible=combinedConstraints)
if visualize_after:
stats = visualize_after()
#print(solution)
#print(stats)
solution_as_ids = np.vectorize(lambda x: decode_patterns[x])(
solution)
solution_tile_grid = pattern_grid_to_tiles(
solution_as_ids, pattern_catalog)
print("Solution:")
#print(solution_tile_grid)
render_tiles_to_output(
solution_tile_grid, tile_catalog, [tile_size, tile_size],
output_destination + filename + "_" + timecode + ".png")
time_solve_end = time.time()
stats.update({"outcome": "success"})
succeeded = True
except StopEarly:
print("Skipping...")
end_early = True
stats.update({"outcome": "skipped"})
except TimedOut as e_c:
print("Timed Out")
if visualize_after:
stats = visualize_after()
stats.update({"outcome": "timed_out"})
except Contradiction as e_c:
print("Contradiction")
if visualize_after:
stats = visualize_after()
stats.update({"outcome": "contradiction"})
profiler.dump_stats(f"logs/profile_{filename}_{timecode}.txt")
outstats = {}
outstats.update(input_stats)
solve_duration = time.time() - time_solve_start
try:
solve_duration = (time_solve_end - time_solve_start)
except TypeError:
pass
adjacency_duration = 0
try:
adjacency_duration = time_solve_start - time_adjacency
except TypeError:
pass
outstats.update({
"attempts": attempts,
"time_start": time_begin,
"time_adjacency": time_adjacency,
"adjacency_duration": adjacency_duration,
"time solve start": time_solve_start,
"time solve end": time_solve_end,
"solve duration": solve_duration,
"pattern count": number_of_patterns
})
outstats.update(stats)
if not log_stats_to_output is None:
log_stats_to_output(outstats,
output_destination + log_filename + ".tsv")
if not solution_tile_grid is None:
return solution_tile_grid
if end_early:
return None
return None
#%% Script
if __name__ == '__main__':
do_board = 'none' # from {'none', 'small', 'large', 'both'}
# convert board to numeric representation for efficiency
board1 = knight.char_board_to_nums(knight.BOARD1)
board2 = knight.char_board_to_nums(knight.BOARD2)
# create the output folder
folder = os.path.join(get_output_dir(), 'knight')
setup_dir(folder)
# create profiler
profile = pprofile.Profile()
# Small board
if do_board in {'small', 'both'}:
print('\nSolve the smaller board for the minimum length solution.')
# enable/disable profiler while running solver
profile.enable()
moves3 = knight.solve_min_puzzle(board1)
profile.disable()
# print solution
print(moves3)
is_valid3 = knight.check_valid_sequence(board1, moves3, print_status=True)
if is_valid3:
knight.print_sequence(board1, moves3)
