def __init__(self):
settings_manager = SettingsManager() # Set up the settings_manager
max_workers = settings_manager.getint('application', 'max-workers') # Get the max workers from settings manager
profiler_on = settings_manager.getint('debugging', 'profiler-on') # Get whether there is a profiler
absolute = settings_manager.getint('save', 'absolute') # Get whether it's an absolute path
save_path = settings_manager.get('save', 'path') # Get whether it's an absolute path
if not absolute:
save_path = PROJECT_PATH + os.path.sep + save_path
executor = ThreadPoolExecutor(max_workers=max_workers, profiler_on=profiler_on) # Set up the thread executor
dis = Disassembler(settings_manager) # Build the disassembler
server = PyDAServer('0.0.0.0',9000) # Set up the PyDA server
save_manager = SaveManager(save_path)
if profiler_on:
profile = Profile()
profile.enable()
app.build_and_run(settings_manager, dis, executor, server, save_manager) # Run the interface
if profiler_on:
profile.disable()
stats = executor.getProfileStats()
if stats == None:
stats = Stats(profile)
else:
stats.add(profile)
with open('profile.stats', 'wb') as statsfile:
stats.stream = statsfile
stats.sort_stats('cumulative').print_stats()
def _run_unit(self, unit, set_title=False):
try:
if set_title:
setproctitle('coordinate worker {0!r} {1!r}'
.format(unit.work_spec_name, unit.key))
profiler = None
if ((self.profile_destination and
unit.work_spec_name in self.profile_work_specs)):
now = datetime.now()
unit_info = {
'work_spec_name': unit.work_spec_name,
'work_unit_key': unit.key,
'ymd': now.strftime('%Y%m%d'),
'hms': now.strftime('%H%M%S'),
}
destination = self.profile_destination % unit_info
profiler = Profile()
profiler.enable()
unit.run()
if profiler:
profiler.disable()
profiler.dump_stats(destination)
unit.finish()
except LostLease:
# We don't own the unit any more so don't try to report on it
logger.warn('Lost Lease on %r %r', unit.work_spec_name, unit.key)
except Exception, exc: # pylint: disable=broad-except
unit.fail(exc)
def profile(to=None, sort_by='cumtime'):
'''Profiles a chunk of code, use with the ``with`` statement::
from halonctl.debug import profile
with profile('~/Desktop/stats'):
pass # Do something performance-critical here...
Results for individual runs are collected into ``to``. The specifics of how
reports are done varies depending on what type ``to`` is.
* **File-like objects**: Stats are dumped, according to ``sort_by``, into the stream, separated by newlines - watch out, the file/buffer may grow very big when used in loops.
* **List-like objects**: A number of pstats.Stats objects are appended.
* **str and unicode**: Treated as a path and opened for appending. Tildes (~) will be expanded, and intermediary directories created if possible.
* **None or omitted**: Results are printed to sys.stderr.
'''
if isinstance(to, six.string_types):
to = open_fuzzy(to, 'a')
to_is_stream = hasattr(to, 'write')
to_is_list = hasattr(to, 'append')
p = Profile()
p.enable()
yield
p.disable()
ps = Stats(p, stream=to if to_is_stream else sys.stderr)
ps.sort_stats('cumtime')
if to_is_stream or to is None:
ps.print_stats()
elif to_is_list:
to.append(ps)
def main():
config = ConfigParser()
try:
config.read(os.path.expanduser("~/.plug/web.conf"))
except Exception as err:
print "Failed to parse config:", str(err)
return 1
port = 8008
if "baseconfig" in config.sections() and "port" in config.options("baseconfig"):
port = int(config.get("baseconfig", "port"))
if "TREEBUG" in os.environ:
last_int = 0
prof = Profile()
while True:
prof.enable()
try:
serve(port=8008)
except KeyboardInterrupt:
if time.time() - last_int < 1:
break
prof.disable()
output = StringIO()
ps = pstats.Stats(prof, stream=output).sort_stats("cumulative")
ps.print_stats()
print "\n".join([line for line in output.getvalue().splitlines()
if not ("/usr/lib" in line or "{" in line)])
output.close()
prof = Profile()
last_int = time.time()
else:
serve(port=port)
def profile(func, max_epochs=15, use_GPU=False, cprofile=True):
"""Run a profiler on the code."""
if cprofile:
p = Profile()
p.enable()
else:
import pycuda
pycuda.driver.start_profiler()
if func == "mnist":
mnist({'use_GPU': use_GPU, 'rng': np.random.RandomState(0)},
{'max_epochs': max_epochs, 'plotting': False, 'batch_size': 7500,
'CG_iter': 10})
elif func == "integrator":
integrator({'shape': [1, 100, 1], 'layers': hf.nl.Logistic(),
'use_GPU': use_GPU, 'debug': False,
'rng': np.random.RandomState(0)},
{'max_epochs': max_epochs, 'CG_iter': 10},
n_inputs=500, sig_len=200, plots=False)
else:
raise ValueError("Unknown profile function")
if cprofile:
p.disable()
ps = pstats.Stats(p)
ps.strip_dirs().sort_stats('time').print_stats(20)
else:
pycuda.driver.stop_profiler()
def build_problem(main_img, patterns_imgs, Semiring=SemiringArgminPlusElement):
image = MatrixPointer(list(main_img.getdata()), (main_img.size[0], main_img.size[1]))
patterns = dict()
for p in patterns_imgs:
patterns[p] = MatrixPointer(list(patterns_imgs[p].getdata()), (patterns_imgs[p].size[0], patterns_imgs[p].size[1]))
start = Vertex('start')
end = Vertex('end')
fake_edge = Edge(start, end, Semiring.get_zero())
min_width = min(patterns[p].get_size()[0] for p in patterns)
if min_width > image.get_size()[0]:
return Graph([start, end], fake_edge)
profile = Profile()
profile.enable()
vertices, edges = process_img(image, patterns, start)
profile.disable()
Stats(profile).sort_stats('time').print_stats()
vertices['start'] = [start]
vertices['end'] = [end]
if image.get_size()[0] in vertices:
edges = edges.union(Edge(v, end, Semiring.get_unity())
for v in vertices[image.get_size()[0]])
problem = DynamicProgramming(sum(vertices.values(), []), edges)
problem.set_start(start)
problem.set_finish(end)
return problem
def profile(filename, log=None):
"""!Context manager for profiling with cProfile
@param filename filename to which to write profile (profiling disabled if None or empty)
@param log log object for logging the profile operations
If profiling is enabled, the context manager returns the cProfile.Profile object (otherwise
it returns None), which allows additional control over profiling. You can obtain this using
the "as" clause, e.g.:
with profile(filename) as prof:
runYourCodeHere()
The output cumulative profile can be printed with a command-line like:
python -c 'import pstats; pstats.Stats("<filename>").sort_stats("cumtime").print_stats(30)'
"""
if not filename:
# Nothing to do
yield
return
from cProfile import Profile
prof = Profile()
if log is not None:
log.info("Enabling cProfile profiling")
prof.enable()
yield prof
prof.disable()
prof.dump_stats(filename)
if log is not None:
log.info("cProfile stats written to %s" % filename)
class BenchTestResult(unittest.runner.TextTestResult):
"""
A textual test result formatter that can display additional information
such as profile output and benchmarks.
"""
def __init__(self, arguments, stream, descriptions, verbosity):
super(BenchTestResult, self).__init__(stream, descriptions, verbosity)
settings = arguments.get_settings("test_result")
self._sort = settings.get("profile_sort")
self._limit = settings.get("profile_limit")
self._benchmark = verbosity > 2
self._profiler = None
def startTest(self, test):
super(BenchTestResult, self).startTest(test)
if self._benchmark:
self._profiler = Profile()
self._profiler.enable()
def stopTest(self, test):
super(BenchTestResult, self).stopTest(test)
if self._benchmark:
self._profiler.disable()
stats = Stats(self._profiler)
stats.sort_stats(self._sort)
stats.print_stats(self._limit)
def profile(func, file_path):
pr = Profile()
pr.enable()
func()
pr.disable()
s = open(file_path, "w")
sortby = "cumulative"
ps = Stats(pr, stream=s).sort_stats(sortby)
ps.print_stats()
def profiled(*args, **kargs):
profile = Profile()
profile.enable()
func(*args, **kargs)
profile.disable()
ident = current_thread().ident
profile.dump_stats("/tmp/%s.%s.%i.pstat" % (
hs.hostname, func.__name__, ident
))
def profiled_func(*args, **kwargs):
p = Profile()
try:
# profile the input function
p.enable()
r = f(*args, **kwargs)
p.disable()
return r
finally:
p.print_stats()
def test_profile_rdg_generation():
profiler = Profile()
profiler.enable()
for _ in range(5):
level = Level(generation_type=LevelGen.Dungeon)
generate_tiles_to(level)
profiler.disable()
profile_util.write_results_log(profiler)
def wrapper(*args, **kwargs):
profile = Profile()
profile.enable()
try:
func(*args, **kwargs)
finally:
profile.disable()
try:
thread = current_thread()
profile.dump_stats('profile_%s.%s.%s.log' % (getpid(), thread.name, thread.ident))
except:
logger.exception('Failed to dump stats')
def profiler(enable, outfile):
try:
if enable:
profiler = Profile()
profiler.enable()
yield
finally:
if enable:
profiler.disable()
stats = Stats(profiler)
stats.sort_stats('tottime')
stats.dump_stats(outfile)
def test(algo, size=10000):
mylist = random_list(size)
profile = Profile()
profile.enable()
result = profile.runcall(algo, mylist)
profile.disable()
stringStream = StringIO.StringIO()
sortby = 'cumulative'
ps = pstats.Stats(profile, stream=stringStream).sort_stats(sortby)
ps.print_stats()
print
print "\t\t\t\t{0} sorting {1} ints".format(algo.__name__, size)
print stringStream.getvalue()
def _wrap(*kl, **kw):
prof = Profile()
prof.enable()
try:
return func(*kl, **kw)
finally:
prof.disable()
result = StringIO()
ps = Stats(prof, stream = result).sort_stats('cumulative')
ps.print_stats()
print result.getvalue()
def pytest_runtest_call(item):
if SHOULD_PROFILE:
p = Profile()
p.enable()
yield
p.disable()
stats = Stats(p)
if SHOULD_PRINT:
stats.sort_stats('cumulative').print_stats(50)
if SHOULD_STORE:
if not os.path.exists(BASEDIR):
os.mkdir(BASEDIR)
p.dump_stats(os.path.join(BASEDIR, '%s.pkl' % item.name))
else:
yield
def do(impl, count):
# warm-up
_res = [impl() for _ in _range(count)]
profile = Profile()
profile.enable()
res = [impl() for _ in _range(count)]
profile.disable()
out = StringIO()
stats = Stats(profile, stream=out)
stats.strip_dirs()
stats.sort_stats('calls').print_stats(10)
print(out.getvalue().lstrip())
out.close()
return _res, res
class BenchmarkThread(Thread):
def __init__(self, thread_num, session, query, values, num_queries, profile):
Thread.__init__(self)
self.thread_num = thread_num
self.session = session
self.query = query
self.values = values
self.num_queries = num_queries
self.profiler = Profile() if profile else None
def start_profile(self):
if self.profiler:
self.profiler.enable()
def finish_profile(self):
if self.profiler:
self.profiler.disable()
self.profiler.dump_stats("profile-%d" % self.thread_num)
def compare_phi():
"""
Problem 38
>>> a, b = compare_phi()
>>> a > b
True
"""
from cProfile import Profile
from pstats import Stats
pr_normal = Profile()
pr_normal.enable()
phi(10090)
pr_normal.disable()
time_normal = Stats(pr_normal).total_tt
pr_improved = Profile()
pr_improved.enable()
phi_improved(10090)
pr_improved.disable()
time_improved = Stats(pr_improved).total_tt
return time_normal, time_improved
def profile_rnn_calc_G(cprofile=True):
"""Run a profiler on the recurrent curvature calculation.
:param bool cprofile: use True if profiling on the CPU, False if using the
CUDA profiler
"""
inputs = np.random.randn(1024, 128, 1).astype(np.float32)
targets = np.random.randn(1024, 128, 1).astype(np.float32)
N = 128
rnn = hf.RNNet([1, N, 1], use_GPU=True)
rnn.optimizer = hf.opt.HessianFree() # for struc_damping check
rnn.cache_minibatch(inputs, targets)
v = np.random.randn(rnn.W.size).astype(np.float32)
for _ in range(2):
# run it a few times to get rid of any startup overhead
rnn.GPU_calc_G(v)
if cprofile:
start = time.time()
p = Profile()
p.enable()
else:
pycuda.driver.start_profiler()
for _ in range(100):
_ = rnn.GPU_calc_G(v)
if cprofile:
p.disable()
print("time", time.time() - start)
ps = pstats.Stats(p)
ps.strip_dirs().sort_stats('time').print_stats(20)
else:
pycuda.driver.stop_profiler()
def test_ensemble(self,test_file):
assert self.authenticated, 'Not authenticated!'
# download a local copy of the ensemble
self.logger.info('Creating local ensemble')
local_ensemble = Ensemble(self.ensemble_res,api=self.api)
# make the Fields object
source = self.api.get_source(self.source_res)
fields = Fields(source['object']['fields'])
self.logger.info('Reading test data and generating predictions')
true_labels = []
predict_labels = []
pr = Profile()
pr.enable()
with open(test_file) as fid:
test_reader = csv.reader(fid)
# skip the header line
test_reader.next()
for row in test_reader:
row_list = [val for val in row]
true_labels.append(row_list.pop())
instance = fields.pair(row_list)
predict_labels.append(local_ensemble.predict(instance,
by_name=False,
method=1))
pr.disable()
ps = Stats(pr)
self.predict_time = ps.total_tt
# eval_args = {'combiner':1}
# evaluation = self.api.create_evaluation(self.ensemble_res,test_data,eval_args)
# check_resource(evaluation['resource'],self.api.get_evaluation)
# evaluation = self.api.get_evaluation(evaluation['resource'])
# matrix = evaluation['object']['result']['model']['confusion_matrix']
# self.predict_time = evaluation['object']['status']['elapsed']/1000
if self.regression:
self.results = (predict_labels,true_labels)
else:
self.results = make_confusion_matrix(true_labels,predict_labels)
def profile_dot(cprofile=True):
"""Run a profiler on the matrix multiplication kernel.
:param bool cprofile: use True if profiling on the CPU, False if using the
CUDA profiler
"""
N = 1024
a = np.random.randn(N, N).astype(np.float32)
b = np.random.randn(N, N).astype(np.float32)
a_gpu = gpuarray.to_gpu(a)
b_gpu = gpuarray.to_gpu(b)
c_gpu = gpuarray.zeros((N, N), np.float32)
for _ in range(2):
# run it a few times to get rid of any startup overhead
hf.gpu.dot(a_gpu, b_gpu, out=c_gpu)
if cprofile:
start = time.time()
p = Profile()
p.enable()
else:
pycuda.autoinit.context.synchronize()
pycuda.driver.start_profiler()
for _ in range(100):
hf.gpu.dot(a_gpu, b_gpu, out=c_gpu, transpose_a=True,
transpose_b=True)
c_gpu.get()
if cprofile:
p.disable()
print("time", time.time() - start)
ps = pstats.Stats(p)
ps.strip_dirs().sort_stats('time').print_stats(20)
else:
pycuda.driver.stop_profiler()
def profile_calc_G(cprofile=True):
"""Run a profiler on the feedforward curvature calculation.
:param bool cprofile: use True if profiling on the CPU, False if using the
CUDA profiler
"""
inputs = np.random.randn(1024, 1).astype(np.float32)
targets = np.random.randn(1024, 1).astype(np.float32)
N = 1024
ff = hf.FFNet([1, N, N, 1], use_GPU=True)
ff.cache_minibatch(inputs, targets)
v = np.random.randn(ff.W.size).astype(np.float32)
for _ in range(5):
# run it a few times to get rid of any startup overhead
ff.GPU_calc_G(v)
if cprofile:
start = time.time()
p = Profile()
p.enable()
else:
pycuda.driver.start_profiler()
for _ in range(500):
_ = ff.GPU_calc_G(v)
if cprofile:
p.disable()
print("time", time.time() - start)
ps = pstats.Stats(p)
ps.strip_dirs().sort_stats('time').print_stats(20)
else:
pycuda.driver.stop_profiler()
class BenchmarkThread(Thread):
def __init__(self, thread_num, session, query, values, num_queries, protocol_version, profile):
Thread.__init__(self)
self.thread_num = thread_num
self.session = session
self.query = query
self.values = values
self.num_queries = num_queries
self.protocol_version = protocol_version
self.profiler = Profile() if profile else None
def start_profile(self):
if self.profiler:
self.profiler.enable()
def run_query(self, key, **kwargs):
return self.session.execute_async(self.query.format(key=key), **kwargs)
def finish_profile(self):
if self.profiler:
self.profiler.disable()
self.profiler.dump_stats("profile-%d" % self.thread_num)
def profile(func, max_epochs=1, use_GPU=False, cprofile=True):
"""Run a profiler on the code.
:param str func: the demo function to be profiled (can be 'mnist' or
'integrator')
:param int max_epochs: maximum number of iterations to run
:param bool use_GPU: run optimization on GPU
:param bool cprofile: if True then run the profiling on the CPU, otherwise
use CUDA profiler
"""
if cprofile:
p = Profile()
p.enable()
else:
import pycuda
pycuda.driver.start_profiler()
if func == "mnist":
mnist({'use_GPU': use_GPU, 'rng': np.random.RandomState(0)},
{'max_epochs': max_epochs, 'plotting': False,
'minibatch_size': 7500, 'CG_iter': 10})
elif func == "integrator":
integrator({'shape': [1, 100, 1], 'layers': hf.nl.Logistic(),
'use_GPU': use_GPU, 'debug': False,
'rng': np.random.RandomState(0)},
{'max_epochs': max_epochs, 'CG_iter': 10},
n_inputs=500, sig_len=200, plots=False)
else:
raise ValueError("Unknown profile function")
if cprofile:
p.disable()
ps = pstats.Stats(p)
ps.strip_dirs().sort_stats('time').print_stats(20)
else:
pycuda.driver.stop_profiler()
class Monitor:
def __init__(self,
monitoring_dir: str,
with_runtime: bool = False,
with_profiler: bool = False):
self._monitoring_dir = monitoring_dir
self._duration_sampler = _new_duration_sampler(monitoring_dir)
self._runtime_bucket = None
self._profiler = None
self._lock = Lock()
if with_runtime:
self._runtime_bucket = _start_runtime_sampler(monitoring_dir)
if with_profiler:
self._profiler = Profile()
self._profiler.enable()
def monitoring_dir(self) -> str:
return self._monitoring_dir
def start_duration_sample(self) -> str:
return self._duration_sampler.sample()
def stop_duration_sample(self, label: str, sample_id: str):
self._duration_sampler.collect(label, sample_id)
def stop(self):
if self._profiler:
with self._lock:
self._profiler.disable()
outfile = _profiler_file_pathname(self._monitoring_dir)
self._profiler.dump_stats(outfile)
self._duration_sampler.bucket().empty()
if self._runtime_bucket:
self._runtime_bucket.empty()
def run(self) -> StatusCode:
"""Run process manager.
:return: Status code of run execution.
:rtype: StatusCode
"""
# Initialize
status = self.initialize()
# Execute
profiler = None
if status == StatusCode.Success:
settings = self.service(ConfigObject)
profile_code = settings.get('doCodeProfiling', False)
if profile_code:
from cProfile import Profile
profiler = Profile()
profiler.enable()
status = self.execute()
if profiler:
profiler.disable()
# Finalize
if status == StatusCode.Success:
status = self.finalize()
# Profiling output
if profiler:
import io
import pstats
profile_output = io.StringIO()
profile_stats = pstats.Stats(profiler, stream=profile_output).sort_stats('time')
profile_stats.print_stats()
self.logger.info('Profiling Statistics (sorted by time):\n{stats}', stats=profile_output.getvalue())
return status
def __run(self, *args, **kwargs):
__start = time.time()
# notify if we don't process quickly
if __start - self.__time_submitted > 0.05:
self.log.warning(
f'Starting of {self.name} took too long: {__start - self.__time_submitted:.2f}s. '
f'Maybe there are not enough threads?')
# start profiler
pr = Profile()
pr.enable()
# Execute the function
try:
self._func(*args, **kwargs)
except Exception as e:
self.__format_traceback(e, *args, **kwargs)
# disable profiler
pr.disable()
# log warning if execution takes too long
__dur = time.time() - __start
if self.__warn_too_long and __dur > 0.8:
self.log.warning(
f'Execution of {self.name} took too long: {__dur:.2f}s')
s = io.StringIO()
ps = Stats(pr, stream=s).sort_stats(STAT_SORT_KEY)
ps.print_stats(0.1) # limit to output to 10% of the lines
for line in s.getvalue().splitlines(
)[4:]: # skip the amount of calls and "Ordered by:"
if line:
self.log.warning(line)
def _test_overhead_timing(dim=15):
# prints timing for simple Gaussian vanilla mcmc
import pstats
from cProfile import Profile
from io import StringIO
# noinspection PyUnresolvedReferences
from cobaya.samplers.mcmc import proposal # one-time numba compile out of profiling
LikeTest = _make_gaussian_like(dim)
info = {
'likelihood': {
'like': LikeTest
},
'debug': False,
'sampler': {
'mcmc': {
'max_samples': 1000,
'burn_in': 0,
"learn_proposal": False,
"Rminus1_stop": 0.0001
}
}
}
prof = Profile()
prof.enable()
run(info)
prof.disable()
# prof.dump_stats("out.prof") # to visualize with e.g. snakeviz
s = StringIO()
ps = pstats.Stats(prof, stream=s)
print_n_calls = 10
ps.strip_dirs()
ps.sort_stats('time')
ps.print_stats(print_n_calls)
ps.sort_stats('cumtime')
ps.print_stats(print_n_calls)
print(s.getvalue())
def main():
config = ConfigParser()
try:
config.read(os.path.expanduser("~/.plug/web.conf"))
except Exception as err:
print "Failed to parse config:", str(err)
return 1
port = 8008
if "baseconfig" in config.sections() and "port" in config.options(
"baseconfig"):
port = int(config.get("baseconfig", "port"))
if "TREEBUG" in os.environ:
last_int = 0
prof = Profile()
while True:
prof.enable()
try:
serve(port=8008)
except KeyboardInterrupt:
if time.time() - last_int < 1:
break
prof.disable()
output = StringIO()
ps = pstats.Stats(prof, stream=output).sort_stats("cumulative")
ps.print_stats()
print "\n".join([
line for line in output.getvalue().splitlines()
if not ("/usr/lib" in line or "{" in line)
])
output.close()
prof = Profile()
last_int = time.time()
else:
serve(port=port)
def perform(self):
"""Run sequentially all child tasks, and close ressources.
"""
result = True
self.thread_id = threading.current_thread().ident
self.prepare()
pr = Profile() if self.should_profile else None
try:
if pr:
pr.enable()
for child in self.children:
child.perform_()
except Exception:
log = logging.getLogger(__name__)
msg = 'The following unhandled exception occured :\n'
log.exception(msg)
self.should_stop.set()
result = False
self.errors['unhandled'] = msg + format_exc()
finally:
if pr:
pr.disable()
meas_name = self.get_from_database('meas_name')
meas_id = self.get_from_database('meas_id')
path = os.path.join(self.default_path,
meas_name + '_' + meas_id + '.prof')
pr.dump_stats(path)
self.release_resources()
if self.should_stop.is_set():
result = False
return result
def perform(self):
"""Run sequentially all child tasks, and close ressources.
"""
result = True
self.thread_id = threading.current_thread().ident
self.prepare()
pr = Profile() if self.should_profile else None
try:
if pr:
pr.enable()
for child in self.children:
child.perform_()
except Exception:
log = logging.getLogger(__name__)
msg = 'The following unhandled exception occured :\n'
log.exception(msg)
self.should_stop.set()
result = False
self.errors['unhandled'] = msg + format_exc()
finally:
if pr:
pr.disable()
meas_name = self.get_from_database('meas_name')
meas_id = self.get_from_database('meas_id')
path = os.path.join(self.default_path,
meas_name + '_' + meas_id + '.prof')
pr.dump_stats(path)
self.release_resources()
if self.should_stop.is_set():
result = False
return result
def with_profiling(f, stats_out=None, text_out=None):
try: from cProfile import Profile
except: from profile import Profile
from pstats import Stats
prof = Profile()
prof.enable()
retval = f()
prof.disable()
if stats_out:
try:
prof.dump_stats(stats_out)
except (IOError,OSError) as e: # not worth losing our return value over
warn('could not write pstats (%s)', e)
if text_out:
try:
MAX_LINES = 20
stats = Stats(prof, stream=text_out)
stats.sort_stats('cumtime')
stats.print_stats(MAX_LINES)
except (IOError,OSError) as e:
warn('could not write profiling summary (%s)', e)
return retval
def profile_stop(
profiler: Profile,
sort_field: str = "cumulative",
dump_filename: str = None,
dump_line_count: int = 100,
) -> None:
"""Stop a given profiler and print or dump stats.
Args:
profiler:
Active profiling instance.
sort_field:
pstats.Stats sort field for ordering profiling results.
dump_filename:
File path to dump profiling stats into.
dump_line_count:
Maximum lines to print out.
"""
profiler.disable()
stats = pstats.Stats(profiler, stream=sys.stderr).sort_stats(sort_field)
if dump_filename is None:
stats.print_stats(dump_line_count)
else:
stats.dump_stats(dump_filename)
def _call_wrapped(self, function, idict, odict):
"""Does the same as _call, but with profiling enabled.
"""
args = idict['args']
kwargs = idict['kwargs']
try:
profile = Profile()
profile.enable()
data = function(*args, **kwargs)
profile.disable()
stream = StringIO()
stats = pstats.Stats(profile, stream=stream)
stats.strip_dirs()
stats.sort_stats(1)
stats.print_stats(20)
except Exception:
odict['error'] = traceback.format_exc()
else:
odict['data'] = data
odict['profile'] = stream.getvalue()
def main():
"""
>>> p = PrimeStore()
>>> p.is_prime(3)
True
>>> p.is_prime(99)
False
>>> p.is_prime(9)
False
"""
from cProfile import Profile
from pstats import Stats
from random import randint
p = PrimeStore()
pr = Profile()
pr.enable()
for i in [randint(0, 10000) for _ in range(1000)]:
p.is_prime(i)
pr.disable()
stats = Stats(pr)
assert stats.total_tt < 0.100
def profile(filename, log=None):
"""Context manager for profiling with cProfile.
Parameters
----------
filename : `str`
Filename to which to write profile (profiling disabled if `None` or empty).
log : `lsst.log.Log`, optional
Log object for logging the profile operations.
If profiling is enabled, the context manager returns the cProfile.Profile object (otherwise
it returns None), which allows additional control over profiling. You can obtain this using
the "as" clause, e.g.:
with profile(filename) as prof:
runYourCodeHere()
The output cumulative profile can be printed with a command-line like::
python -c 'import pstats; pstats.Stats("<filename>").sort_stats("cumtime").print_stats(30)'
"""
if not filename:
# Nothing to do
yield
return
from cProfile import Profile
profile = Profile()
if log is not None:
log.info("Enabling cProfile profiling")
profile.enable()
yield profile
profile.disable()
profile.dump_stats(filename)
if log is not None:
log.info("cProfile stats written to %s" % filename)
def run(self):
watch = StopWatch()
watch.tag('inference', self.verbose)
# load the MLN
if isinstance(self.mln, MLN):
mln = self.mln
else:
raise Exception('No MLN specified')
if self.use_emln and self.emln is not None:
mlnstrio = io.StringIO()
mln.write(mlnstrio)
mlnstr = mlnstrio.getvalue()
mlnstrio.close()
emln = self.emln
mln = parse_mln(mlnstr + emln,
grammar=self.grammar,
logic=self.logic)
# load the database
if isinstance(self.db, Database):
db = self.db
elif isinstance(self.db, list) and len(self.db) == 1:
db = self.db[0]
elif isinstance(self.db, list) and len(self.db) == 0:
db = Database(mln)
elif isinstance(self.db, list):
raise Exception(
'Got {} dbs. Can only handle one for inference.'.format(
len(self.db)))
else:
raise Exception('DB of invalid format {}'.format(type(self.db)))
# expand the
# parameters
params = dict(self._config)
if 'params' in params:
params.update(eval("dict(%s)" % params['params']))
del params['params']
params['verbose'] = self.verbose
if self.verbose:
print((tabulate(sorted(list(params.items()),
key=lambda k_v: str(k_v[0])),
headers=('Parameter:', 'Value:'))))
if type(db) is list and len(db) > 1:
raise Exception('Inference can only handle one database at a time')
elif type(db) is list:
db = db[0]
params['cw_preds'] = [x for x in self.cw_preds if bool(x)]
# extract and remove all non-algorithm
for s in GUI_SETTINGS:
if s in params: del params[s]
if self.profile:
prof = Profile()
print('starting profiler...')
prof.enable()
# set the debug level
olddebug = logger.level
logger.level = (eval('logs.%s' %
params.get('debug', 'WARNING').upper()))
result = None
try:
mln_ = mln.materialize(db)
mrf = mln_.ground(db)
inference = self.method(mrf, self.queries, **params)
if self.verbose:
print()
print((headline('EVIDENCE VARIABLES')))
print()
mrf.print_evidence_vars()
result = inference.run()
if self.verbose:
print()
print((headline('INFERENCE RESULTS')))
print()
inference.write()
if self.verbose:
print()
inference.write_elapsed_time()
except SystemExit:
traceback.print_exc()
print('Cancelled...')
finally:
if self.profile:
prof.disable()
print((headline('PROFILER STATISTICS')))
ps = pstats.Stats(prof,
stream=sys.stdout).sort_stats('cumulative')
ps.print_stats()
# reset the debug level
logger.level = olddebug
if self.verbose:
print()
watch.finish()
watch.printSteps()
return result
a, b, c = Vertex(3)
ase = AbstractStateExplorer(Constraint(), [
Action(
'sort_ball', Graph([
Edge('Ball', (a, )),
Edge('Hand', (b, )),
]),
Graph([
Edge('Ball', (a, )),
Edge('Sorted', (a, )),
Edge('Hand', (b, )),
])),
Action('buy_orange', Graph([]), Graph([
Edge('Orange', (a, )),
]))
])
from cProfile import Profile
from pstats import Stats
p = Profile()
p.enable()
for i in range(10):
ase.compile(i)
print(len(ase.CompoundActionsList))
p.disable()
#sol, end_state = ase.find_solution(g) #right now it just returns the first action it applies just because anything meets the constraints
#print('solution', sol.CompoundActionTracker)
#print(end_state)
#Stats(p).sort_stats('cumtime').print_stats()
# Build state information
# default is <none of these> for all variables
state_variables = [num_var_values[i] - 1 for i in range(num_vars)]
for state_item in xml_root:
assert state_item.tag == 'atom',\
"Cannot handle non-atom state items yet (got %s)" % \
(state_item.tag)
current_atom = state_item.find('predicate').text + "(" + \
", ".join([t.text for t in state_item.findall('term')]) + ")"
assert current_atom in atom_to_var_value,\
"SAS+ variable for %s unknown" % current_atom
var_index, var_value = atom_to_var_value[current_atom]
state_variables[var_index] = var_value
if (profiling):
build_state_profiler.disable()
if (verbosity > 3):
stdout.write("Current state variables:\n")
for (i, sv) in enumerate(state_variables):
stdout.write("\tvar%d: %d\n" % (i, sv))
if (profiling):
profiling_call_accumulator = 0
get_actions_profiler.enable()
action = None
actions_to_consider = policy.get_actions(state_variables)
actions_to_avoid = de_policy.get_actions(state_variables)
actions_valid = [
a for a in actions_to_consider
class Measure:
time_factor = 1000
def __init__(self,
measure_session=None,
measure_point_name=None,
save_queue=None,
data=None):
if data is None:
self.measure_session = measure_session
self.point_name = measure_point_name
self._save_queue = save_queue
self._profile = Profile()
else:
self.measure_session = data[0]
self.point_name = data[1]
self.load(data[2])
def load(self, data):
self.timings = data
def start(self):
self._profile.enable()
return self
def stop(self):
self._profile.disable()
self.stats = self._profile.getstats()
self.timings = list()
self.timestamp = datetime.utcnow()
for stat in self.stats:
if stat.calls is not None:
calls = list()
for call in stat.calls:
calls.append(
TimingStat(self.timestamp, str(self.measure_session),
str(self.point_name), str(call.code),
call.callcount, call.reccallcount,
call.totaltime * time_factor,
call.inlinetime * time_factor, None))
else:
calls = None
self.timings.append(
TimingStat(self.timestamp, str(self.measure_session),
str(self.point_name), str(stat.code),
stat.callcount, stat.reccallcount,
stat.totaltime * time_factor,
stat.inlinetime * time_factor, calls))
return self
def save(self):
self._save_queue.put_nowait(
(self.measure_session, self.point_name, self.timings))
return self
def __str__(self):
buf = "<{0}: measure_session='{1}' point_name='{1}'".format(
self.__class__, self.measure_session, self.point_name)
if hasattr(self, "stat"):
buf += ", stat='{}'".format(self.stats)
if hasattr(self, "timing"):
buf += ", timing='{}'".format(self.timings)
buf += ">"
return buf
class run_info_dict(dict):
"""
Simple dictionary class for collecting runtime information
The typical use is as follows:
>> my_run_info = run_info_dict()
>> my_run_info(my_function)(my_parameters)
With this, all runtime information is automatically collected in my_run_info.
We can enable time-and-usage and memory profiling simply by calling
enable_profile_time_and_usage(...) or enable_profile_memory(...), respectively,
before we run our function.
We can also use the data structure directly and control the population ourselves,
however, memory profiling is not supported by default in this case but we need to
set and run the memory profiler ourselves, since memory_profiler expects that it
can wrap the function
"""
DEFAULT_TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%f'
def __init__(self, *args, **kwargs):
super(run_info_dict, self).__init__(*args, **kwargs)
self.__profile_time_and_usage = False
self.__profile_memory = False
self.__time_and_use_profiler = None
self.__memory_profiler = None
self.mpi_comm = mpi_helper.get_comm_world()
self.mpi_root = 0
self.gather_data = True
def __call__(self, func):
"""
:param func: The function to be wrapped for execution
:return: A wrapped function for which we track the runtime information in self
"""
from functools import wraps
@wraps(func)
def wrapper(*args, **kwargs):
# Pre-execute recording
self.clear() # Clear all runtime information data and profilers
self.record_preexecute(
) # Record system provenance and pre-execution data
start_time = time.time() # Start the execution timer
# Execute the function
if not self.get_profile_memory():
result = func(
*args,
**kwargs) # Execute the function without memory profiling
else:
self.__memory_profiler = memory_profiler.LineProfiler()
result = self.__memory_profiler(func)(
*args,
**kwargs) # Execute the function with memory profiling
# Post-execute recording
execution_time = time.time(
) - start_time # Compute the execution time
self.record_postexecute(
execution_time=execution_time) # Record post-execution data
self.clean_up() # Clean up empty data
if self.gather_data:
self.gather() # Gather the data from all MPI ranks
# Return the result
return result
# Return our wrapped function
return wrapper
def clear(self):
"""
Clear the dictionary and other internal parameters
Side Effects
* Remove all key/value pairs from the dict
* Set self.__time_and_use_profiler to None
* Set self.__memory_profiler to None
* Set self.__profile_memory to False if invalid (i.e, if set to True but memory profiling is unavailable)
* Set self.__profile_time_and_usage to False if invalid (i.e., if set to True but profiling is unavailable)
"""
# Make sure profiling settings are valid
if self.get_profile_memory() and not PROFILE_MEMORY_AVAILABLE:
self.enable_profile_time_and_usage(False)
if self.get_profile_time_and_usage() and not PROFILE_AVAILABLE:
self.enable_profile_memory(False)
# Remove old profilers
self.__time_and_use_profiler = None
self.__memory_profiler = None
# Clear all data from the dictionary
return super(run_info_dict, self).clear()
def enable_profile_memory(self, enable=True):
"""
Enable/disable profiling of memory usage
:param enable: boolean to enable (True) or disable (False) memory profiling
"""
if PROFILE_MEMORY_AVAILABLE:
if not enable and self.__profile_memory:
log_helper.debug(__name__,
"Disabled memory profiling. ",
root=self.mpi_root,
comm=self.mpi_comm)
if enable and not self.__profile_memory:
log_helper.debug(__name__,
"Enabled memory profiling. ",
root=self.mpi_root,
comm=self.mpi_comm)
self.__profile_memory = enable
else:
self.__profile_memory = False
if enable:
log_helper.warning(
__name__, 'Profiling of memory usage not available.' +
' Missing memory_profiler or StringIO package')
def enable_profile_time_and_usage(self, enable=True):
"""
Enable/disable time and usage profiling
:param enable: boolean to enable (True) or disable (False) time and usage profiling
"""
if PROFILE_AVAILABLE:
if not enable and self.__profile_time_and_usage:
log_helper.debug(__name__,
"Disabled time and usage profiling. ",
root=self.mpi_root,
comm=self.mpi_comm)
if enable and not self.__profile_time_and_usage:
log_helper.debug(__name__,
"Enabled time and usage profiling. ",
root=self.mpi_root,
comm=self.mpi_comm)
self.__profile_time_and_usage = enable
else:
self.__profile_time_and_usage = False
if enable:
log_helper.warning(
__name__, 'Profiling of time and usage not available.' +
' Missing profile and/or pstats package')
def get_profile_time_and_usage(self):
"""
Check whether time and usage profiling is enabled
:return: Boolean indicating whether time and usage profiling is enabled
"""
return self.__profile_time_and_usage
def get_profile_memory(self):
"""
Check whether profiling of memory usage is enabled
:return: Boolean indicating whether memory profiling is enabled
"""
return self.__profile_memory
def record_preexecute(self):
"""
Record basic runtime information in this dict before the exeuction is started.
Function used to record runtime information prior to executing the process we want to track, e.g.,
the `execute_analysis(...)` of a standard analysis.
The function may be overwritten in child classes to add recording of
additional runtime information. All runtime data should be recorded in the
main dict (i.e, self). This ensures in the case of standard analysis that
the data is stored in the HDF5 file. Other data should be stored in separate
variables that we may add to the object.
When overwriting the function we should typically call super(...,self).runinfo_record_pretexecute()
last in the custom version to ensure that the start_time is properly recorded right before
the execution of the analysis.
"""
log_helper.debug(__name__,
'Recording pre-execution runtime data',
root=self.mpi_root,
comm=self.mpi_comm)
# Record basic runtime environment information using the platform module
try:
self['architecture'] = unicode(platform.architecture())
self['java_ver'] = unicode(platform.java_ver())
self['libc_ver'] = unicode(platform.libc_ver())
self['linux_distribution'] = unicode(platform.linux_distribution())
self['mac_ver'] = unicode(platform.mac_ver())
self['machine'] = unicode(platform.machine())
self['node'] = unicode(platform.node())
self['platform'] = unicode(platform.platform())
self['processor'] = unicode(platform.processor())
self['python_branch'] = unicode(platform.python_branch())
self['python_build'] = unicode(platform.python_build())
self['python_compiler'] = unicode(platform.python_compiler())
self['python_implementation'] = unicode(
platform.python_implementation())
self['python_revision'] = unicode(platform.python_revision())
self['python_version'] = unicode(platform.python_version())
self['release'] = unicode(platform.release())
self['system'] = unicode(platform.system())
self['uname'] = unicode(platform.uname())
self['version'] = unicode(platform.version())
self['win32_ver'] = unicode(platform.win32_ver())
except:
warnings.warn(
"WARNING: Recording of platform provenance failed: " +
str(sys.exc_info()))
# Attempt to record the svn version information
try:
import subprocess
self['svn_ver'] = subprocess.check_output('svnversion').rstrip(
'\n')
except ImportError:
log_helper.warning(
__name__,
'Recording of svn version not possible. subprocess not installed',
root=self.mpi_root,
comm=self.mpi_comm)
except:
warnings.warn("Recording of svn version information failed: " +
str(sys.exc_info()))
# Attempt to record software library version
try:
import numpy as np
self['numpy_version_full_version'] = unicode(
np.version.full_version)
self['numpy_version_release'] = unicode(np.version.release)
self['numpy_version_git_revision'] = unicode(
np.version.git_revision)
except ImportError:
log_helper.warning(__name__,
'Recording of numpy version not possible.',
root=self.mpi_root,
comm=self.mpi_comm)
# Attempt to record psutil data
try:
import psutil
self['logical_cpu_count'] = unicode(psutil.cpu_count())
self['cpu_count'] = unicode(psutil.cpu_count(logical=False))
process = psutil.Process()
self['open_files'] = unicode(process.open_files())
self['memory_info_before'] = unicode(process.memory_info())
except ImportError:
log_helper.warning(
__name__,
'psutil not installed. Recording of part of runtime information not possible',
root=self.mpi_root,
comm=self.mpi_comm)
except:
warnings.warn(
"Recording of psutil-based runtime information failed: " +
str(sys.exc_info()))
# Record the start time for the analysis
self['start_time'] = unicode(datetime.datetime.now())
# Enable time and usage profiling if requested
if self.__profile_time_and_usage:
self.__time_and_use_profiler = Profile()
self.__time_and_use_profiler.enable()
def record_postexecute(self, execution_time=None):
"""
Function used to record runtime information after the task we want to track is comleted, e.g.
the `execute_analysis(...)` function of a standard analysis.
The function may be overwritten in child classes to add recording of
additional runtime information.
When overwriting the function we should call super(...,self).runinfo_record_postexecute(execution_time)
in the custom version to ensure that the execution and end_time are properly
recorded.
:param execution_time: The total time it took to execute the analysis. May be None, in which
case the function will attempt to compute the execution time based on the start_time
(if available) and the the current time.
:param comm: Used for logging only. The MPI communicator to be used. Default value is None,
in which case MPI.COMM_WORLD is used.
"""
log_helper.debug(__name__,
'Recording post-execution runtime data',
root=self.mpi_root,
comm=self.mpi_comm)
# Finalize recording of post execution provenance
self['end_time'] = unicode(datetime.datetime.now())
if execution_time is not None:
self['execution_time'] = unicode(execution_time)
elif 'start_time' in self:
start_time = run_info_dict.string_to_time(self['start_time'])
stop_time = run_info_dict.string_to_time(self['end_time'])
self['execution_time'] = unicode(
stop_time - start_time
) # TODO: This only gives execution time in full seconds right now
else:
self['execution_time'] = None
# Attempt to record psutil data
try:
import psutil
process = psutil.Process()
self['memory_info_after'] = unicode(process.memory_info())
except ImportError:
log_helper.warning(
__name__,
'psutil not installed. Recording of part of runtime information not possible',
root=self.mpi_root,
comm=self.mpi_comm)
except:
warnings.warn(
"Recording of psutil-based runtime information failed: " +
str(sys.exc_info()))
# Record the time and use profiling data if possible
if self.__time_and_use_profiler is not None:
self.__time_and_use_profiler.disable()
self.__time_and_use_profiler.create_stats()
self['profile'] = unicode(self.__time_and_use_profiler.stats)
# Save the summary statistics for the profiling data
stats_io = StringIO.StringIO()
profiler_stats = pstats.Stats(
self.__time_and_use_profiler,
stream=stats_io).sort_stats('cumulative')
profiler_stats.print_stats()
self['profile_stats'] = stats_io.getvalue()
# Record the memory profiling data if possible
if self.__memory_profiler is not None and self.get_profile_memory():
log_helper.debug(__name__,
'Recording memory profiling data',
root=self.mpi_root,
comm=self.mpi_comm)
mem_stats_io = StringIO.StringIO()
memory_profiler.show_results(self.__memory_profiler,
stream=mem_stats_io)
self['profile_mem'] = unicode(self.__memory_profiler.code_map)
self['profile_mem_stats'] = mem_stats_io.getvalue()
def clean_up(self):
"""
Clean up the runinfo object. In particular remove empty keys that
either recorded None or recorded just an empty string.
This function may be overwritten to also do clean-up needed
due to additional custom runtime instrumentation.
When overwriting this function we should call super(..., self).runinfo_clean_up()
at the end of the function to ensure that the runinfo dictionary
is clean, i.e., does not contain any empty entries.
"""
log_helper.debug(__name__,
'Clean up runtime data',
root=self.mpi_root,
comm=self.mpi_comm)
# Remove empty items from the run_info dict
for ri_key, ri_value in self.items():
try:
if ri_value is None or len(ri_value) == 0:
self.pop(ri_key)
except:
pass
def gather(self):
"""
Simple helper function to gather the runtime information---that has been collected on
multiple processes when running using MPI---on a single root process
:return: If we have more than one processes then this function returns a
dictionary with the same keys as usual for the run_info but the
values are now lists with one entry per mpi processes. If we only have
a single process, then the run_info object will be returned without
changes. NOTE: Similar to mpi gather, the function only collects
information on the root. All other processes will return just their
own private runtime information.
"""
if mpi_helper.MPI_AVAILABLE:
if self.mpi_comm.Get_size() > 1:
log_helper.debug(__name__,
'Gather runtime data from parallel tasks',
root=self.mpi_root,
comm=self.mpi_comm)
self['mpi_rank'] = self.mpi_comm.Get_rank()
run_data = self.mpi_comm.gather(self, self.mpi_root)
if self.mpi_comm.Get_rank() == self.mpi_root:
merged_run_data = {}
for run_dict in run_data:
for key in run_dict:
try:
merged_run_data[key].append(run_dict[key])
except KeyError:
merged_run_data[key] = [run_dict[key]]
return merged_run_data
return self
def get_profile_stats_object(self, consolidate=True, stream=None):
"""
Based on the execution profile of the execute_analysis(..) function get
``pstats.Stats`` object to help with the interpretation of the data.
:param consolidate: Boolean flag indicating whether multiple stats (e.g., from multiple cores)
should be consolidated into a single stats object. Default is True.
:param stream: The optional stream parameter to be used fo the pstats.Stats object.
:return: A single pstats.Stats object if consolidate is True. Otherwise the function
returns a list of pstats.Stats objects, one per recorded statistic. None is returned
in case that the stats objects cannot be created or no profiling data is available.
"""
from ast import literal_eval
if stream is None:
import sys
stream = sys.stdout
if 'profile' in self:
# Parse the profile data (which is stored as a string) or in the case of MPI we may
# have a list of strings from each MPI processes
if isinstance(self['profile'], list):
# Convert the profile from each MPI process independently
profile_data = [
literal_eval(profile) for profile in self['profile']
]
else:
# If we only have a single stat, then convert our data to a list, so that we can
# handle the single and multiple statistics case in the same way in the remainder of this function
profile_data = [
literal_eval(self['profile']),
]
# Create a list of profile objects that the pstats.Stats class understands
profile_dummies = []
for profile_i in profile_data:
# Here we are creating for each statistic a dummy class on the fly that holds our
# profile_data in the stats attributes and has an empty create_stats function.
# This trick allows us to create a pstats.Stats object without having to write our
# stats data to file or having to create a cProfile.Profile object first. Writing
# the data to file involves overhead and is ugly and creating a profiler and
# overwriting its stats is potentially problematic
profile_dummies.append(
type('Profile', (object, ), {
'stats': profile_i,
'create_stats': lambda x: None
})())
# Create the statistics object and return it
if consolidate:
profile_stats = pstats.Stats(*profile_dummies, stream=stream)
return profile_stats
else:
profile_stats = [
pstats.Stats(profile_i, stream=stream)
for profile_i in profile_dummies
]
return profile_stats
else:
return None
@staticmethod
def string_to_structime(time_string, time_format=None):
"""
Covert a time string to a time.struct_time using time.strptime
:param time_string: String with the time, e.g, with the start time of a program.
:param time_format: The time format to be used or None in which case run_info_dict.DEFAULT_TIME_FORMAT
will be used.
"""
return time.strptime(
time_string, time_format
if time_format is not None else run_info_dict.DEFAULT_TIME_FORMAT)
@staticmethod
def string_to_time(time_string, time_format=None):
"""
Convert a time string to local time object using time.mktime.
:param time_string: String with the time, e.g, with the start time of a program.
:param time_format: The time format to be used or None in which case run_info_dict.DEFAULT_TIME_FORMAT
will be used.
"""
return time.mktime(
time.strptime(
time_string, time_format if time_format is not None else
run_info_dict.DEFAULT_TIME_FORMAT))
def main():
'''
Entry point for the command line interface.
Additional sub-commands can be added by specifying them in an entry point in your
package's setup.py like this::
'owmeta_core.commands': [
'subcommand_name = module.path.for:TheSubCommand',
'sub.sub.subcommand_name = module.path.for:TheSubSubSubCommand',
],
Where, ``subcommand_name`` will be the name of the sub-command under the top-level
``owm`` command and ``module.path.for.TheSubCommand`` will be the class implementing
the command. To add to existing sub-commands one indicate the place in the command
hierarchy as in ``sub.sub.subcommand_name``: ``TheSubSubSubCommand`` would be
available under the (hypothetical) existing ``owm sub sub`` command as ``owm sub sub
subcommand_name``
So-called "hints" can affect the way command implementations are interepreted such as
indicating whether a method argument should be read in as a positional argument or
an option and what a command-line option should be named (as opposed to deriving it
from a parameter name or member variable). There is a set of hints which are a part of
owmeta-core (see `CLI_HINTS`), but these can be augmented by specifying entry points
like this::
'owmeta_core.cli_hints': 'hints = module.path.for:CLI_HINTS',
If ``module.path.for.CLI_HINTS`` is a dictionary, it will get added to the hints,
potentially affecting any sub-commands without hints already available. The entry
point name (``hints`` in the example) is only used for error-reporting by this module.
Although this is not strictly enforced, adding hints for sub-commands published by
other modules, including owmeta-core, should be avoided to ensure consistent behavior
across installations. See `owmeta_core.cli_hints` source for the format of hints.
See `CLICommandWrapper` for more details on how the command line options are constructed.
'''
logging.basicConfig()
top_command = _augment_subcommands_from_entry_points()
p = top_command()
p.log_level = 'WARN'
p.message = print
p.repository_provider = GitRepoProvider()
p.non_interactive = False
if environ.get('OWM_CLI_PROFILE'):
from cProfile import Profile
profiler = Profile()
profiler.enable()
try:
_helper(p)
except (CLIUserError, GenericUserError) as e:
s = str(e)
if not s:
from .utils import FCN
# In case someone forgets to add a helpful message for their user error
s = 'Received error: ' + FCN(type(e))
die(s)
finally:
# Call 'disconnect' to clean up. If our top_command doesn't have a disconnect(), we
# don't want to error-out, so check it actually exists.
disconnect_method = getattr(p, 'disconnect', None)
if disconnect_method:
disconnect_method()
if environ.get('OWM_CLI_PROFILE'):
profiler.disable()
profiler.dump_stats(environ['OWM_CLI_PROFILE'])
def average_consumption():
df = read_frame()
print '500000 random unique households with average consumption found'
df = df.drop_duplicates(keep=False)
df = df.sample(n=500000)
df['Average'] = (df['Sub_metering_1'] + df['Sub_metering_2'] + df['Sub_metering_3'])/3
print df[:5]
def after_18():
df = read_frame()
print 'Households whete after 18:00 Global_active_power per minute is more than 5 kW, Sub_metering_2' + \
'is more than others, choosen every second result from first part and every fourth from second part'
df = df[(df['Time'] > '18:00:00') & (df['Global_active_power'] > 5) &
(df['Sub_metering_2'] > df['Sub_metering_1']) & (df['Sub_metering_2'] > df['Sub_metering_3'])]
df1 = df[:len(df.index)/2:2]
df2 = df[len(df.index)/2::4]
frames = [df1, df2]
result = pd.concat(frames)
print result[:5]
if __name__ == '__main__':
profile.enable()
clean_data()
profile.disable()
Stats(profile).sort_stats('time').print_stats()
active_power()
voltage()
intensity()
average_consumption()
after_18()
class frame(IFrame):
framelist = set()
def __init__(self,
address="http://127.0.0.1:12000/",
time_step=500,
instrument=False,
profiling=False,
wire_format="cbor",
compress=False):
frame.framelist.add(self)
self.thread = None
self.__app = None
self.__appname = ""
self.__host_typemap = {}
self.__host_wire_format = {}
self.__typemap = {}
self.__name2type = {}
self.object_store = dataframe()
self.object_store.start_recording = True
if not address.endswith('/'):
address += '/'
self.__address = address
self.__default_wire_format = wire_format
self.__compress = compress
self.__time_step = (float(time_step) / 1000)
self.__new = {}
self.__mod = {}
self.__del = {}
self.__observed_types = set()
self.__observed_types_new = set()
self.__observed_types_mod = set()
self.__curtime = time.time()
self.__curstep = 0
self.__start_time = time.strftime("%Y-%m-%d_%H-%M-%S")
self.__instrumented = instrument
self.__profiling = profiling
self.__sessions = {}
self.__host_to_push_groupkey = {}
if instrument:
self._instruments = {}
self._instrument_headers = []
self._instrument_headers.append('bytes sent')
self._instrument_headers.append('bytes received')
def __register_app(self, app):
self.logger = self.__setup_logger("spacetime@" + self.__appname)
self.__host_typemap = {}
for address, tpmap in self.__app.__declaration_map__.items():
if address == "default":
address = self.__address
fulladdress = address + self.__appname
if fulladdress not in self.__host_typemap:
self.__host_typemap[fulladdress] = tpmap
else:
for declaration in tpmap:
self.__host_typemap[fulladdress].setdefault(
declaration, set()).update(set(tpmap[declaration]))
self.__default_wire_format = (
self.__app.__special_wire_format__["default"]
if "default" in self.__app.__special_wire_format__ else
self.__default_wire_format)
for host in self.__host_typemap:
self.__host_wire_format[host] = (
self.__app.__special_wire_format__[host]
if host in self.__app.__special_wire_format__ else
self.__default_wire_format)
all_types = set()
for host in self.__host_typemap:
wire_format = self.__host_wire_format[host]
jobj = dict([(k, [tp.__realname__ for tp in v])
for k, v in self.__host_typemap[host].items()])
producing, getting, gettingsetting, deleting, setting, tracking = (
self.__host_typemap[host].setdefault(Modes.Producing, set()),
self.__host_typemap[host].setdefault(Modes.Getter, set()),
self.__host_typemap[host].setdefault(Modes.GetterSetter,
set()),
self.__host_typemap[host].setdefault(Modes.Deleter, set()),
self.__host_typemap[host].setdefault(Modes.Setter, set()),
self.__host_typemap[host].setdefault(Modes.Tracker, set()))
self.__typemap.setdefault(Modes.Producing, set()).update(producing)
self.__typemap.setdefault(Modes.Getter, set()).update(getting)
self.__typemap.setdefault(Modes.GetterSetter,
set()).update(gettingsetting)
self.__typemap.setdefault(Modes.Deleter, set()).update(deleting)
self.__typemap.setdefault(Modes.Setter, set()).update(setting)
self.__typemap.setdefault(Modes.Tracker, set()).update(tracking)
all_types_host = tracking.union(producing).union(getting).union(
gettingsetting).union(deleting).union(setting)
all_types.update(all_types_host)
self.__observed_types.update(all_types_host)
self.__observed_types_new.update(
self.__host_typemap[host][Modes.Tracker].union(
self.__host_typemap[host][Modes.Getter]).union(
self.__host_typemap[host][Modes.GetterSetter]))
self.__observed_types_mod.update(
self.__host_typemap[host][Modes.Getter].union(
self.__host_typemap[host][Modes.GetterSetter]))
jsonobj = json.dumps({
"sim_typemap": jobj,
"wire_format": wire_format,
"app_id": self.__app.app_id
})
try:
self.__sessions[host] = Session()
if platform.system() == 'Java':
ignoreJavaSSL()
self.logger.info("Using custom HTTPAdapter for Jython")
self.__sessions[host].mount(host, MyJavaHTTPAdapter())
self.__sessions[host].verify = False
resp = requests.put(
host,
data=jsonobj,
headers={'content-type': 'application/json'})
except HTTPError as exc:
self.__handle_request_errors(resp, exc)
return False
except ConnectionError:
self.logger.exception("Cannot connect to host.")
self.__disconnected = True
return False
self.__name2type = dict([(tp.__realname__, tp) for tp in all_types])
self.object_store.add_types(all_types)
for host in self.__host_typemap:
self.__host_to_push_groupkey[host] = set([
self.object_store.get_group_key(tp)
for tp in self.__host_typemap[host][Modes.GetterSetter].union(
self.__host_typemap[host][Modes.Setter]).union(
self.__host_typemap[host][Modes.Producing]).union(
self.__host_typemap[host][Modes.Deleter])
])
return True
@staticmethod
def loop():
SpacetimeConsole().cmdloop()
def get_instrumented(self):
"""
Returns if frame is running instrumentation. (True/False)
"""
return self.__instrumented
def get_curtime(self):
"""
Returns the timestamp of the current step.
"""
return self.__curtime
def get_curstep(self):
"""
Returns the current step value of the simulation.
"""
return self.__curstep
def get_timestep(self):
"""
Returns the time-step value in milliseconds.
"""
return self.__time_step
def get_app(self):
"""
Returns a reference to the application.
"""
return self.__app
def attach_app(self, app):
"""
Receives reference to application (implementing IApplication).
Arguments:
app : spacetime-conformant Application
Exceptions:
None
"""
self.__app = app
self.__appname = app.__class__.__name__ + "_" + self.__app.app_id
def run_async(self):
"""
Starts application in non-blocking mode.
Arguments:
None
Exceptions:
None
"""
self.thread = Parallel(target=self.__run)
self.thread.daemon = True
self.thread.start()
def run_main(self):
self.__run()
def run(self):
"""
Starts application in blocking mode.
Arguments:
None
Exceptions:
None
"""
self.thread = Parallel(target=self.__run)
self.thread.daemon = True
self.thread.start()
self.thread.join()
def __clear(self):
self.__disconnected = False
self.__app.done = False
self.object_store.clear_all()
self.__new = {}
self.__mod = {}
self.__del = {}
def __run(self):
self.__clear()
if not self.__app:
raise NotImplementedError("App has not been attached")
success = self.__register_app(self.__app)
if success:
try:
if self.__profiling:
try:
from cProfile import Profile # @UnresolvedImport
if not os.path.exists('stats'):
os.mkdir('stats')
self.__profile = Profile()
self.__profile.enable()
self.logger.info("starting profiler for %s",
self.__appname)
except:
self.logger.error(
"Could not import cProfile (not supported in Jython)."
)
self.__profile = None
self.__profiling = None
self.__pull()
self.__app.initialize()
self.__push()
while not self.__app.done:
st_time = time.time()
self.__pull()
self.__app.update()
self.__push()
end_time = time.time()
timespent = end_time - st_time
self.__curstep += 1
self.__curtime = time.time()
# time spent on execution loop
if timespent < self.__time_step:
time.sleep(float(self.__time_step - timespent))
else:
self.logger.info("loop exceeded maximum time: %s ms",
timespent)
# Writes down total time spent in spacetime methods
if self.__instrumented:
si.record_instruments(timespent, self)
# One last time, because _shutdown may delete objects from the store
self.__pull()
self._shutdown()
self.__push()
self.__unregister_app()
except ConnectionError as cerr:
self.logger.error("A connection error occurred: %s",
cerr.message)
except HTTPError as herr:
self.logger.error(
"A fatal error has occurred while communicating with the server: %s",
herr.message)
except:
self.logger.exception("An unknown error occurred.")
raise
finally:
if self.__profiling:
self.__profile.disable()
self.__profile.create_stats()
self.__profile.dump_stats(
os.path.join(
'stats', "%s_stats_%s.ps" %
(self.__start_time, self.__appname)))
else:
self.logger.info("Could not register, exiting run loop...")
def app_done(self):
"""
app_done
Returns whether app has finished running or not
"""
return self.__app.done
def get(self, tp, oid=None):
"""
Retrieves objects from local data storage. If id is provided, returns
the object identified by id. Otherwise, returns the list of all objects
matching type tp.
Arguments:
tp : PCC set type being fetched
oid : primary key of an individual object.
Exceptions:
- ID does not exist in store
- Application does not annotate that type
"""
if tp in self.__observed_types:
if oid:
# Have to get this to work
return self.object_store.get(tp, oid)
return self.object_store.get(tp)
else:
raise Exception("Application %s does not annotate type %s" %
(self.__appname, tp))
def add(self, obj):
"""
Adds an object to be stored and tracked by spacetime.
Arguments:
obj : PCC object to stored
Exceptions:
- Application is not annotated as a producer
"""
if obj.__class__ in self.__typemap[Modes.Producing]:
self.object_store.append(obj.__class__, obj)
else:
raise Exception("Application %s is not a producer of type %s" %
(self.__appname, obj.__class__))
def delete(self, tp, obj):
"""
Deletes an object currently stored and tracked by spacetime.
Arguments:
tp: PCC type of object to be deleted
obj : PCC object to be deleted
Exceptions:
- Application is not annotated as a Deleter
"""
if tp in self.__typemap[Modes.Deleter]:
self.object_store.delete(tp, obj)
else:
raise Exception("Application %s is not registered to delete %s" %
(self.__appname, tp))
def get_new(self, tp):
"""
Retrieves new objects of type 'tp' retrieved in last pull (i.e. since
last tick).
Arguments:
tp: PCC type for retrieving list of new objects
Exceptions:
None
Note:
Application should be annotated as a Getter, GetterSetter, or Tracker,
otherwise result is always an empty list.
"""
if tp in self.__observed_types_new:
return self.object_store.get_new(tp)
else:
self.logger.warn(
("Checking for new objects of type %s, but not "
"a Getter, GetterSetter, or Tracker of type. Empty list "
"always returned"), tp)
return []
def get_mod(self, tp):
"""
Retrieves objects of type 'tp' that were modified since last pull
(i.e. since last tick).
Arguments:
tp: PCC type for retrieving list of modified objects
Exceptions:
None
Note:
Application should be annotated as a Getter,or GetterSetter, otherwise
result is always an empty list.
"""
if tp in self.__observed_types_mod:
return self.object_store.get_mod(tp)
else:
self.logger.warn(("Checking for modifications in objects of type "
"%s, but not a Getter or GetterSetter of type. "
"Empty list always returned"), tp)
return []
def get_deleted(self, tp):
"""
Retrieves objects of type 'tp' that were deleted since last pull
(i.e. since last tick).
Arguments:
tp: PCC type for retrieving list of deleted objects
Exceptions:
None
Note:
Application should be annotated as a Getter, GetterSetter, or Tracker,
otherwise result is always an empty list.
"""
if tp in self.__observed_types_new:
return self.object_store.get_deleted(tp)
else:
self.logger.warn(
("Checking for deleted objects of type %s, but "
"not a Getter, GetterSetter, or Tracker of type. Empty list "
"always returned"), tp)
return []
def __handle_request_errors(self, resp, exc):
if resp.status_code == 401:
self.logger.error(
"This application is not registered at the server. Stopping..."
)
raise
else:
self.logger.warn("Non-success code received from server: %s %s",
resp.status_code, resp.reason)
@timethis
def __process_pull_resp(self, resp):
if resp and "gc" in resp:
self.object_store.apply_changes(resp)
self.object_store.clear_record()
@timethis
def __pull(self):
if self.__disconnected:
return
if self.__instrumented:
self._instruments['bytes received'] = 0
updates = DataframeChanges_Base()
try:
for host in self.__host_typemap:
type_dict = {}
# Need to give mechanism to selectively ask for some changes. Very hard to implement in current dataframe scheme.
resp = self.__sessions[host].get(host + "/updated", data={})
try:
resp.raise_for_status()
if self.__instrumented:
self._instruments['bytes received'] = len(resp.content)
data = resp.content
#print data
DF_CLS, content_type = FORMATS[
self.__host_wire_format[host]]
dataframe_change = DF_CLS()
dataframe_change.ParseFromString(data)
updates.CopyFrom(dataframe_change)
except HTTPError as exc:
self.__handle_request_errors(resp, exc)
#json.dump(updates, open("pull_" + self.__appname + ".json", "a") , sort_keys = True, separators = (',', ': '), indent = 4)
self.__process_pull_resp(updates)
except ConnectionError:
self.logger.exception("Disconnected from host.")
self.__disconnected = True
self._stop()
@timethis
def __push(self):
if self.__disconnected:
return
if self.__instrumented:
self._instruments['bytes sent'] = 0
changes = self.object_store.get_record()
#json.dump(changes, open("push_" + self.__appname + ".json", "a") , sort_keys = True, separators = (',', ': '), indent = 4)
for host in self.__host_typemap:
try:
DF_CLS, content_type = FORMATS[self.__host_wire_format[host]]
changes_for_host = DF_CLS()
changes_for_host["gc"] = RecursiveDictionary([
(gck, gc) for gck, gc in changes["gc"].items()
if gck in self.__host_to_push_groupkey[host]
])
if "types" in changes:
changes_for_host["types"] = changes["types"]
dictmsg = changes_for_host.SerializeToString()
#update_dict = {"update_dict": protomsg}
if self.__instrumented:
self._instruments['bytes sent'] = sys.getsizeof(dictmsg)
headers = {'content-type': content_type}
if self.__compress:
headers['content-encoding'] = 'gzip'
dictmsg = zlib.compress(dictmsg)
resp = self.__sessions[host].post(host + "/updated",
data=dictmsg,
headers=headers)
except TypeError:
self.logger.exception("error encoding obj. Object: %s",
changes_for_host)
except HTTPError as exc:
self.__handle_request_errors(resp, exc)
except ConnectionError:
self.logger.exception("Disconnected from host.")
self.__disconnected = True
self._stop()
self.object_store.clear_record()
self.object_store.clear_buffer()
def _shutdown(self):
"""
_shutdown
Called after the frame execution loop stops, in the last pull/push
iteration
"""
self.__app.shutdown()
def _stop(self):
"""
_stop
Called by frame's command prompt on quit/exit
"""
self.__app.done = True
def __unregister_app(self):
for host in self.__host_typemap:
resp = requests.delete(host)
self.logger.info("Successfully deregistered from %s", host)
def __setup_logger(self, name, file_path=None):
logger = logging.getLogger(name)
# Set default logging handler to avoid "No handler found" warnings.
logger.addHandler(NullHandler())
logger.setLevel(logging.DEBUG)
logger.debug("Starting logger for %s", name)
return logger
class BaseExperiment(object):
memory_measure_interval = 0.1
"""Indicates how often the memory should be measured, in seconds."""
run_descriptions = {
'setup_authsetup': 'always',
'register_keygen': 'always',
'encrypt': 'always',
'update_keys': 'always',
'data_update': 'always',
'policy_update': 'always',
'decrypt': 'always'
}
"""
Description of which steps to run during the experiment. Values can be one of either:
- 'always': This step is run always, in each iteration for each case
- 'once': The step is run for each implementation once, prior to all experiments. This can be helpful if only
the encryption and decryption is relevant.
- 'never': This step is never executed.
"""
attribute_authority_descriptions = [{
'name':
'AUTHORITY0',
'attributes':
list(
map(lambda a: a + '@AUTHORITY0', [
'ONE', 'TWO', 'THREE', 'FOUR', 'FIVE', 'SIX', 'SEVEN', 'EIGHT',
'NINE', 'TEN'
]))
}, {
'name':
'AUTHORITY1',
'attributes':
list(
map(lambda a: a + '@AUTHORITY1', [
'ONE', 'TWO', 'THREE', 'FOUR', 'FIVE', 'SIX', 'SEVEN', 'EIGHT',
'NINE', 'TEN'
]))
}] # type: List[Dict[str, Any]]
"""
Description of the attribute authorities to use in this experiment.
Is a list of dicts, where each dict contains at least a name and a list of attributes.
"""
user_descriptions = [ # type: List[Dict[str, Any]]
{
'gid': 'BOB',
'attributes': {
'AUTHORITY0':
attribute_authority_descriptions[0]['attributes'],
'AUTHORITY1': attribute_authority_descriptions[1]['attributes']
}
},
{
'gid': 'DOCTOR',
'attributes': {
'AUTHORITY0':
attribute_authority_descriptions[0]['attributes'],
'AUTHORITY1': attribute_authority_descriptions[1]['attributes']
}
},
]
"""Description of the users to use in this experiment."""
generated_file_sizes = [10 * 1024 * 1024] # type: List[int]
"""List of sizes of files to randomly generate as input files before the experiment."""
generated_file_amount = 2
"""Amount of random files to generate for each size."""
encrypted_file_size = generated_file_sizes[0]
"""Size of the file to encrypt and decrypt."""
read_policy = '(ONE@AUTHORITY0 OR SIX@AUTHORITY1)' \
' AND (TWO@AUTHORITY0 OR SEVEN@AUTHORITY1)' \
' AND (THREE@AUTHORITY0 OR EIGHT@AUTHORITY1)'
"""The read policy to use when encrypting."""
write_policy = read_policy
"""The write policy to use when encrypting."""
updated_read_policy = '(SIX@AUTHORITY0 OR ONE@AUTHORITY1)' \
' AND (SEVEN@AUTHORITY0 OR TWO@AUTHORITY1)' \
' AND (EIGHT@AUTHORITY0 OR THREE@AUTHORITY1)'
"""The read policy to use when updating the policy"""
updated_write_policy = updated_read_policy
"""The write policy to use when updating the policy"""
measurement_types = [
MeasurementType.timings, MeasurementType.cpu, MeasurementType.memory
]
"""The types of measurements to perform in this experiment for each run."""
measurement_types_once = [MeasurementType.storage_and_network]
"""The types of measurements to perform only once during this experiment."""
implementations = implementations
"""The implementations to run this experiments on."""
measurement_repeat = 100
"""The amount of times to repeat every measurement for each case and implementation."""
def __init__(self, cases: List[ExperimentCase] = None) -> None:
self.state = ExperimentState() # type: ExperimentState
self.output = ExperimentOutput(self.get_name(),
self.state) # type: ExperimentOutput
"""
The current state of the experiment.
This shows for example which implementation we currently use, and which measurements are performed.
"""
# Experiment variables
self.location = None # type: str
"""Location of the encrypted data. Is set during the experiment"""
self.memory_usages = None # type: List[Tuple[str, List[float]]]
self.cpu_times = None # type: List[Tuple[str, float]]
self.profiler = None # type: Profile
self.psutil_process = None # type: Process
# Use case actors
self.central_authority = None # type: CentralAuthority
self.attribute_authorities = None # type: List[AttributeAuthority]
self.user_clients = None # type: List[UserClient]
self.insurance = None # type: InsuranceService
# Experiment cases
if cases is None:
cases = [ExperimentCase('base', None)]
self.cases = cases # type: List[ExperimentCase]
def global_setup(self) -> None:
"""
Setup all things for this experiment independent of run, implementation and case,
like generating random input files.
This method is only called once for each experiment, namely at the very start.
"""
self.generate_files()
def generate_files(self) -> None:
"""Generate all input files as specified by the generated_file_sizes property."""
file_generator = RandomFileGenerator()
input_path = self.get_experiment_input_path()
for size in self.generated_file_sizes:
file_generator.generate(size,
self.generated_file_amount,
input_path,
skip_if_exists=True,
verbose=True)
@property
def file_name(self) -> str:
"""File name of the file to encrypt."""
return join(self.get_experiment_input_path(),
'%i-0' % self.encrypted_file_size)
@property
def update_file_name(self) -> str:
"""File name of the file containing the data to use when updating the data."""
return join(self.get_experiment_input_path(),
'%i-1' % self.encrypted_file_size)
def setup(self) -> None:
"""
Setup for a single run of this experiment for a single implementation, case and measurement type.
"""
if OUTPUT_DETAILED and not path.exists(
self.output.experiment_case_iteration_results_directory()):
os.makedirs(
self.output.experiment_case_iteration_results_directory())
self.reset_user_clients()
self.clear_insurance_storage()
def reset_user_clients(self):
if self.user_clients is not None:
for user_client in self.user_clients:
user_client.monitor_network = self.state.measurement_type == MeasurementType.storage_and_network
user_client.reset_connections()
def tear_down(self) -> None:
"""
Tear down after a single run of the experiment for a single implementation, case and measurement type.
Note: this is run after the measurements are stopped, but before the measurements are finished.
"""
if self.state.measurement_type == MeasurementType.storage_and_network:
for authority in self.attribute_authorities:
authority.save_attribute_keys()
def reset_variables(self):
self.location = None
self.memory_usages = None
self.cpu_times = None
self.profiler = None
self.psutil_process = None
# Use case actors
self.central_authority = None
self.attribute_authorities = None
self.user_clients = None
self.insurance = None
def setup_implementation_directories(self) -> None:
"""
Setup the directories used in this experiment for a single implementation.
Empties directories and create them if they do not exist.
"""
assert self.state.implementation is not None
# Empty storage directories
if os.path.exists(self.get_user_client_storage_path()):
shutil.rmtree(self.get_user_client_storage_path())
# Create directories
if not os.path.exists(self.get_experiment_input_path()):
os.makedirs(self.get_experiment_input_path())
os.makedirs(self.get_user_client_storage_path())
def create_central_authority(self):
self.central_authority = self.state.implementation.create_central_authority(
storage_path=self.get_central_authority_storage_path())
def create_attribute_authorities(
self, implementation: BaseImplementation) -> None:
"""
Create the attribute authorities defined in the descriptions (self.attribute_authority_descriptions).
:param implementation: The implementation to use.
:return: A list of attribute authorities.
"""
self.attribute_authorities = list(
map(lambda d: self.create_attribute_authority(d, implementation),
self.attribute_authority_descriptions))
def create_attribute_authority(
self, authority_description: Dict[str, Any],
implementation: BaseImplementation) -> AttributeAuthority:
"""
Create an attribute authority defined in a description.
:param authority_description: The description of the authority.
:param implementation: The implementation to use.
:return: The attribute authority.
"""
attribute_authority = implementation.create_attribute_authority(
authority_description['name'],
storage_path=self.get_attribute_authority_storage_path())
return attribute_authority
def create_user_clients(self, implementation: BaseImplementation) -> None:
"""
Create the user clients defined in the descriptions (self.user_descriptions).
:param implementation: The implementation to use.
:return: A list of user clients.
"""
self.user_clients = list(
map(lambda d: self.create_user_client(d, implementation),
self.user_descriptions))
def create_user_client(self, user_description: Dict[str, Any],
implementation: BaseImplementation) -> UserClient:
"""
Create a user client defined in the descriptions (self.user_descriptions).
:param user_description: The description of the user.
:param implementation: The implementation to use.
:return: A list of user clients.
"""
user = User(user_description['gid'], implementation)
client = UserClient(user,
implementation,
storage_path=self.get_user_client_storage_path(),
monitor_network=self.state.measurement_type ==
MeasurementType.storage_and_network)
return client
def _run_setup(self) -> None:
# Create central authority
self.central_authority.central_setup()
self.central_authority.save_global_parameters()
self._setup_insurance()
def _setup_insurance(self) -> None:
# Create insurance service
self.insurance = InsuranceService(
self.state.implementation.serializer,
self.central_authority,
self.state.implementation.public_key_scheme,
storage_path=self.get_insurance_storage_path())
def _run_authsetup(self, authority: AttributeAuthority) -> None:
attributes = next(
description['attributes']
for description in self.attribute_authority_descriptions
if description['name'] == authority.name) # type: List[str]
authority.setup(self.central_authority, attributes, 1)
self.insurance.add_authority(authority)
authority.save_attribute_keys()
def _run_register(self, user_client: UserClient) -> None:
# Create user clients
user_client.register(self.insurance)
def _run_keygen(self, user_client: UserClient) -> None:
"""
Generate the user secret keys for each current user client by generating the
keys at the attribute authorities. The attributes to issue/generate are taken from the user
descriptions (self.user_descriptions)
:requires: self.user_clients is not None
"""
attributes = next(description['attributes']
for description in self.user_descriptions
if description['gid'] == user_client.user.gid)
user_client.request_secret_keys_multiple_authorities(attributes,
1) # type: ignore
user_client.save_user_secret_keys()
def _run_encrypt(self) -> None:
self.location = self.user_clients[0].encrypt_file(
self.file_name, self.read_policy, self.write_policy)
def _run_update_keys(self, authority: AttributeAuthority) -> None:
authority.update_keys(1)
def _run_data_update(self) -> None:
with open(self.update_file_name, 'rb') as update_file:
self.user_clients[1].update_file(self.location, update_file.read())
def _run_policy_update(self) -> None:
# Performed by the owner, as only the owner is allowed to do this
self.user_clients[0].update_policy_file(self.location,
self.updated_read_policy,
self.updated_write_policy, 1)
def _run_decrypt(self) -> None:
self.user_clients[1].decrypt_file(self.location)
def run(self) -> None:
self.global_setup()
for implementation in self.implementations:
self.state.implementation = implementation
self.setup_implementation_directories()
if self.run_descriptions['setup_authsetup'] == 'once':
self.create_central_authority()
self.create_attribute_authorities(self.state.implementation)
self._run_setup()
for authority in self.attribute_authorities:
self._run_authsetup(authority)
if self.run_descriptions['register_keygen'] == 'once':
self.create_user_clients(self.state.implementation)
for user_client in self.user_clients:
self._run_register(user_client)
self._run_keygen(user_client)
if self.run_descriptions['encrypt'] == 'once':
self._run_encrypt()
if self.run_descriptions['update_keys'] == 'once':
for authority in self.attribute_authorities:
self._run_update_keys(authority)
if self.run_descriptions['decrypt'] == 'once':
self._run_decrypt()
for i in range(0, self.measurement_repeat):
for case in self.cases:
for measurement_type in self.measurement_types: # type: ignore
self.state.iteration = i
self.state.case = case
self.state.measurement_type = measurement_type
self.run_current_state()
for case in self.cases:
for measurement_type in self.measurement_types_once: # type: ignore
self.state.iteration = 0
self.state.case = case
self.state.measurement_type = measurement_type
self.run_current_state()
self.reset_variables()
def run_current_state(self) -> None:
self.log_current_state()
# noinspection PyBroadException
try:
self.setup()
self.start_measurements()
if self.run_descriptions['setup_authsetup'] == 'always':
self.create_central_authority()
self.create_attribute_authorities(self.state.implementation)
self.run_step(ABEStep.setup, self._run_setup)
for authority in self.attribute_authorities:
self.run_step(ABEStep.authsetup, self._run_authsetup,
[authority])
if self.run_descriptions['register_keygen'] == 'always':
self.create_user_clients(self.state.implementation)
for user_client in self.user_clients:
self.run_step(ABEStep.register, self._run_register,
[user_client])
self.run_step(ABEStep.keygen, self._run_keygen,
[user_client])
if self.run_descriptions['encrypt'] == 'always':
self.run_step(ABEStep.encrypt, self._run_encrypt)
if self.run_descriptions['update_keys'] == 'always':
for authority in self.attribute_authorities:
self.run_step(ABEStep.update_keys, self._run_update_keys,
[authority])
if self.run_descriptions['data_update'] == 'always':
self.run_step(ABEStep.data_update, self._run_data_update)
if self.run_descriptions['policy_update'] == 'always':
self.run_step(ABEStep.policy_update, self._run_policy_update)
if self.run_descriptions['decrypt'] == 'always':
self.run_step(ABEStep.decrypt, self._run_decrypt)
self.stop_measurements()
self.tear_down()
self.finish_measurements()
except KeyboardInterrupt:
raise
except:
self.output.output_error()
def run_step(self,
abe_step: ABEStep,
method: Callable[..., None],
args: List[Any] = list()):
if self.state.measurement_type == MeasurementType.memory:
u = memory_usage((method, args, {}),
interval=self.memory_measure_interval)
self.memory_usages.append(
(abe_step.name, [min(u),
max(u),
max(u) - min(u),
len(u)]))
elif self.state.measurement_type == MeasurementType.cpu:
times_before = self.psutil_process.cpu_times()
method(*args) # type: ignore
times_after = self.psutil_process.cpu_times()
self.cpu_times.append(
(abe_step.name, (times_after.user - times_before.user) +
(times_after.system - times_before.system)))
else:
method(*args) # type: ignore
def log_current_state(self) -> None:
"""
Log the current state of the experiment, so the progress of the sequence can be followed.
"""
logging.info(
"=> Running %s with implementation %s (%d/%d), iteration %d/%d, case %s, measurement %s"
% (self.get_name(), self.state.implementation.get_name(),
implementations.index(self.state.implementation) + 1,
len(implementations), self.state.iteration + 1,
self.measurement_repeat, self.state.case.name,
str(self.state.measurement_type)))
def start_measurements(self) -> None:
"""
Start the measurements for a single run.
"""
logging.debug("Experiment.start")
if self.state.measurement_type == MeasurementType.timings:
self.profiler = Profile()
self.profiler.enable()
elif self.state.measurement_type == MeasurementType.cpu:
self.cpu_times = list()
self.psutil_process = Process()
elif self.state.measurement_type == MeasurementType.memory:
self.memory_usages = list()
def stop_measurements(self) -> None:
"""
Stop the measurements for the current run, but do not export the results yet.
"""
if self.state.measurement_type == MeasurementType.timings:
self.profiler.disable()
def finish_measurements(self) -> None:
"""
Finish the stopped measurements by exporting the results to the output files.
"""
logging.debug("Experiment.finish")
if self.state.measurement_type == MeasurementType.timings:
self.output.output_timings(self.profiler)
elif self.state.measurement_type == MeasurementType.memory:
self.output.output_case_results(
'memory',
self.memory_usages,
variables=['min', 'max', 'diff', 'amount'])
elif self.state.measurement_type == MeasurementType.storage_and_network:
self.output.output_connections(self.get_connections())
self.output.output_storage_space([{
'path':
self.get_insurance_storage_path(),
'filename_mapper':
lambda file: path.splitext(file)[1].strip('.')
}, {
'path':
self.get_user_client_storage_path()
}, {
'path':
self.get_attribute_authority_storage_path()
}, {
'path':
self.get_central_authority_storage_path()
}])
elif self.state.measurement_type == MeasurementType.cpu:
self.output.output_cpu_times(self.cpu_times)
def get_user_client(self, gid: str) -> UserClient:
"""
Gets the UserClient for the given global identifier, or returns None.
:param gid: The global identifier.
:return: The user client or None.
"""
return next((x for x in self.user_clients if x.user.gid == gid), None)
def get_attribute_authority(self, name: str) -> AttributeAuthority:
"""
Gets the AttributeAuthority for the given name, or returns None.
:param name: The authority name.
:return: The attribute authority or None.
"""
return next((x for x in self.attribute_authorities if x.name == name),
None)
def get_connections(self) -> List[BaseConnection]:
"""
Get all connections used in this experiment of which the usage should be outputted.
:return: A list of connections
"""
result = [] # type: List[BaseConnection]
for user_client in self.user_clients:
result += [user_client.insurance_connection]
result += user_client.authority_connections.values()
return result
def get_name(self) -> str:
"""
Gets the name of this experiment.
:return: The name of this experiment.
"""
return self.__class__.__name__
def clear_insurance_storage(self) -> None:
"""
Clear the storage as used by the insurance company for the ciphertexts.
"""
if os.path.exists(self.get_insurance_storage_path()):
shutil.rmtree(self.get_insurance_storage_path())
os.makedirs(self.get_insurance_storage_path())
def clear_attribute_authority_storage(self) -> None:
"""
Clear the storage as used by the insurance company for the ciphertexts.
"""
if os.path.exists(self.get_attribute_authority_storage_path()):
shutil.rmtree(self.get_attribute_authority_storage_path())
os.makedirs(self.get_attribute_authority_storage_path())
def get_experiment_storage_base_path(self) -> str:
"""
Gets the base path of the location to be used for storage in this experiment.
"""
return os.path.join(os.path.dirname(os.path.realpath(__file__)),
'../data/experiments/%s' % self.get_name())
def get_experiment_input_path(self) -> str:
"""
Gets the path of the location to be used for the inputs of the experiment.
"""
return os.path.join(self.get_experiment_storage_base_path(), 'input')
def get_user_client_storage_path(self) -> str:
"""
Gets the path of the location to be used for the storage of user client data.
"""
return os.path.join(self.get_experiment_storage_base_path(),
self.state.implementation.get_name(), 'client')
def get_insurance_storage_path(self) -> str:
"""
Gets the path of the location to be used for the storage of the insurance service.
"""
return os.path.join(self.get_experiment_storage_base_path(),
'insurance')
def get_attribute_authority_storage_path(self) -> str:
"""
Gets the path of the location to be used for the storage of the attribute authorities.
"""
return os.path.join(self.get_experiment_storage_base_path(),
self.state.implementation.get_name(),
'authorities')
def get_central_authority_storage_path(self) -> str:
"""
Gets the path of the location to be used for the storage of the central authorities.
"""
return os.path.join(self.get_experiment_storage_base_path(),
self.state.implementation.get_name(),
'central_authority')
def inspect(self, method):
profile = Profile()
profile.enable()
method()
profile.disable()
return StatsDict(profile).sort_stats('cumtime')
def main():
with open("CTRL", mode="r") as fp:
site_t = None
class_t = None
plat_n = 0
for line in fp:
word = line.split()
if word[0] == "STRUC":
for w in word:
worde = w.split("=")
if worde[0] == "ALAT":
alat = float(worde[1])
plat_n = 4
if word[0] == "DIM":
for w in word:
worde = w.split("=")
if worde[0] == "NBAS":
nbas = int(worde[1])
if worde[0] == "NCLASS":
nclass = int(worde[1])
# print(alat,nbas)
if plat_n == 3:
Plat = np.zeros((3, 3))
if len(word) == 3:
worde = word[0].split("=")
Plat[0][0] = float(worde[1])
Plat[0][1] = float(word[1])
Plat[0][2] = float(word[2])
else:
Plat[0][0] = float(word[1])
Plat[0][1] = float(word[2])
Plat[0][2] = float(word[3])
elif plat_n == 2:
Plat[1][0] = float(word[0])
Plat[1][1] = float(word[1])
Plat[1][2] = float(word[2])
elif plat_n == 1:
Plat[2][0] = float(word[0])
Plat[2][1] = float(word[1])
Plat[2][2] = float(word[2])
plat_n -= 1
if word[0] == "CLASS":
class_t = True
class_n = nclass
allclass_ctrl = []
if class_t:
if len(word) < 5:
continue
if class_n == nclass:
if word[2] == "Z=":
class_ctrl = Class_ctrl(word[1].split("=")[1], word[3],
word[4].split("=")[1])
else:
class_ctrl = Class_ctrl(word[1].split("=")[1],
word[2].split("=")[1],
word[3].split("=")[1])
allclass_ctrl.append(class_ctrl)
else:
if word[1] == "Z=":
class_ctrl = Class_ctrl(word[0].split("=")[1], word[2],
word[3].split("=")[1])
else:
class_ctrl = Class_ctrl(word[0].split("=")[1],
word[1].split("=")[1],
word[2].split("=")[1])
allclass_ctrl.append(class_ctrl)
class_n -= 1
if class_n == 0:
class_t = False
if word[0] == "SITE":
site_t = True
site_n = nbas
allatom = []
if site_t:
if site_n == nbas:
atom = Atom()
for w in word:
worde = w.split("=")
if worde[0] == "ATOM":
atom.name = worde[1]
if len(word) == 5:
worde = word[2].split("=")
atom.pos = np.array(
[float(worde[1]),
float(word[3]),
float(word[4])])
elif len(word) == 6:
atom.pos = np.array(
[float(word[3]),
float(word[4]),
float(word[5])])
else:
print("error")
sys.exit()
allatom.append(atom)
else:
atom = Atom()
atom.name = (word[0].split("=")[1])
if len(word) == 4:
worde = word[1].split("=")
atom.pos = np.array(
[float(worde[1]),
float(word[2]),
float(word[3])])
elif len(word) == 5:
atom.pos = np.array(
[float(word[2]),
float(word[3]),
float(word[4])])
else:
print("error")
sys.exit()
allatom.append(atom)
site_n -= 1
if site_n == 0:
site_t = False
if site_t == False:
break
for i in allatom:
i.ctrltuika(allclass_ctrl)
for i in range(len(allatom)):
print(allatom[i].name, allatom[i].ze, allatom[i].rad, allatom[i].pos)
# print(Plat)
# sys.exit()
# Pos_np=np.array(Pos)
# print(Atom)
# print(Pos_np)
# Plat=Plat*alat*BOHR
# Pos_np=Pos_np*alat*BOHR
# print(Pos_np)
print("which is center ?(Atomname or Coordinate)")
centeratom = input().split()
c = 0
origincart = np.array([0., 0., 0.])
if len(centeratom) == 1:
for i in range(nbas):
if centeratom == allatom[i].name:
c = i
break
origincart = allatom[c].pos
elif len(centeratom) == 3:
for i in range(3):
origincart[i] = float(centeratom[i])
else:
print("error")
sys.exit()
print("How is clustersize ? (please input cluster radius)")
clradius = float(input())
with open("cluster.in", mode="w") as clin:
for i in range(3):
clin.write(str("{:>11}".format("{:.8f}".format(Plat[i][0]))))
clin.write(str("{:>11}".format("{:.8f}".format(Plat[i][1]))))
clin.write(str("{:>11}".format("{:.8f}".format(Plat[i][2]))))
clin.write("\n")
clin.write(" 0 0 0\n")
clin.write(str("{:>12}".format("{:.8f}".format(allatom[c].pos[0]))))
clin.write(str("{:>12}".format("{:.8f}".format(allatom[c].pos[1]))))
clin.write(str("{:>12}".format("{:.8f}".format(allatom[c].pos[2]))))
clin.write("\n")
clin.write(" " + str(nbas) + "\n")
for i in range(nbas):
clin.write(str("{:>3}".format(allatom[i].name)))
clin.write(str("{:>12}".format("{:.8f}".format(
allatom[i].pos[0]))))
clin.write(str("{:>12}".format("{:.8f}".format(
allatom[i].pos[1]))))
clin.write(str("{:>12}".format("{:.8f}".format(
allatom[i].pos[2]))))
clin.write("\n")
clin.write(str("{:.4f}".format(alat * BOHR)) + "\n")
for i in range(3):
clin.write(str(clradius) + "\n")
sys.exit()
cutoff = clradius
i0max = int(cutoff * FACTOR / np.sqrt(np.sum(Plat[0]**2)))
i1max = int(cutoff * FACTOR / np.sqrt(np.sum(Plat[1]**2)))
i2max = int(cutoff * FACTOR / np.sqrt(np.sum(Plat[2]**2)))
clusteratom = []
y = np.array([0., 0., 0.])
n = 0
pr = Profile()
pr.enable()
t1 = time.time()
for i0 in range(-i0max, i0max + 1):
for i1 in range(-i1max, i1max + 1):
for i2 in range(-i2max, i2max + 1):
for k in range(nbas):
y = i0 * Plat[0] + i1 * Plat[1] + i2 * Plat[2] + allatom[
k].pos - origincart
y *= alat * BOHR
if np.sqrt(sum(y**2)) < clradius:
atom = Atom()
atom.name = allatom[k].name
atom.ze = allatom[k].ze
atom.pos = copy.deepcopy(y)
clusteratom.append(atom)
n += 1
print(n)
t2 = time.time()
print(t2 - t1)
sys.exit()
pr.disable()
pr.print_stats()
sys.exit()
for i in range(len(clusteratom)):
print(clusteratom[i].name, clusteratom[i].ze, clusteratom[i].pos)
action="store_true",
dest="profile",
default=False,
help="Enable this flag to the profile the script")
args = parser.parse_args()
# Check if the file paths from the user are valid
for file_path in [
args.input_file, args.occupations_output_file,
args.states_output_file
]:
if not path.isdir(path.dirname(file_path)):
print("error: invalid file path: " + file_path)
exit(1)
return args
if __name__ == "__main__":
args = get_args()
if args.profile:
profile = Profile()
profile.enable()
main(args)
profile.disable()
profile_stats = Stats(profile).sort_stats(SortKey.CUMULATIVE)
profile_stats.print_stats()
else:
main(args)
class ProfilerResource(BaseResource):
loggers_names = ["tamarco.profiler"]
def __init__(self, *args, **kwargs):
self.logger = logging.getLogger("tamarco.profiler")
self.profiler = None
self.profiler_file_path = None
self.cpu_watcher_task = None
super().__init__(*args, **kwargs)
async def start(self):
if await self.is_profiler_enabled():
self.profiler_file_path = f"/tmp/{self.microservice.name}_profile"
self._initialize_profiler()
self.logger.info(
f"Started Profiler resource with file: {self.profiler_file_path} and "
f"time between snapshots: {TIME_BETWEEN_SNAPSHOTS}")
self.cpu_watcher_task = asyncio.ensure_future(
self.cpu_watcher(), loop=self.microservice.loop)
else:
self.logger.debug("Profiler resource disabled")
await super().start()
async def stop(self):
if self.profiler:
self._stop_profiler()
if self.cpu_watcher_task:
self.cpu_watcher_task.cancel()
await super().stop()
def _initialize_profiler(self):
self.profiler = Profile()
self.profiler.enable()
def _stop_profiler(self):
assert self.profiler, "Trying to stop a profiler when it isn't initialized"
self.profiler.disable()
self.profiler = None
def _restart_profiler(self):
self._stop_profiler()
self._initialize_profiler()
async def is_profiler_enabled(self) -> bool:
try:
microservices_with_profiler = await self.settings.get(
"microservices_with_profiler")
except SettingNotFound:
return False
else:
return self.microservice.name in microservices_with_profiler
async def cpu_watcher(self):
while True:
await asyncio.sleep(TIME_BETWEEN_SNAPSHOTS)
self.save_profile_snapshot_to_file()
def save_profile_snapshot_to_file(self):
try:
with open(self.profiler_file_path, "a") as profile_file:
self.logger.debug(
f"Opened profile file {self.profiler_file_path}. Saving profile information"
)
profile_file.write(
f"\n\n###############\n# DATE : {datetime.datetime.now()}\n###############\n"
)
stats = pstats.Stats(self.profiler, stream=profile_file)
stats.sort_stats("tottime")
stats.print_stats(100)
self._restart_profiler()
except Exception:
self.logger.warning(
"Unexpected exception saving profile information")
def test_performance(self):
@define
def test_func(a, b):
def body(a_id, a_name, b_id, b_name):
pass
return body, [a.id, a.name, b.id, b.name]
struct = Construct({
'r1':
if_(self.a_cls.id, then_=test_func.defn(self.a_cls, self.b_cls)),
'r2':
if_(self.a_cls.name, then_=test_func.defn(self.a_cls, self.b_cls)),
'r3':
if_(self.b_cls.id, then_=test_func.defn(self.a_cls, self.b_cls)),
'r4':
if_(self.b_cls.name, then_=test_func.defn(self.a_cls, self.b_cls)),
})
row = (self.session.query(*struct._columns).join(self.b_cls.a).first())
# warm-up
for _ in _range(5000):
struct._from_row(row)
profile1 = Profile()
profile1.enable()
for _ in _range(5000):
struct._from_row(row)
profile1.disable()
out1 = StringIO()
stats1 = Stats(profile1, stream=out1)
stats1.strip_dirs()
stats1.sort_stats('calls').print_stats(10)
print(out1.getvalue().lstrip())
out1.close()
row = (self.session.query(
self.a_cls.id.label('a_id'),
self.a_cls.name.label('a_name'),
self.b_cls.id.label('b_id'),
self.b_cls.name.label('b_name'),
).join(self.b_cls.a).first())
def make_object(row):
Object(
dict(
r1=(test_func.func(row.a_id, row.a_name, row.b_id,
row.b_name) if row.a_id else None),
r2=(test_func.func(row.a_id, row.a_name, row.b_id,
row.b_name) if row.a_name else None),
r3=(test_func.func(row.a_id, row.a_name, row.b_id,
row.b_name) if row.b_id else None),
r4=(test_func.func(row.a_id, row.a_name, row.b_id,
row.b_name) if row.b_name else None),
))
# warm-up
for _ in _range(5000):
make_object(row)
profile2 = Profile()
profile2.enable()
for _ in _range(5000):
make_object(row)
profile2.disable()
out2 = StringIO()
stats2 = Stats(profile2, stream=out2)
stats2.strip_dirs()
stats2.sort_stats('calls').print_stats(10)
print(out2.getvalue().lstrip())
out2.close()
self.assertEqual(stats1.total_calls, stats2.total_calls)
print("Taking off")
vehicle.simple_takeoff(altitude) # take off to altitude
# Wait until vehicle reaches minimum altitude
while vehicle.location.global_relative_frame.alt < altitude * 0.95:
print("Altitude: ", vehicle.location.global_relative_frame.alt)
sleep(1)
print("Reached target altitude")
def land(vehicle):
'''Commands vehicle to land'''
print("Returning to launch")
vehicle.mode = VehicleMode("RTL")
print("Closing vehicle object")
vehicle.close()
if __name__ == "__main__":
PR = Profile()
PR.enable()
main()
PR.disable()
STREAM = StringIO()
SORT_BY = "cumulative"
PS = pstats.Stats(PR, stream=STREAM).sort_stats(SORT_BY)
PS.print_stats()
print(STREAM.getvalue())
def profile_while_timing(statement, setup=Environment, number=5000):
profile = Profile()
profile.enable()
timeit.timeit(statement, setup, number=number)
profile.disable()
profile.print_stats('cumtime')
# -*- coding: utf-8 -*-
"""
Created on Fri Jan 31 09:10:22 2020
@author: disbr007
"""
import os
from cProfile import Profile
from pstats import Stats
prof = Profile()
# Don't time imports
prof.disable()
# Import main function to run
from dem_utils import dem_valid_data
# Set output directories for profile stats
profile_dir = r'E:\disbr007\umn\ms\scratch'
stats = os.path.join(profile_dir, '{}.stats'.format(__name__))
profile_txt = os.path.join(profile_dir, '{}_profile.txt'.format(__name__))
# Turn timing back on
prof.enable()
DEMS_PATH = r'E:\disbr007\umn\ms\scratch\banks_dems_multispec_test_5.shp'
OUT_SHP = r'E:\disbr007\umn\ms\scratch\banks_dems_multispec_test_5_vp.shp'
PRJ_DIR = r'E:\disbr007\umn\ms\scratch'
SCRATCH_DIR = r'E:\disbr007\umn\ms\scratch'
LOG_FILE = r'E:\disbr007\umn\ms\scratch\vp_profile_5.log'
PROCESSED = r'E:\disbr007\umn\ms\scratch\vp_processed_5.txt'
dem_valid_data.main(DEMS_PATH=DEMS_PATH,
def run(self):
'''
Run the MLN learning with the given parameters.
'''
# load the MLN
if isinstance(self.mln, MLN):
mln = self.mln
else:
raise Exception('No MLN specified')
# load the training databases
if type(self.db) is list and all(
[isinstance(e, Database) for e in self.db]):
dbs = self.db
elif isinstance(self.db, Database):
dbs = [self.db]
elif isinstance(self.db, str):
db = self.db
if db is None or not db:
raise Exception('no trainig data given!')
dbpaths = [os.path.join(self.directory, 'db', db)]
dbs = []
for p in dbpaths:
dbs.extend(Database.load(mln, p, self.ignore_unknown_preds))
else:
raise Exception(
'Unexpected type of training databases: %s' % type(self.db))
if self.verbose:
print(('loaded %d database(s).' % len(dbs)))
watch = StopWatch()
if self.verbose:
confg = dict(self._config)
confg.update(eval("dict(%s)" % self.params))
if type(confg.get('db', None)) is list:
confg['db'] = '%d Databases' % len(confg['db'])
print((tabulate(
sorted(list(confg.items()), key=lambda key_v: str(key_v[0])),
headers=('Parameter:', 'Value:'))))
params = dict([(k, getattr(self, k)) for k in (
'multicore', 'verbose', 'profile', 'ignore_zero_weight_formulas')])
# for discriminative learning
if issubclass(self.method, DiscriminativeLearner):
if self.discr_preds == QUERY_PREDS: # use query preds
params['qpreds'] = self.qpreds
elif self.discr_preds == EVIDENCE_PREDS: # use evidence preds
params['epreds'] = self.epreds
# gaussian prior settings
if self.use_prior:
params['prior_mean'] = self.prior_mean
params['prior_stdev'] = self.prior_stdev
# expand the parameters
params.update(self.params)
if self.profile:
prof = Profile()
print('starting profiler...')
prof.enable()
else:
prof = None
# set the debug level
olddebug = logger.level
logger.level = eval('logs.%s' % params.get('debug', 'WARNING').upper())
mlnlearnt = None
try:
# run the learner
mlnlearnt = mln.learn(dbs, self.method, **params)
if self.verbose:
print()
print(headline('LEARNT MARKOV LOGIC NETWORK'))
print()
mlnlearnt.write()
except SystemExit:
print('Cancelled...')
finally:
if self.profile:
prof.disable()
print(headline('PROFILER STATISTICS'))
ps = pstats.Stats(prof, stream=sys.stdout).sort_stats(
'cumulative')
ps.print_stats()
# reset the debug level
logger.level = olddebug
print()
watch.finish()
watch.printSteps()
return mlnlearnt
class GProfiler(gtk.Window):
"""An interactive graphical profiler.
It is not visible by default; you will need to call .show() on
the returned window.
"""
def __init__(self):
super(GProfiler, self).__init__()
self.set_title(_("Layer Profiler"))
self.set_default_size(400, 300)
self.profile = Profile()
self.add(gtk.VBox())
self.ui = gtk.UIManager()
actions = gtk.ActionGroup("ProfilerActions")
actions.add_actions([
('Profiles', None, _("_Profiles")),
('OpenProfile', gtk.STOCK_OPEN, None, "<control>O", None, None),
('SaveProfile', gtk.STOCK_SAVE, None, "<control>S", None, None),
('ExportProfile', gtk.STOCK_CONVERT, _("_Export as Text"),
"<control>E", None, None),
("Quit", gtk.STOCK_QUIT, None, None, None,
lambda a: self.destroy()),
("Help", None, _("_Help")),
("About", gtk.STOCK_ABOUT, None, None, None, None),
])
self.ui.insert_action_group(actions, -1)
self.ui.add_ui_from_string(UI)
self.add_accel_group(self.ui.get_accel_group())
self.child.pack_start(self.ui.get_widget("/Menu"), expand=False)
toolbar = gtk.Toolbar()
self.child.pack_start(toolbar, expand=False, fill=True)
self.model = ProfileModel()
filtered = self.model.filter_new()
align = gtk.Alignment(xscale=1.0, yscale=1.0)
align.set_padding(0, 0, 6, 0)
label = gtk.Label()
self.entry = FilterEntry(filtered)
label.set_mnemonic_widget(self.entry)
label.set_text_with_mnemonic(_("_Filter:"))
self.view = gtk.TreeView(gtk.TreeModelSort(filtered))
self.view.connect('row-activated', self._open_file)
self.view.set_enable_search(True)
self.view.set_search_equal_func(self.model.matches)
sw = gtk.ScrolledWindow()
sw.set_policy(gtk.POLICY_NEVER, gtk.POLICY_AUTOMATIC)
sw.set_shadow_type(gtk.SHADOW_IN)
sw.add(self.view)
box = gtk.HBox(spacing=3)
box.pack_start(label, expand=False)
box.pack_start(self.entry)
align.add(box)
self.child.pack_start(align, expand=False)
self.child.pack_start(sw, expand=True)
cell = gtk.CellRendererText()
cell.props.ellipsize = pango.ELLIPSIZE_MIDDLE
column = gtk.TreeViewColumn(_("Filename"), cell)
column.add_attribute(cell, 'text', 1)
column.set_sort_column_id(2)
column.set_resizable(True)
column.set_expand(True)
self.view.append_column(column)
column = gtk.TreeViewColumn(_("Function"), cell)
column.add_attribute(cell, 'text', 2)
column.set_sort_column_id(2)
column.set_resizable(True)
column.set_expand(True)
self.view.append_column(column)
cell = gtk.CellRendererText()
column = gtk.TreeViewColumn(_("Call #"), cell)
column.add_attribute(cell, 'text', 4)
column.set_sort_column_id(4)
self.view.append_column(column)
column = gtk.TreeViewColumn(_("Total"), cell)
column.add_attribute(cell, 'text', 6)
column.set_sort_column_id(6)
self.view.append_column(column)
column = gtk.TreeViewColumn(_("Inline"), cell)
column.add_attribute(cell, 'text', 7)
column.set_sort_column_id(7)
self.view.append_column(column)
self.totalstats = gtk.Statusbar()
self.child.pack_start(self.totalstats, expand=False)
toggle = gtk.ToggleToolButton(gtk.STOCK_MEDIA_RECORD)
toggle.set_active(False)
toggle.connect('toggled', self.toggle)
toolbar.insert(toggle, 0)
self.entry.grab_focus()
self.running = False
self.child.show_all()
def _open_file(self, view, path, column):
code = view.get_model().model[path][0].code
try:
lineno = "+%d" % code.co_firstlineno
filename = code.co_filename
except AttributeError:
pass
else:
args = ['sensible-editor', lineno, filename]
gobject.spawn_async(args, flags=gobject.SPAWN_SEARCH_PATH)
def toggle(self, button):
"""Turn profiling on if off and vice versa."""
if button.get_active():
self.start()
else:
self.stop()
def snapshot(self):
"""Update the UI with a snapshot of the current profile state."""
self.model.stats = self.profile.getstats()
totalcalls = 0
totaltime = 0
for entry in self.model.stats:
totalcalls += entry.callcount
totaltime += entry.inlinetime
text = _("%(calls)d calls in %(time)f CPU seconds.") % dict(
calls=totalcalls, time=totaltime)
self.totalstats.pop(0)
self.totalstats.push(0, text)
def start(self):
"""Start profiling (adding to existing stats)."""
if not self.running:
self.profile.enable()
self.running = True
def stop(self):
"""Stop profiling (but retain stats)."""
if self.running:
self.profile.disable()
self.running = False
self.snapshot()
try:
print("clusterize")
kmeans.similarity_method = tfidf.euclidean_similarity
clusters = kmeans.clusterize()
# for cluster_name, cluster in clusters.items():
# print(cluster_name, "has", len(cluster.items), "docs")
# for item in cluster.items:
# print('\t',item.name)
items = list(
chain.from_iterable(cluster.items for cluster in clusters.values()))
# with open("results.csv", "w") as csv_file:
# writer = csv.DictWriter(csv_file, ["name"] + [item.name for item in items])
# writer.writeheader()
# for cu_item in items:
# d = {item.name:cosine_similarity(item.content, cu_item.content) for item in items}
# d["name"] = cu_item.name
# writer.writerow(d)
except KeyboardInterrupt:
print("interupted")
else:
print('======================')
for cluster_name, cluster in clusters.items():
print(cluster_name, "has", len(cluster.items), "docs")
finally:
profiler.disable()
profiler.dump_stats("profile.txt")
print("we have profile")
# profiler.print_stats()
def main(*args, **kwargs):
"""
Entry point for the 'bw' command line utility.
The args and path parameters are used for integration tests.
"""
if not args:
args = argv[1:]
text_args = [force_text(arg) for arg in args]
parser_bw = build_parser_bw()
pargs = parser_bw.parse_args(args)
if not hasattr(pargs, 'func'):
parser_bw.print_help()
exit(2)
if pargs.profile:
profile = Profile()
profile.enable()
path = abspath(pargs.repo_path)
io.debug_mode = pargs.debug
io.activate()
io.debug(_("invocation: {}").format(" ".join([force_text(arg) for arg in argv])))
environ.setdefault('BW_ADD_HOST_KEYS', "1" if pargs.add_ssh_host_keys else "0")
if len(text_args) >= 1 and (
text_args[0] == "--version" or
(len(text_args) >= 2 and text_args[0] == "repo" and text_args[1] == "create") or
text_args[0] == "zen" or
"-h" in text_args or
"--help" in text_args
):
# 'bw repo create' is a special case that only takes a path
repo = path
else:
while True:
try:
repo = Repository(path)
break
except NoSuchRepository:
if path == dirname(path):
io.stderr(_(
"{x} {path} "
"is not a BundleWrap repository."
).format(path=quote(abspath(pargs.repo_path)), x=red("!!!")))
io.deactivate()
exit(1)
else:
path = dirname(path)
except MissingRepoDependency as exc:
io.stderr(str(exc))
io.deactivate()
exit(1)
except Exception:
io.stderr(format_exc())
io.deactivate()
exit(1)
# convert all string args into text
text_pargs = {key: force_text(value) for key, value in vars(pargs).items()}
try:
pargs.func(repo, text_pargs)
finally:
io.deactivate()
if pargs.profile:
profile.disable()
profile.dump_stats(pargs.profile)
def run(self):
watch = StopWatch()
watch.tag('inference', self.verbose)
# load the MLN
if isinstance(self.mln, MLN):
mln = self.mln
else:
raise Exception('No MLN specified')
if self.use_emln and self.emln is not None:
mlnstr = StringIO.StringIO()
mln.write(mlnstr)
mlnstr.close()
mlnstr = str(mlnstr)
emln = self.emln
mln = parse_mln(mlnstr + emln,
grammar=self.grammar,
logic=self.logic)
# load the database
if isinstance(self.db, Database):
db = self.db
elif isinstance(self.db, list) and len(self.db) == 1:
db = self.db[0]
elif isinstance(self.db, list):
raise Exception(
'Got {} dbs. Can only handle one for inference.'.format(
len(self.db)))
else:
raise Exception('DB of invalid format {}'.format(type(self.db)))
# expand the
# parameters
params = dict(self._config)
if 'params' in params:
params.update(eval("dict(%s)" % params['params']))
del params['params']
if self.verbose:
print tabulate(sorted(list(params.viewitems()),
key=lambda (k, v): str(k)),
headers=('Parameter:', 'Value:'))
# create the MLN and evidence database and the parse the queries
# mln = parse_mln(modelstr, searchPath=self.dir.get(), logic=self.config['logic'], grammar=self.config['grammar'])
# db = parse_db(mln, db_content, ignore_unknown_preds=params.get('ignore_unknown_preds', False))
if type(db) is list and len(db) > 1:
raise Exception('Inference can only handle one database at a time')
elif type(db) is list:
db = db[0]
# parse non-atomic params
# if type(self.queries) is not list:
# queries = parse_queries(mln, str(self.queries))
params['cw_preds'] = filter(lambda x: bool(x), self.cw_preds)
# extract and remove all non-algorithm
for s in GUI_SETTINGS:
if s in params: del params[s]
if self.profile:
prof = Profile()
print 'starting profiler...'
prof.enable()
# set the debug level
olddebug = praclog.level()
praclog.level(
eval('logging.%s' % params.get('debug', 'WARNING').upper()))
result = None
try:
mln_ = mln.materialize(db)
mrf = mln_.ground(db)
inference = self.method(mrf, self.queries, **params)
if self.verbose:
print
print headline('EVIDENCE VARIABLES')
print
mrf.print_evidence_vars()
result = inference.run()
if self.verbose:
print
print headline('INFERENCE RESULTS')
print
inference.write()
if self.verbose:
print
inference.write_elapsed_time()
except SystemExit:
print 'Cancelled...'
finally:
if self.profile:
prof.disable()
print headline('PROFILER STATISTICS')
ps = pstats.Stats(prof,
stream=sys.stdout).sort_stats('cumulative')
ps.print_stats()
# reset the debug level
praclog.level(olddebug)
if self.verbose:
print
watch.finish()
watch.printSteps()
return result
