def bench_publish(endpoint, appkey, channel, size, profile, ack=True):
publisher = satori.rtm.connection.Connection(endpoint + '?appkey=' +
appkey)
publisher.start()
message = binascii.hexlify(os.urandom(size // 2)).decode('ascii')
print('Message size is {}'.format(len(message)))
last_usage = [resource.getrusage(resource.RUSAGE_SELF)]
print(
'Duration, s\tRate, msgs/s\tMax RSS, MB\tUser time, s\tSystem time, s')
def report(duration, count):
usage = resource.getrusage(resource.RUSAGE_SELF)
maxrss = usage.ru_maxrss // 1024
if sys.platform == 'darwin':
maxrss = maxrss // 1024
print('{0:2.2f}\t\t{1}\t\t{2}\t\t{3:2.2f}\t\t{4:2.2f}'.format(
duration, int(count / duration), maxrss,
usage.ru_utime - last_usage[0].ru_utime,
usage.ru_stime - last_usage[0].ru_stime))
sys.stdout.flush()
last_usage[0] = usage
count = [0]
def publish_without_ack():
publisher.publish(channel, message)
count[0] += 1
def publish_with_ack():
def callback(ack):
count[0] += 1
publisher.publish(channel, message, callback)
publish = publish_with_ack if ack else publish_without_ack
before = time.time()
try:
if profile:
profiler = Profiler()
profiler.start()
while True:
now = time.time()
if now - before >= sampling_interval:
report(now - before, count[0])
if profile:
profiler.stop()
print(profiler.output_text(unicode=True, color=True))
profiler = Profiler()
profiler.start()
count[0] = 0
before = time.time()
publish()
except KeyboardInterrupt:
sys.exit(0)
def _wrapper(*args, **kwargs):
_profiler = Profiler(
sampling_interval_seconds
) if sampling_interval_seconds is not None else Profiler()
_profiler.start()
result = _func(*args, **kwargs)
_profiler.stop()
if print_to_console:
print(_profiler.output_text(unicode=True, color=True))
if log_to_file:
name = file_name if file_name is not None else _func.__name__ + ".profile"
with open(f"{name}.html", "w") as f:
f.write(_profiler.output_html())
return result
def profile_ctx(engine='pyinstrument'):
"""
A context manager which profiles the body of the with statement
with the supplied profiling engine and returns the profiling object
in a list.
Arguments
---------
engine: str
The profiling engine, e.g. 'pyinstrument' or 'snakeviz'
Returns
-------
sessions: list
A list containing the profiling session.
"""
if engine == 'pyinstrument':
from pyinstrument import Profiler
prof = Profiler()
prof.start()
elif engine == 'snakeviz':
prof = Profile()
prof.enable()
elif engine is None:
pass
sessions = []
yield sessions
if engine == 'pyinstrument':
sessions.append(prof.stop())
elif engine == 'snakeviz':
prof.disable()
sessions.append(prof)
def wrapper(*args, **kwargs):
profiler = Profiler()
profiler.start()
r = func(*args, **kwargs)
profiler.stop()
print profiler.output_text(color=True)
return r
def __init__(self, conf, grass_use_file, args):
self.conf = conf
self.grass_use_file = grass_use_file
if args.p:
self.prof = Profiler()
else:
self.prof = None
def forward(self, x):
profiler = Profiler()
profiler.start()
x = normalization(x)
if x.shape[-1] > 3:
pos = x[:, :, :3]
feat = x[:, :, 3:]
else:
pos = x
feat = None
index_voxels, context_points, mask = self.selfvoxels(pos)
pos, feat = self.sa_module1(pos, feat, index_voxels, context_points,
mask)
index_voxels, context_points, mask = self.selfvoxels(pos)
pos, feat = self.sa_module2(pos, feat, index_voxels, context_points,
mask)
index_voxels, context_points, mask = self.selfvoxels(pos)
h = self.sa_module3(pos, feat, index_voxels, context_points, mask)
h = self.mlp1(h)
h = self.bn1(h)
h = F.relu(h)
h = self.drop1(h)
h = self.mlp2(h)
h = self.bn2(h)
h = F.relu(h)
h = self.drop2(h)
out = self.mlp_out(h)
profiler.stop()
print(profiler.output_text(unicode=True, color=True))
return out
def new_func(*args, **kwargs):
import time
from flask import has_request_context, g, current_app, request
if (not has_request_context() or "profile" in request.args
or not current_app.config.get("PROFILE")
or current_app.config.get("PROFILE") == "False"):
return func(*args, **kwargs)
g.profiler = Profiler() if "profiler" not in g else g.profiler
result = None
with g.profiler:
result = func(*args, **kwargs)
# 写入文件
if g.profiler._last_session.duration * 1000 > timeout:
time_format = time.strftime("%Y%m%d_%H:%M:%S",
time.localtime())
file_path = f"./log/profile_{func.__name__}_{time_format}.txt"
profile_file = open(file_path, "w")
with profile_file:
profile_file.write(g.profiler.output_text())
# 返回
return result
def profile(self, fct, **kwargs):
"""
Profiles function *fct*, calls it
*repeat* times.
@param fct function to profile (no argument)
@param kwargs stores additional information
about the profiling
"""
if self.module in ('pyinstrument', 'cProfile'):
if self.module == 'pyinstrument':
profiler = Profiler(interval=self.interval)
start = profiler.start
stop = profiler.stop
else:
profiler = c_Profile()
start = profiler.enable
stop = profiler.disable
start()
for _ in range(self.repeat):
fct()
stop()
elif self.module == "profile":
profiler = py_Profile()
def lf():
for _ in range(self.repeat):
fct()
profiler.runcall(lf)
else:
raise ValueError( # pragma: no cover
"Unknown profiler '{}'.".format(self.module))
self.profiled.append(profiler)
self.kwargs.append(kwargs)
def instrument(
cls_runner: tp.Type[Perf],
pattern_func: str,
timeline: bool = False,
) -> None:
'''
Profile the `sf` function from the supplied class.
'''
runner = cls_runner()
for name in runner.iter_function_names(pattern_func):
f = getattr(runner, name)
profiler = Profiler(interval=0.0001) # default is 0.001, 1 ms
if timeline:
profiler.start()
f()
profiler.stop()
else:
profiler.start()
for _ in range(runner.NUMBER):
f()
profiler.stop()
print(
profiler.output_text(unicode=True,
color=True,
timeline=timeline,
show_all=True))
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
# creating random file for each usage of this function
f= open("/home/"+str(random.randint(1,1100))+".html","w+")
profiler = Profiler(use_signal=False)
profiler.start()
n = int(req.params.get('n', 15))
if not n:
try:
req_body = req.get_json()
except ValueError:
pass
else:
name = req_body.get('n')
if n:
output = str(fibnonci_approach(n))
# stopping profiler and writing it to random file generated earlier
profiler.stop()
f.write(profiler.output_html())
return func.HttpResponse("Fibnonci of " + str(n) + " using regular approach:" + output)
else:
return func.HttpResponse(
"Please pass a name on the query string or in the request body",
status_code=400
)
def test_get_symmetries():
"""Check that the function correctly creates six new reflected boards and
policy vectors, one for each axis of symmetry.
"""
game = BloomsGame(size=4, score_target=15)
board = game.getInitBoard()
# Place stones
board.place_stone(position=(3, 1), colour=1)
board.place_stone(position=(5, 1), colour=2)
board.place_stone(position=(3, 5), colour=3)
board.place_stone(position=(1, 5), colour=4)
# board.visualise(show_coords=True, title="Original")
# Create a dummy policy vector
pi = np.random.random_sample(game.getActionSize())
profiler = Profiler()
profiler.start()
symmetrical_states = game.getSymmetries(board, pi)
profiler.stop()
print(profiler.output_text(unicode=True, color=True))
assert len(symmetrical_states) == 24
assert symmetrical_states[1][1][37] == pi[52]
assert symmetrical_states[1][1][614] == pi[1277]
assert all(
[sum(x[1]) == pytest.approx(sum(pi)) for x in symmetrical_states])
def _report_from_filters(
self, n_clicks, dd_name, input_name, *filter_values
):
if n_clicks == 0:
raise ValueError("no true report request, only toggle")
stime = time.time()
name = input_name
for elem in dash.callback_context.triggered:
if ElemIds.is_dd(elem["prop_id"]):
name = dd_name
logger.info("start filtering")
if self._timing:
logger.info("starting time profiling")
profiler = Profiler()
profiler.start()
filtered_ddf = self._filter_ddf(filter_values)
logger.info("calculating elements")
elems = self.get_report_elems(filtered_ddf)
logger.info(f"{len(elems)} elements returned")
logger.info("parsing elements to output values")
outputs = self._parse_report_elems_to_outputs(elems, name)
logger.info(f"{len(outputs)} outputs parsed")
logger.info("report done", runtime=round(time.time() - stime, 3))
if self._timing:
profiler.stop()
log_profiler(profiler, logger, 20)
return (name, name, *outputs)
def forward(self, x):
profiler = Profiler()
profiler.start()
if x.shape[-1] > 3:
pos = x[:, :, :3]
feat = x[:, :, 3:]
else:
pos = x
feat = None
pos, feat = self.sa_msg_module1(pos, feat)
pos, feat = self.sa_msg_module2(pos, feat)
h = self.sa_module3(pos, feat)
h = self.mlp1(h)
h = self.bn1(h)
h = F.relu(h)
h = self.drop1(h)
h = self.mlp2(h)
h = self.bn2(h)
h = F.relu(h)
h = self.drop2(h)
out = self.mlp_out(h)
profiler.stop()
print(profiler.output_text(unicode=True, color=True))
return out
def profile(output_format='html', output_filename=None, html_open=False):
"""Profiles a block of code with Pyinstrument.
Intended to be used in a `with` statement.
Parameters
----------
output_format : {'html', 'text'}, optional
Shows the result either as text or in an HTML file. If :code:`output_format = 'html'`,
the file is saved according to the :code:`output_filename` parameter.
The default is 'html'.
output_filename : str, optional
Only taken into account if :code:`output_format = 'html'`.
If not given (default), the html output is saved to the same directory the caller resides.
The name of the html file is the same as that of the caller.
html_open : bool, optional
Only taken into account if :code:`output_format = 'html'`.
If True, the generated HTML file is opened in the default browser.
The default is False.
Yields
------
Notes
-----
In the implementation, we move two levels up in the stack frame, one for exiting the context
manager and one for exiting this generator. This assumes that :meth:`profile` was called as
a context manager. As a good practice, provide the :code:`output_filename` input argument.
Examples
--------
Measure the time needed to generate 1 million uniformly distributed random numbers.
>>> import random
>>> with profile('html') as p:
... for _ in range(1000000):
... rand_num = random.uniform(1, 2.2)
"""
# If required, guess the path where the results will be saved
if output_format == 'html' and not output_filename:
caller = inspect.currentframe().f_back.f_back.f_locals['__file__']
output_filename = os.path.splitext(caller)[0] + '.html'
# Start the profiler
profiler = Profiler()
profiler.start()
# Give back the execution to the caller function
yield
# Finish profiling and show the results
profiler.stop()
if output_format == 'html':
if html_open:
HTMLRenderer().open_in_browser(profiler.last_session,
output_filename=output_filename)
else:
with codecs.open(output_filename, 'w', 'utf-8') as f:
f.write(HTMLRenderer().render(profiler.last_session))
elif output_format == 'text':
print(profiler.output_text(unicode=True, color=True, show_all=True))
def test_context_manager():
with Profiler() as profiler:
long_function_a()
long_function_b()
frame = profiler.first_interesting_frame()
assert frame.function == 'test_context_manager'
assert len(frame.children) == 2
def wrapper(*args, **kwargs):
from pyinstrument import Profiler
profiler = Profiler()
profiler.start()
result = func(*args, **kwargs)
profiler.stop()
print(profiler.output_text(unicode=True, color=True))
return result
def test_context_manager():
with Profiler() as profiler:
long_function_a()
long_function_b()
frame = profiler.last_session.root_frame()
assert frame.function == 'test_context_manager'
assert len(frame.children) == 2
def train_models(
self,
partitions_path,
graphs_path,
init_lr=1e-4,
train_batch_size=32,
valid_batch_size=32,
epoch_batch=5,
epochs=100,
checkpoint_path="checkpoint/aux_30_epoch_", # This version only looks at race + age (not gender)
):
with open(partitions_path, "rb") as fp: # Unpickling
set_partitions = pickle.load(fp)
# Training Auxiliary Models!
for i in range(0, int(epochs / epoch_batch)):
data_generator_aux = UtkFaceDataGeneratorAuxOneModel(
self.dataset_folder_name,
set_partitions[i],
self.dataset_dict,
num_classes=self.num_classes)
aux_train_idx, aux_valid_idx, _ = data_generator_aux.generate_split_indexes(
)
train_images_collection, train_status_collection = data_generator_aux.pre_generate_images(
aux_train_idx, batch_size=train_batch_size)
valid_images_collection, valid_status_collection = data_generator_aux.pre_generate_images(
aux_valid_idx, batch_size=valid_batch_size)
aux_train_gen = data_generator_aux.generate_images(
is_training=True,
images_collection=train_images_collection,
status_collection=train_status_collection)
aux_valid_gen = data_generator_aux.generate_images(
is_training=True,
images_collection=valid_images_collection,
status_collection=valid_status_collection)
aux_model = self.build_model()
es = EarlyStopping(monitor='val_loss', mode='min', patience=10)
profiler = Profiler()
profiler.start()
history = aux_model.fit(
aux_train_gen,
steps_per_epoch=len(aux_train_idx) // train_batch_size,
epochs=(i + 1) * 5,
validation_data=aux_valid_gen,
validation_steps=len(aux_valid_idx) // valid_batch_size,
callbacks=[es])
profiler.stop()
profiler.print(show_all=True)
aux_model.save(str(checkpoint_path) + "_" + str((i + 1) * 5))
y = history.history['val_loss']
plt.plot([i for i in range(len(y))], history.history['val_loss'])
plt.title("Auxiliary Model Validation Loss - {} Epochs".format(
(i + 1) * 5))
plt.savefig(graphs_path / "aux_30_epoch_val_loss_{}".format(
(i + 1) * 5))
def process_request(self, request):
profile_dir = getattr(settings, 'PYINSTRUMENT_PROFILE_DIR', None)
if getattr(settings, 'PYINSTRUMENT_URL_ARGUMENT',
'profile') in request.GET or profile_dir:
profiler = Profiler()
profiler.start()
request.profiler = profiler
async def main():
p = Profiler(async_mode='disabled')
with p:
for _ in range(10_000):
await asyncio.gather(
example(random.randint(10, 1000)),
example(random.randint(10, 10000)),
example(random.randint(10, 1000)),
)
def process_request(self, request):
if getattr(settings, 'PYINSTRUMENT_URL_ARGUMENT',
'profile') in request.GET:
self.profiler = Profiler()
try:
self.profiler.start()
except NotMainThreadError:
raise NotMainThreadError(not_main_thread_message)
self.profiler = None
def train_model(self,
init_lr=1e-4,
train_batch_size=16,
valid_batch_size=16,
epoch_batch=5,
epochs=100,
checkpoint_dir="checkpoint/base_epochs_"):
opt = Adam(lr=init_lr, decay=init_lr / epochs)
self.model.compile(optimizer=opt,
loss={
'age_output': 'mse',
'race_output': 'categorical_crossentropy',
'gender_output': 'binary_crossentropy'
},
loss_weights={
'age_output': 4.,
'race_output': 1.5,
'gender_output': 0.1
},
metrics={
'age_output': 'mae',
'race_output': 'accuracy',
'gender_output': 'accuracy'
})
for i in range(int(epochs / epoch_batch)):
current_checkpoint = checkpoint_dir / "base_epochs_{}".format(
str((i + 1) * epoch_batch))
if i != 0:
self.model = load_model(
checkpoint_dir /
"base_epochs_{}".format(str((i) * epoch_batch)))
print(len(self.train_idx), len(self.valid_idx))
train_images_collection, train_status_collection = self.data_generator.pre_generate_images(
self.train_idx, batch_size=train_batch_size)
valid_images_collection, valid_status_collection = self.data_generator.pre_generate_images(
self.valid_idx, batch_size=valid_batch_size)
train_gen = self.data_generator.generate_images(
is_training=True,
images_collection=train_images_collection,
status_collection=train_status_collection)
valid_gen = self.data_generator.generate_images(
is_training=True,
images_collection=valid_images_collection,
status_collection=valid_status_collection)
profiler = Profiler()
profiler.start()
history = self.model.fit_generator(
train_gen,
steps_per_epoch=len(self.train_idx) // train_batch_size,
# epochs=epoch_batch,
epochs=1,
validation_data=valid_gen,
validation_steps=len(self.valid_idx) // valid_batch_size)
profiler.stop()
profiler.print(show_all=True)
def profile():
profiler = Profiler()
profiler.start()
exit_code = runtests.run_tests_all(parallel=False)
profiler.stop()
print(profiler.output_text(unicode=False, color=True))
return exit_code
def setUp(self):
"""Create a test dataset."""
self.profiler = Profiler()
dataset_file = '../tests/data/tomo_setup.pickle.lzma'
with lzma.open(dataset_file, 'rb') as file:
[
self.data,
self.theta,
self.original,
] = pickle.load(file)
def setUp(self):
self.profiler = Profiler()
dataset_file = '../tests/data/lamino_setup.pickle.lzma'
with lzma.open(dataset_file, 'rb') as file:
[
self.data,
self.original,
self.theta,
self.tilt,
] = pickle.load(file)
def pytinstrument_decorator(*args, **kwargs):
# Create profiler instance before function is called.
profiler = Profiler()
profiler.start()
# Call function.
func_return = func(*args, **kwargs)
# Stop profiler and print results after function is called.
profiler.stop()
print(profiler.output_text(unicode=False, color=True))
return func_return
def test_profiler_doesnt_overflow_on_large_call_stacks():
profiler = Profiler()
profiler.start()
# give 170 frames of leeway for the test runner and for pyinstrument to do its work.
recursion_depth = sys.getrecursionlimit() - 170
recurse(recursion_depth)
profiler.stop()
print(profiler.output_text())
def deep_profiler_session():
profiler = Profiler()
profiler.start()
# give 120 frames for pyinstrument to do its work.
recursion_depth = sys.getrecursionlimit() - current_stack_depth() - 120
recurse(recursion_depth)
profiler.stop()
return profiler.last_session
def test_profiler_doesnt_overflow_on_large_call_stacks():
profiler = Profiler()
profiler.start()
# give ourselves 150 frames of leeway to do our work.
recursion_depth = sys.getrecursionlimit() - 150
recurse(recursion_depth)
profiler.stop()
print(profiler.output_text())
def wrapper(*args, **kwargs):
profiler = None
if CONFIG.profiler_enable:
profiler = Profiler()
profiler.start()
try:
return f(*args, **kwargs)
finally:
if profiler is not None:
profiler.stop()
print(profiler.output_text(unicode=True, color=True))
