def test_get_symmetries():
"""Check that the function correctly creates six new reflected boards and
policy vectors, one for each axis of symmetry.
"""
game = BloomsGame(size=4, score_target=15)
board = game.getInitBoard()
# Place stones
board.place_stone(position=(3, 1), colour=1)
board.place_stone(position=(5, 1), colour=2)
board.place_stone(position=(3, 5), colour=3)
board.place_stone(position=(1, 5), colour=4)
# board.visualise(show_coords=True, title="Original")
# Create a dummy policy vector
pi = np.random.random_sample(game.getActionSize())
profiler = Profiler()
profiler.start()
symmetrical_states = game.getSymmetries(board, pi)
profiler.stop()
print(profiler.output_text(unicode=True, color=True))
assert len(symmetrical_states) == 24
assert symmetrical_states[1][1][37] == pi[52]
assert symmetrical_states[1][1][614] == pi[1277]
assert all(
[sum(x[1]) == pytest.approx(sum(pi)) for x in symmetrical_states])
class ProfilerMiddleware(object):
def __init__(self):
self.profiler = None
def process_request(self, request):
if getattr(settings, 'PYINSTRUMENT_URL_ARGUMENT',
'profile') in request.GET:
self.profiler = Profiler()
try:
self.profiler.start()
except NotMainThreadError:
raise NotMainThreadError(not_main_thread_message)
self.profiler = None
def process_response(self, request, response):
if self.profiler:
try:
self.profiler.stop()
return HttpResponse(self.profiler.output_html())
except NotMainThreadError:
raise NotMainThreadError(not_main_thread_message)
finally:
self.profiler = None
else:
return response
class ProfilerMiddleware(object):
def __init__(self):
self.profiler = None
def process_request(self, request):
if getattr(settings, 'PYINSTRUMENT_URL_ARGUMENT', 'profile') in request.GET:
self.profiler = Profiler()
try:
self.profiler.start()
except NotMainThreadError:
raise NotMainThreadError(not_main_thread_message)
self.profiler = None
def process_response(self, request, response):
if self.profiler:
try:
self.profiler.stop()
return HttpResponse(self.profiler.output_html())
except NotMainThreadError:
raise NotMainThreadError(not_main_thread_message)
finally:
self.profiler = None
else:
return response
class BenchmarkTomo(unittest.TestCase):
"""Run benchmarks for tomography reconstruction."""
def setUp(self):
"""Create a test dataset."""
self.profiler = Profiler()
dataset_file = '../tests/data/tomo_setup.pickle.lzma'
with lzma.open(dataset_file, 'rb') as file:
[
self.data,
self.theta,
self.original,
] = pickle.load(file)
@unittest.skip('Demonstrate skipped tests.')
def test_never(self):
"""Never run this test."""
pass
def test_cgrad(self):
"""Use pyinstrument to benchmark tomo.grad on one core."""
logging.disable(logging.WARNING)
result = {'obj': np.zeros(self.original.shape, dtype=np.complex64)}
self.profiler.start()
for i in range(50):
result = tike.tomo.reconstruct(
**result,
theta=self.theta,
integrals=self.data,
algorithm='cgrad',
num_iter=1,
)
self.profiler.stop()
print('\n')
print(self.profiler.output_text(unicode=True, color=True))
def _report_from_filters(
self, n_clicks, dd_name, input_name, *filter_values
):
if n_clicks == 0:
raise ValueError("no true report request, only toggle")
stime = time.time()
name = input_name
for elem in dash.callback_context.triggered:
if ElemIds.is_dd(elem["prop_id"]):
name = dd_name
logger.info("start filtering")
if self._timing:
logger.info("starting time profiling")
profiler = Profiler()
profiler.start()
filtered_ddf = self._filter_ddf(filter_values)
logger.info("calculating elements")
elems = self.get_report_elems(filtered_ddf)
logger.info(f"{len(elems)} elements returned")
logger.info("parsing elements to output values")
outputs = self._parse_report_elems_to_outputs(elems, name)
logger.info(f"{len(outputs)} outputs parsed")
logger.info("report done", runtime=round(time.time() - stime, 3))
if self._timing:
profiler.stop()
log_profiler(profiler, logger, 20)
return (name, name, *outputs)
def instrument(
cls_runner: tp.Type[Perf],
pattern_func: str,
timeline: bool = False,
) -> None:
'''
Profile the `sf` function from the supplied class.
'''
runner = cls_runner()
for name in runner.iter_function_names(pattern_func):
f = getattr(runner, name)
profiler = Profiler(interval=0.0001) # default is 0.001, 1 ms
if timeline:
profiler.start()
f()
profiler.stop()
else:
profiler.start()
for _ in range(runner.NUMBER):
f()
profiler.stop()
print(
profiler.output_text(unicode=True,
color=True,
timeline=timeline,
show_all=True))
def forward(self, x):
profiler = Profiler()
profiler.start()
x = normalization(x)
if x.shape[-1] > 3:
pos = x[:, :, :3]
feat = x[:, :, 3:]
else:
pos = x
feat = None
index_voxels, context_points, mask = self.selfvoxels(pos)
pos, feat = self.sa_module1(pos, feat, index_voxels, context_points,
mask)
index_voxels, context_points, mask = self.selfvoxels(pos)
pos, feat = self.sa_module2(pos, feat, index_voxels, context_points,
mask)
index_voxels, context_points, mask = self.selfvoxels(pos)
h = self.sa_module3(pos, feat, index_voxels, context_points, mask)
h = self.mlp1(h)
h = self.bn1(h)
h = F.relu(h)
h = self.drop1(h)
h = self.mlp2(h)
h = self.bn2(h)
h = F.relu(h)
h = self.drop2(h)
out = self.mlp_out(h)
profiler.stop()
print(profiler.output_text(unicode=True, color=True))
return out
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
# creating random file for each usage of this function
f= open("/home/"+str(random.randint(1,1100))+".html","w+")
profiler = Profiler(use_signal=False)
profiler.start()
n = int(req.params.get('n', 15))
if not n:
try:
req_body = req.get_json()
except ValueError:
pass
else:
name = req_body.get('n')
if n:
output = str(fibnonci_approach(n))
# stopping profiler and writing it to random file generated earlier
profiler.stop()
f.write(profiler.output_html())
return func.HttpResponse("Fibnonci of " + str(n) + " using regular approach:" + output)
else:
return func.HttpResponse(
"Please pass a name on the query string or in the request body",
status_code=400
)
class ManagerProfilePyinstrument(ManagerProfile):
def __init__(self, sync: bool):
self.sync = sync
self.profiler = Profiler()
def start(self):
self.profiler.start()
return nullcontext()
def stop_and_write(self,
path_profile: str,
is_docker: bool,
api: str,
render_browser: bool = False):
self.profiler.stop()
mode = "sync" if self.sync else "async"
filename = f"pyinstrument_profile_{mode}_{api}.html"
if not is_docker:
output_html = self.profiler.output_html()
self._write_output_file(path_profile,
output_html,
filename=filename)
if render_browser:
self.profiler.open_in_browser()
print(self.profiler.output_text(unicode=True, color=True))
def _write_output_file(self, path_profile: str, output_html: str,
filename: str):
output_html_path = self._prepare_output_path(path_profile, filename)
with open(output_html_path, "w") as file:
file.write(output_html)
def wrapper(*args, **kwargs):
profiler = Profiler()
profiler.start()
r = func(*args, **kwargs)
profiler.stop()
print profiler.output_text(color=True)
return r
def wrapper(*args, **kwargs):
profiler = Profiler()
profiler.start()
r = func(*args, **kwargs)
profiler.stop()
print profiler.output_text(color=True)
return r
def forward(self, x):
profiler = Profiler()
profiler.start()
if x.shape[-1] > 3:
pos = x[:, :, :3]
feat = x[:, :, 3:]
else:
pos = x
feat = None
pos, feat = self.sa_msg_module1(pos, feat)
pos, feat = self.sa_msg_module2(pos, feat)
h = self.sa_module3(pos, feat)
h = self.mlp1(h)
h = self.bn1(h)
h = F.relu(h)
h = self.drop1(h)
h = self.mlp2(h)
h = self.bn2(h)
h = F.relu(h)
h = self.drop2(h)
out = self.mlp_out(h)
profiler.stop()
print(profiler.output_text(unicode=True, color=True))
return out
def profile(output_format='html', output_filename=None, html_open=False):
"""Profiles a block of code with Pyinstrument.
Intended to be used in a `with` statement.
Parameters
----------
output_format : {'html', 'text'}, optional
Shows the result either as text or in an HTML file. If :code:`output_format = 'html'`,
the file is saved according to the :code:`output_filename` parameter.
The default is 'html'.
output_filename : str, optional
Only taken into account if :code:`output_format = 'html'`.
If not given (default), the html output is saved to the same directory the caller resides.
The name of the html file is the same as that of the caller.
html_open : bool, optional
Only taken into account if :code:`output_format = 'html'`.
If True, the generated HTML file is opened in the default browser.
The default is False.
Yields
------
Notes
-----
In the implementation, we move two levels up in the stack frame, one for exiting the context
manager and one for exiting this generator. This assumes that :meth:`profile` was called as
a context manager. As a good practice, provide the :code:`output_filename` input argument.
Examples
--------
Measure the time needed to generate 1 million uniformly distributed random numbers.
>>> import random
>>> with profile('html') as p:
... for _ in range(1000000):
... rand_num = random.uniform(1, 2.2)
"""
# If required, guess the path where the results will be saved
if output_format == 'html' and not output_filename:
caller = inspect.currentframe().f_back.f_back.f_locals['__file__']
output_filename = os.path.splitext(caller)[0] + '.html'
# Start the profiler
profiler = Profiler()
profiler.start()
# Give back the execution to the caller function
yield
# Finish profiling and show the results
profiler.stop()
if output_format == 'html':
if html_open:
HTMLRenderer().open_in_browser(profiler.last_session,
output_filename=output_filename)
else:
with codecs.open(output_filename, 'w', 'utf-8') as f:
f.write(HTMLRenderer().render(profiler.last_session))
elif output_format == 'text':
print(profiler.output_text(unicode=True, color=True, show_all=True))
def wrapper(*args, **kwargs):
from pyinstrument import Profiler
profiler = Profiler()
profiler.start()
result = func(*args, **kwargs)
profiler.stop()
print(profiler.output_text(unicode=True, color=True))
return result
def train_models(
self,
partitions_path,
graphs_path,
init_lr=1e-4,
train_batch_size=32,
valid_batch_size=32,
epoch_batch=5,
epochs=100,
checkpoint_path="checkpoint/aux_30_epoch_", # This version only looks at race + age (not gender)
):
with open(partitions_path, "rb") as fp: # Unpickling
set_partitions = pickle.load(fp)
# Training Auxiliary Models!
for i in range(0, int(epochs / epoch_batch)):
data_generator_aux = UtkFaceDataGeneratorAuxOneModel(
self.dataset_folder_name,
set_partitions[i],
self.dataset_dict,
num_classes=self.num_classes)
aux_train_idx, aux_valid_idx, _ = data_generator_aux.generate_split_indexes(
)
train_images_collection, train_status_collection = data_generator_aux.pre_generate_images(
aux_train_idx, batch_size=train_batch_size)
valid_images_collection, valid_status_collection = data_generator_aux.pre_generate_images(
aux_valid_idx, batch_size=valid_batch_size)
aux_train_gen = data_generator_aux.generate_images(
is_training=True,
images_collection=train_images_collection,
status_collection=train_status_collection)
aux_valid_gen = data_generator_aux.generate_images(
is_training=True,
images_collection=valid_images_collection,
status_collection=valid_status_collection)
aux_model = self.build_model()
es = EarlyStopping(monitor='val_loss', mode='min', patience=10)
profiler = Profiler()
profiler.start()
history = aux_model.fit(
aux_train_gen,
steps_per_epoch=len(aux_train_idx) // train_batch_size,
epochs=(i + 1) * 5,
validation_data=aux_valid_gen,
validation_steps=len(aux_valid_idx) // valid_batch_size,
callbacks=[es])
profiler.stop()
profiler.print(show_all=True)
aux_model.save(str(checkpoint_path) + "_" + str((i + 1) * 5))
y = history.history['val_loss']
plt.plot([i for i in range(len(y))], history.history['val_loss'])
plt.title("Auxiliary Model Validation Loss - {} Epochs".format(
(i + 1) * 5))
plt.savefig(graphs_path / "aux_30_epoch_val_loss_{}".format(
(i + 1) * 5))
def train_model(self,
init_lr=1e-4,
train_batch_size=16,
valid_batch_size=16,
epoch_batch=5,
epochs=100,
checkpoint_dir="checkpoint/base_epochs_"):
opt = Adam(lr=init_lr, decay=init_lr / epochs)
self.model.compile(optimizer=opt,
loss={
'age_output': 'mse',
'race_output': 'categorical_crossentropy',
'gender_output': 'binary_crossentropy'
},
loss_weights={
'age_output': 4.,
'race_output': 1.5,
'gender_output': 0.1
},
metrics={
'age_output': 'mae',
'race_output': 'accuracy',
'gender_output': 'accuracy'
})
for i in range(int(epochs / epoch_batch)):
current_checkpoint = checkpoint_dir / "base_epochs_{}".format(
str((i + 1) * epoch_batch))
if i != 0:
self.model = load_model(
checkpoint_dir /
"base_epochs_{}".format(str((i) * epoch_batch)))
print(len(self.train_idx), len(self.valid_idx))
train_images_collection, train_status_collection = self.data_generator.pre_generate_images(
self.train_idx, batch_size=train_batch_size)
valid_images_collection, valid_status_collection = self.data_generator.pre_generate_images(
self.valid_idx, batch_size=valid_batch_size)
train_gen = self.data_generator.generate_images(
is_training=True,
images_collection=train_images_collection,
status_collection=train_status_collection)
valid_gen = self.data_generator.generate_images(
is_training=True,
images_collection=valid_images_collection,
status_collection=valid_status_collection)
profiler = Profiler()
profiler.start()
history = self.model.fit_generator(
train_gen,
steps_per_epoch=len(self.train_idx) // train_batch_size,
# epochs=epoch_batch,
epochs=1,
validation_data=valid_gen,
validation_steps=len(self.valid_idx) // valid_batch_size)
profiler.stop()
profiler.print(show_all=True)
def bench_publish(endpoint, appkey, channel, size, profile, ack=True):
publisher = satori.rtm.connection.Connection(endpoint + '?appkey=' +
appkey)
publisher.start()
message = binascii.hexlify(os.urandom(size // 2)).decode('ascii')
print('Message size is {}'.format(len(message)))
last_usage = [resource.getrusage(resource.RUSAGE_SELF)]
print(
'Duration, s\tRate, msgs/s\tMax RSS, MB\tUser time, s\tSystem time, s')
def report(duration, count):
usage = resource.getrusage(resource.RUSAGE_SELF)
maxrss = usage.ru_maxrss // 1024
if sys.platform == 'darwin':
maxrss = maxrss // 1024
print('{0:2.2f}\t\t{1}\t\t{2}\t\t{3:2.2f}\t\t{4:2.2f}'.format(
duration, int(count / duration), maxrss,
usage.ru_utime - last_usage[0].ru_utime,
usage.ru_stime - last_usage[0].ru_stime))
sys.stdout.flush()
last_usage[0] = usage
count = [0]
def publish_without_ack():
publisher.publish(channel, message)
count[0] += 1
def publish_with_ack():
def callback(ack):
count[0] += 1
publisher.publish(channel, message, callback)
publish = publish_with_ack if ack else publish_without_ack
before = time.time()
try:
if profile:
profiler = Profiler()
profiler.start()
while True:
now = time.time()
if now - before >= sampling_interval:
report(now - before, count[0])
if profile:
profiler.stop()
print(profiler.output_text(unicode=True, color=True))
profiler = Profiler()
profiler.start()
count[0] = 0
before = time.time()
publish()
except KeyboardInterrupt:
sys.exit(0)
def profile():
profiler = Profiler()
profiler.start()
exit_code = runtests.run_tests_all(parallel=False)
profiler.stop()
print(profiler.output_text(unicode=False, color=True))
return exit_code
def pytinstrument_decorator(*args, **kwargs):
# Create profiler instance before function is called.
profiler = Profiler()
profiler.start()
# Call function.
func_return = func(*args, **kwargs)
# Stop profiler and print results after function is called.
profiler.stop()
print(profiler.output_text(unicode=False, color=True))
return func_return
def test_profiler_doesnt_overflow_on_large_call_stacks():
profiler = Profiler()
profiler.start()
# give 170 frames of leeway for the test runner and for pyinstrument to do its work.
recursion_depth = sys.getrecursionlimit() - 170
recurse(recursion_depth)
profiler.stop()
print(profiler.output_text())
def deep_profiler_session():
profiler = Profiler()
profiler.start()
# give 120 frames for pyinstrument to do its work.
recursion_depth = sys.getrecursionlimit() - current_stack_depth() - 120
recurse(recursion_depth)
profiler.stop()
return profiler.last_session
def test_profiler_doesnt_overflow_on_large_call_stacks():
profiler = Profiler()
profiler.start()
# give ourselves 150 frames of leeway to do our work.
recursion_depth = sys.getrecursionlimit() - 150
recurse(recursion_depth)
profiler.stop()
print(profiler.output_text())
def main():
usage = "usage: %prog [-h] [-o output_file_path] scriptfile [arg] ..."
parser = OptionParser(usage=usage)
parser.allow_interspersed_args = False
parser.add_option('', '--html',
dest="output_html", action='store_true',
help="output HTML instead of text", default=False)
parser.add_option('-o', '--outfile',
dest="outfile", action='store',
help="save stats to <outfile>", default=None)
if not sys.argv[1:]:
parser.print_usage()
sys.exit(2)
(options, args) = parser.parse_args()
sys.argv[:] = args
if len(args) > 0:
progname = args[0]
sys.path.insert(0, os.path.dirname(progname))
with open(progname, 'rb') as fp:
code = compile(fp.read(), progname, 'exec')
globs = {
'__file__': progname,
'__name__': '__main__',
'__package__': None,
}
profiler = Profiler()
profiler.start()
try:
exec code in globs, None
except SystemExit, KeyboardInterrupt:
pass
profiler.stop()
if options.outfile:
f = codecs.open(options.outfile, 'w', 'utf-8')
unicode = True
color = False
else:
f = sys.stdout
unicode = stdout_supports_unicode()
color = stdout_supports_color()
if options.output_html:
f.write(profiler.output_html())
else:
f.write(profiler.output_text(unicode=unicode, color=color))
f.close()
def wrapper(*args, **kwargs):
profiler = None
if CONFIG.profiler_enable:
profiler = Profiler()
profiler.start()
try:
return f(*args, **kwargs)
finally:
if profiler is not None:
profiler.stop()
print(profiler.output_text(unicode=True, color=True))
def main():
profiler = Profiler()
sim = Simulation()
sim.setup()
profiler.start()
for i in range(100):
sim.update()
profiler.stop()
print(profiler.output_text(unicode=True, color=True))
def profiler_session(session: str) -> None:
"invoke code with sampling profiler"
has_profiler = profiler_enable()
if has_profiler:
profiler = Profiler(interval=profiler_interval())
profiler.start()
try:
yield
finally:
if has_profiler:
profiler.stop()
profiler_produce_report(profiler, session)
def instrument(filename="results"):
profiler = Profiler()
directory = Path("profiling/results/")
try:
profiler.start()
yield
finally:
profiler.stop()
directory.mkdir(parents=True, exist_ok=True)
path = os.path.join(directory, f"{filename}.html")
with open(path, "w") as fs:
fs.write(profiler.output_html(timeline=True))
def decorated_view(*args, **kwargs):
profiler = Profiler()
profiler.start()
result = func(*args, **kwargs)
profiler.stop()
print(profiler.output_text(unicode=True, color=True))
return result
def reverse(self, num_steps=1, width=5, height=5, msb_bits=10):
old_boards_positions = [
(1000,
LifeBoard(board=-1 * np.ones(self.board.shape, dtype=np.int32)))
]
# levels - ranges of each tilling
levels = self.build_levels(width, height)
max_old_boards, max_tiling, indices = 3, 200, None
# TODO - find good start (high density - 1's number)
debug_performance, type_performance = False, 1
if debug_performance:
if type_performance == 0:
pr = cProfile.Profile()
pr.enable()
else:
profiler = Profiler()
profiler.start()
for level in levels:
# print("level:", level)
tic_level = time.time()
interior_str = self.interior_str([(1, 1), (1, 1)], level)
old_boards_tilings = self.read_tilings(interior_str, width, height,
max_tiling)
# TODO - soften selection (another parameter)
# TODO - selection based on probability
# TODO - selection based on real fit for delta
old_boards_positions = old_boards_positions[:max_old_boards]
#print("first selection: ", len(old_boards_tilings), len(old_boards_positions))
#TODO - constrint new to current
old_boards_results = [
ob_pos[1].position(ob_tiling, level, constraint=self)
for ob_pos in old_boards_positions
for ob_tiling in old_boards_tilings
]
#print("stage 1(fit): ", time.time() - tic_level)
#indices = np.argsort([result[0] for result in old_boards_results])[:max_old_boards]
old_boards_positions = sorted(old_boards_results,
key=lambda pos: pos[0])
#print("stage 2(sort): ", time.time() - tic_level)
# print("second selection: ", len(old_boards_positions),old_boards_positions[0][0])
if debug_performance:
if type_performance == 0:
pr.disable()
pr.print_stats(sort="time")
else:
profiler.stop()
print(profiler.output_text(unicode=True, color=True))
# print(len(old_boards_positions),old_boards_positions[0][0])
return old_boards_positions[0][1]
def _wrapper(*args, **kwargs):
_profiler = Profiler(
sampling_interval_seconds
) if sampling_interval_seconds is not None else Profiler()
_profiler.start()
result = _func(*args, **kwargs)
_profiler.stop()
if print_to_console:
print(_profiler.output_text(unicode=True, color=True))
if log_to_file:
name = file_name if file_name is not None else _func.__name__ + ".profile"
with open(f"{name}.html", "w") as f:
f.write(_profiler.output_html())
return result
def instrument(cls: tp.Type[PerfTest], function: str = 'sf') -> None:
'''
Profile the `sf` function from the supplied class.
'''
f = getattr(cls, function)
profiler = Profiler()
profiler.start()
for _ in range(cls.NUMBER):
f()
profiler.stop()
print(profiler.output_text(unicode=True, color=True))
def test_profiler_retains_multiple_calls():
profiler = Profiler()
profiler.start()
long_function_a()
long_function_b()
long_function_a()
long_function_b()
profiler.stop()
print(profiler.output_text())
frame = profiler.last_session.root_frame()
assert frame.function == 'test_profiler_retains_multiple_calls'
assert len(frame.children) == 4
def profile_ctx(engine='pyinstrument'):
"""
A context manager which profiles the body of the with statement
with the supplied profiling engine and returns the profiling object
in a list.
Arguments
---------
engine: str
The profiling engine, e.g. 'pyinstrument' or 'snakeviz'
Returns
-------
sessions: list
A list containing the profiling session.
"""
if engine == 'pyinstrument':
from pyinstrument import Profiler
prof = Profiler()
prof.start()
elif engine == 'snakeviz':
prof = Profile()
prof.enable()
elif engine is None:
pass
sessions = []
yield sessions
if engine == 'pyinstrument':
sessions.append(prof.stop())
elif engine == 'snakeviz':
prof.disable()
sessions.append(prof)
def test_profiler_retains_multiple_calls():
profiler = Profiler()
profiler.start()
long_function_a()
long_function_b()
long_function_a()
long_function_b()
profiler.stop()
print(profiler.output_text())
frame = profiler.first_interesting_frame()
assert frame.function == 'test_profiler_retains_multiple_calls'
assert len(frame.children) == 4
def main():
usage = "usage: %prog [-h] [[-o output_file_path] scriptfile [arg] ...] | [ -i infile ]"
parser = OptionParser(usage=usage)
parser.allow_interspersed_args = False
parser.add_option('', '--html',
dest="output_html", action='store_true',
help="output HTML instead of text", default=False)
parser.add_option('', '--json',
dest="output_json", action='store_true',
help="output raw JSON dump instead of text or HTML", default=False)
parser.add_option('-o', '--outfile',
dest="outfile", action='store',
help="save stats to <outfile>", default=None)
parser.add_option('-i', '--infile',
dest="infile", action='store',
help="load stats from JSON file <infile>", default=None)
if not sys.argv[1:]:
parser.print_usage()
sys.exit(2)
(options, args) = parser.parse_args()
sys.argv[:] = args
if len(args) > 0:
progname = args[0]
sys.path.insert(0, os.path.dirname(progname))
with open(progname, 'rb') as fp:
code = compile(fp.read(), progname, 'exec')
globs = {
'__file__': progname,
'__name__': '__main__',
'__package__': None,
}
profiler = Profiler()
profiler.start()
try:
exec code in globs, None
except SystemExit, KeyboardInterrupt:
pass
profiler.stop()
write_output( options, profiler )
def run_profiling(args):
lprofiler = LineProfiler()
monitor_fuctions = [api.problem.submit_key, api.problem.get_unlocked_pids, api.problem.get_solved_pids,
api.problem.get_all_problems, api.problem.get_solved_problems, api.stats.get_score,
api.cache.memoize, api.autogen.grade_problem_instance, api.autogen.get_problem_instance,
api.autogen.get_number_of_instances]
for func in monitor_fuctions:
lprofiler.add_function(func)
lprofiler.enable()
if args.stack:
profiler = Profiler(use_signal=False)
profiler.start()
for func, a, kw in operations:
func(*a, **kw)
if args.stack:
profiler.stop()
lprofiler.disable()
if args.print:
print(profiler.output_text(unicode=True, color=True))
lprofiler.print_stats()
output = open(args.output, "w")
if args.stack:
output.write(profiler.output_text(unicode=True))
if args.output_html is not None:
output_html = open(args.output_html, "w")
output_html.write(profiler.output_html())
output_html.close()
print("Wrote test info to " + args.output_html)
lprofiler.print_stats(output)
output.close()
print("Wrote test info to " + args.output)
def test_collapses_multiple_calls_by_default():
profiler = Profiler()
profiler.start()
long_function_a()
long_function_b()
long_function_a()
long_function_b()
profiler.stop()
text_output = profiler.output_text()
# output should be something like:
# 1.513 test_collapses_multiple_calls_by_default test/test_profiler.py:25
# |- 0.507 long_function_a test/test_profiler.py:17
# |- 0.503 long_function_b test/test_profiler.py:20
assert text_output.count('test_collapses_multiple_calls_by_default') == 1
assert text_output.count('long_function_a') == 1
assert text_output.count('long_function_b') == 1
def test_two_functions():
profiler = Profiler()
profiler.start()
long_function_a()
long_function_b()
profiler.stop()
print(profiler.output_text())
frame = profiler.last_session.root_frame()
assert frame.function == 'test_two_functions'
assert len(frame.children) == 2
frame_b, frame_a = sorted(frame.children, key=lambda f: f.time(), reverse=True)
assert frame_a.function == 'long_function_a'
assert frame_b.function == 'long_function_b'
assert 0.2 < frame_a.time() < 0.3
assert 0.45 < frame_b.time() < 0.55
def main():
"""Command-line utility for using (and testing) s3 utility methods."""
logging.basicConfig(level=logging.DEBUG)
arg_parser = argparse.ArgumentParser(description='Perform obfuscation of forum .mongo dump files.')
arg_parser.add_argument(
'input',
help='Read mongo files from this location.',
)
arg_parser.add_argument(
'-o', '--output',
help='Write obfuscated mongo files to this location in the local file system.',
default=None
)
arg_parser.add_argument(
'-u', '--userinfo',
help='For events, read a custom user-info file from the local fs that contains username, email, user-id, fullname.',
default=None
)
arg_parser.add_argument(
'--log-context',
help='characters on each side of match',
type=int,
default=50,
)
#####################
# Flags to indicate what to obfuscate.
#####################
arg_parser.add_argument(
'--forum',
help='Read in and obfuscate forum posts.',
action='store_true',
)
arg_parser.add_argument(
'--wiki',
help='Read in and obfuscate wiki documents.',
action='store_true',
)
arg_parser.add_argument(
'--courseware',
help='Read in and obfuscate courseware_studentmodule records.',
action='store_true',
)
arg_parser.add_argument(
'--event',
help='Read in and obfuscate events.',
action='store_true',
)
#####################
# Various flags to indicate what to look for.
#####################
arg_parser.add_argument(
'--phone',
help='Extract phone numbers',
action='store_true',
)
arg_parser.add_argument(
'--possible-phone',
help='Extract phone numbers',
action='store_true',
)
arg_parser.add_argument(
'--email',
help='Extract email addresses',
action='store_true',
)
arg_parser.add_argument(
'--phone-context',
help='Extract phone number context',
action='store_true',
)
arg_parser.add_argument(
'--email-context',
help='Extract email address context',
action='store_true',
)
arg_parser.add_argument(
'--name-context',
help='Extract name context',
action='store_true',
)
arg_parser.add_argument(
'--facebook',
help='Extract facebook urls',
action='store_true',
)
arg_parser.add_argument(
'--username',
help='Extract username',
action='store_true',
)
arg_parser.add_argument(
'--fullname',
help='Extract fullname.',
action='store_true',
)
arg_parser.add_argument(
'--userid',
help='Extract user-id.',
action='store_true',
)
arg_parser.add_argument(
'--skip-post',
help='Skip performing filtering on event.POST entries.',
action='store_true',
)
arg_parser.add_argument(
'--pyinstrument',
help='Profile the run and write the output to stderr',
action='store_true'
)
args = arg_parser.parse_args()
kwargs = vars(args)
profiler = None
if args.pyinstrument:
profiler = Profiler() # or Profiler(use_signal=False), see below
profiler.start()
try:
obfuscator = BulkObfuscator(**kwargs)
obfuscator.obfuscate_directory(args.input, args.output)
finally:
if profiler:
profiler.stop()
print >>sys.stderr, profiler.output_text(unicode=True, color=True)
def main():
usage = ("usage: pyinstrument [options] scriptfile [arg] ...")
version_string = 'pyinstrument {v}, on Python {pyv[0]}.{pyv[1]}.{pyv[2]}'.format(
v=pyinstrument.__version__,
pyv=sys.version_info,
)
parser = optparse.OptionParser(usage=usage, version=version_string)
parser.allow_interspersed_args = False
def dash_m_callback(option, opt, value, parser):
parser.values.module_name = value
# everything after the -m argument should be passed to that module
parser.values.module_args = parser.rargs + parser.largs
parser.rargs[:] = []
parser.largs[:] = []
parser.add_option('', '--load-prev',
dest='load_prev', action='store', metavar='ID',
help="Instead of running a script, load a previous report")
parser.add_option('-m', '',
dest='module_name', action='callback', callback=dash_m_callback,
type="str",
help="run library module as a script, like 'python -m module'")
parser.add_option('-o', '--outfile',
dest="outfile", action='store',
help="save to <outfile>", default=None)
parser.add_option('-r', '--renderer',
dest='renderer', action='store', type='string',
help=("how the report should be rendered. One of: 'text', 'html', 'json', or python "
"import path to a renderer class"),
default='text')
parser.add_option('', '--html',
dest="output_html", action='store_true',
help=optparse.SUPPRESS_HELP, default=False) # deprecated shortcut for --renderer=html
parser.add_option('-t', '--timeline',
dest='timeline', action='store_true',
help="render as a timeline - preserve ordering and don't condense repeated calls")
parser.add_option('', '--hide',
dest='hide_fnmatch', action='store', metavar='EXPR',
help=("glob-style pattern matching the file paths whose frames to hide. Defaults to "
"'*{sep}lib{sep}*'.").format(sep=os.sep),
default='*{sep}lib{sep}*'.format(sep=os.sep))
parser.add_option('', '--hide-regex',
dest='hide_regex', action='store', metavar='REGEX',
help=("regex matching the file paths whose frames to hide. Useful if --hide doesn't give "
"enough control."))
parser.add_option('', '--show',
dest='show_fnmatch', action='store', metavar='EXPR',
help=("glob-style pattern matching the file paths whose frames to "
"show, regardless of --hide or --hide-regex. For example, use "
"--show '*/<library>/*' to show frames within a library that "
"would otherwise be hidden."))
parser.add_option('', '--show-regex',
dest='show_regex', action='store', metavar='REGEX',
help=("regex matching the file paths whose frames to always show. "
"Useful if --show doesn't give enough control."))
parser.add_option('', '--show-all',
dest='show_all', action='store_true',
help="show everything", default=False)
parser.add_option('', '--unicode',
dest='unicode', action='store_true',
help='(text renderer only) force unicode text output')
parser.add_option('', '--no-unicode',
dest='unicode', action='store_false',
help='(text renderer only) force ascii text output')
parser.add_option('', '--color',
dest='color', action='store_true',
help='(text renderer only) force ansi color text output')
parser.add_option('', '--no-color',
dest='color', action='store_false',
help='(text renderer only) force no color text output')
if not sys.argv[1:]:
parser.print_help()
sys.exit(2)
options, args = parser.parse_args()
if args == [] and options.module_name is None and options.load_prev is None:
parser.print_help()
sys.exit(2)
if not options.hide_regex:
options.hide_regex = fnmatch.translate(options.hide_fnmatch)
if not options.show_regex and options.show_fnmatch:
options.show_regex = fnmatch.translate(options.show_fnmatch)
if options.show_all:
options.show_regex = r'.*'
if options.load_prev:
session = load_report(options.load_prev)
else:
if options.module_name is not None:
sys.argv[:] = [options.module_name] + options.module_args
code = "run_module(modname, run_name='__main__')"
globs = {
'run_module': runpy.run_module,
'modname': options.module_name
}
else:
sys.argv[:] = args
progname = args[0]
sys.path.insert(0, os.path.dirname(progname))
with open(progname, 'rb') as fp:
code = compile(fp.read(), progname, 'exec')
globs = {
'__file__': progname,
'__name__': '__main__',
'__package__': None,
}
profiler = Profiler()
profiler.start()
try:
exec_(code, globs, None)
except (SystemExit, KeyboardInterrupt):
pass
profiler.stop()
session = profiler.last_session
if options.output_html:
options.renderer = 'html'
output_to_temp_file = (options.renderer == 'html'
and not options.outfile
and file_is_a_tty(sys.stdout))
if options.outfile:
f = codecs.open(options.outfile, 'w', 'utf-8')
should_close_f_after_writing = True
elif not output_to_temp_file:
if PY2:
f = codecs.getwriter('utf-8')(sys.stdout)
else:
f = sys.stdout
should_close_f_after_writing = False
renderer_kwargs = {'processor_options': {
'hide_regex': options.hide_regex,
'show_regex': options.show_regex,
}}
if options.timeline is not None:
renderer_kwargs['timeline'] = options.timeline
if options.renderer == 'text':
unicode_override = options.unicode != None
color_override = options.color != None
unicode = options.unicode if unicode_override else file_supports_unicode(f)
color = options.color if color_override else file_supports_color(f)
renderer_kwargs.update({'unicode': unicode, 'color': color})
renderer_class = get_renderer_class(options.renderer)
renderer = renderer_class(**renderer_kwargs)
# remove this frame from the trace
renderer.processors.append(remove_first_pyinstrument_frame_processor)
if output_to_temp_file:
output_filename = renderer.open_in_browser(session)
print('stdout is a terminal, so saved profile output to %s' % output_filename)
else:
f.write(renderer.render(session))
if should_close_f_after_writing:
f.close()
if options.renderer == 'text':
_, report_identifier = save_report(session)
print('To view this report with different options, run:')
print(' pyinstrument --load-prev %s [options]' % report_identifier)
print('')
def main():
usage = ("usage: python -m pyinstrument [options] scriptfile [arg] ...")
parser = OptionParser(usage=usage)
parser.allow_interspersed_args = False
parser.add_option('', '--html',
dest="output_html", action='store_true',
help="output HTML instead of text", default=False)
parser.add_option('-o', '--outfile',
dest="outfile", action='store',
help="save report to <outfile>", default=None)
parser.add_option('', '--unicode',
dest='unicode', action='store_true',
help='force unicode text output')
parser.add_option('', '--no-unicode',
dest='unicode', action='store_false',
help='force ascii text output')
parser.add_option('', '--color',
dest='color', action='store_true',
help='force ansi color text output')
parser.add_option('', '--no-color',
dest='color', action='store_false',
help='force no color text output')
if not sys.argv[1:]:
parser.print_help()
sys.exit(2)
(options, args) = parser.parse_args()
sys.argv[:] = args
if len(args) > 0:
progname = args[0]
sys.path.insert(0, os.path.dirname(progname))
with open(progname, 'rb') as fp:
code = compile(fp.read(), progname, 'exec')
globs = {
'__file__': progname,
'__name__': '__main__',
'__package__': None,
}
try:
profiler = Profiler()
except SignalUnavailableError:
profiler = Profiler(use_signal=False)
profiler.start()
try:
exec_(code, globs, None)
except (SystemExit, KeyboardInterrupt):
pass
profiler.stop()
if options.outfile:
f = codecs.open(options.outfile, 'w', 'utf-8')
else:
f = sys.stdout
unicode_override = options.unicode != None
color_override = options.color != None
unicode = options.unicode if unicode_override else file_supports_unicode(f)
color = options.color if color_override else file_supports_color(f)
if options.output_html:
f.write(profiler.output_html())
else:
f.write(profiler.output_text(unicode=unicode, color=color))
f.close()
else:
parser.print_usage()
return parser
class OpenAssessmentTest(WebAppTest):
"""
UI-level acceptance tests for Open Assessment.
"""
TEST_COURSE_ID = "course-v1:edx+ORA203+course"
PROBLEM_LOCATIONS = {
'staff_only':
u'courses/{test_course_id}/courseware/'
u'61944efb38a349edb140c762c7419b50/415c3ee1b7d04b58a1887a6fe82b31d6/'.format(test_course_id=TEST_COURSE_ID),
'self_only':
u'courses/{test_course_id}/courseware/'
u'a4dfec19cf9b4a6fb5b18be6ccd9cecc/338a4affb58a45459629e0566291381e/'.format(test_course_id=TEST_COURSE_ID),
'peer_only':
u'courses/{test_course_id}/courseware/'
u'a4dfec19cf9b4a6fb5b18be6ccd9cecc/417e47b2663a4f79b62dba20b21628c8/'.format(test_course_id=TEST_COURSE_ID),
'student_training':
u'courses/{test_course_id}/courseware/'
u'676026889c884ac1827688750871c825/5663e9b038434636977a4226d668fe02/'.format(test_course_id=TEST_COURSE_ID),
'file_upload':
u'courses/{test_course_id}/courseware/'
u'57a3f9d51d424f6cb922f0d69cba868d/bb563abc989340d8806920902f267ca3/'.format(test_course_id=TEST_COURSE_ID),
'full_workflow_staff_override':
u'courses/{test_course_id}/courseware/'
u'676026889c884ac1827688750871c825/181ea9ff144c4766be44eb8cb360e34f/'.format(test_course_id=TEST_COURSE_ID),
'full_workflow_staff_required':
u'courses/{test_course_id}/courseware/'
u'8d9584d242b44343bc270ea5ef04ab03/0b0dcc728abe45138c650732af178afb/'.format(test_course_id=TEST_COURSE_ID),
}
SUBMISSION = u"This is a test submission."
LATEX_SUBMISSION = u"[mathjaxinline]( \int_{0}^{1}xdx )[/mathjaxinline]"
OPTIONS_SELECTED = [1, 2]
STAFF_OVERRIDE_OPTIONS_SELECTED = [0, 1]
STAFF_OVERRIDE_SCORE = 1
STAFF_GRADE_EXISTS = "COMPLETE"
STAFF_OVERRIDE_LEARNER_STEPS_NOT_COMPLETE = "YOU MUST COMPLETE THE STEPS ABOVE TO VIEW YOUR GRADE"
STAFF_AREA_SCORE = "Final grade: {} out of 8"
STAFF_OVERRIDE_STAFF_AREA_NOT_COMPLETE = "The problem has not been completed."
EXPECTED_SCORE = 6
STUDENT_TRAINING_OPTIONS = [
[1, 2],
[0, 2]
]
TEST_PASSWORD = "******"
def setUp(self, problem_type, staff=False):
"""
Configure page objects to test Open Assessment.
Args:
problem_type (str): The type of problem being tested,
used to choose which part of the course to load.
staff (bool): If True, runs the test with a staff user (defaults to False).
"""
super(OpenAssessmentTest, self).setUp()
if PROFILING_ENABLED:
self.profiler = Profiler(use_signal=False)
self.profiler.start()
self.problem_loc = self.PROBLEM_LOCATIONS[problem_type]
self.auto_auth_page = AutoAuthPage(self.browser, course_id=self.TEST_COURSE_ID, staff=staff)
self.submission_page = SubmissionPage(self.browser, self.problem_loc)
self.self_asmnt_page = AssessmentPage('self-assessment', self.browser, self.problem_loc)
self.peer_asmnt_page = AssessmentPage('peer-assessment', self.browser, self.problem_loc)
self.student_training_page = AssessmentPage('student-training', self.browser, self.problem_loc)
self.staff_asmnt_page = AssessmentPage('staff-assessment', self.browser, self.problem_loc)
self.grade_page = GradePage(self.browser, self.problem_loc)
def log_to_file(self):
with open('{}-profile.log'.format(self.id()), 'w') as f:
f.write(self.profiler.output_text())
def tearDown(self):
if PROFILING_ENABLED:
self.profiler.stop()
self.log_to_file()
def login_user(self, learner, email):
"""
Logs in an already existing user.
Args:
learner (str): the username of the user.
email (str): email address of the user.
"""
auto_auth_page = AutoAuthPage(
self.browser, email=email, password=self.TEST_PASSWORD, username=learner,
course_id=self.TEST_COURSE_ID, staff=True
)
auto_auth_page.visit()
def do_self_assessment(self):
"""
Creates a user, submits a self assessment, verifies the grade, and returns the username of the
learner for which the self assessment was submitted.
"""
self.auto_auth_page.visit()
username, _ = self.auto_auth_page.get_username_and_email()
self.submission_page.visit().submit_response(self.SUBMISSION)
self.assertTrue(self.submission_page.has_submitted)
# Submit a self-assessment
self.submit_self_assessment(self.OPTIONS_SELECTED)
# Verify the grade
self.assertEqual(self.EXPECTED_SCORE, self.grade_page.wait_for_page().score)
return username
def submit_self_assessment(self, options=OPTIONS_SELECTED):
"""
Submit a self assessment for the currently logged in student. Do not verify grade.
Args:
options: the options to select for the self assessment
(will use OPTIONS_SELECTED if not specified)
"""
self.self_asmnt_page.wait_for_page().wait_for_response()
self.assertIn(self.SUBMISSION, self.self_asmnt_page.response_text)
self.self_asmnt_page.assess("self", options).wait_for_complete()
self.assertTrue(self.self_asmnt_page.is_complete)
def _verify_staff_grade_section(self, expected_status, expected_message_title):
"""
Verifies the expected status and message text in the Staff Grade section
(as shown to the learner).
"""
self.staff_asmnt_page.wait_for_page()
self.assertEqual("Staff Grade", self.staff_asmnt_page.label)
self.staff_asmnt_page.verify_status_value(expected_status)
self.assertEqual(expected_message_title, self.staff_asmnt_page.message_title)
def do_training(self):
"""
Complete two training examples, satisfying the requirements.
"""
for example_num, options_selected in enumerate(self.STUDENT_TRAINING_OPTIONS):
if example_num > 0:
try:
self.student_training_page.wait_for_num_completed(example_num)
except BrokenPromise:
msg = "Did not complete at least {num} student training example(s).".format(num=example_num)
self.fail(msg)
self.student_training_page.wait_for_page().wait_for_response().assess("training", options_selected)
# Check that we've completed student training
try:
self.student_training_page.wait_for_complete()
except BrokenPromise:
self.fail("Student training was not marked complete.")
def do_peer_assessment(self, count=1, options=OPTIONS_SELECTED):
"""
Does the specified number of peer assessments.
Args:
count: the number of assessments that must be completed (defaults to 1)
options: the options to use (defaults to OPTIONS_SELECTED)
"""
self.peer_asmnt_page.visit()
for count_assessed in range(1, count + 1):
self.peer_asmnt_page.wait_for_page().wait_for_response().assess("peer", options)
self.peer_asmnt_page.wait_for_num_completed(count_assessed)
def do_staff_override(self, username, final_score=STAFF_AREA_SCORE.format(STAFF_OVERRIDE_SCORE)):
"""
Complete a staff assessment (grade override).
Args:
username: the learner to grade
final_score: the expected final score as shown in the staff area
(defaults to the staff override score value)
"""
self.staff_area_page.visit()
self.staff_area_page.show_learner(username)
self.staff_area_page.expand_learner_report_sections()
self.staff_area_page.staff_assess(self.STAFF_OVERRIDE_OPTIONS_SELECTED, "override")
self.staff_area_page.verify_learner_final_score(final_score)
def do_staff_assessment(self, number_to_assess=0, options_selected=OPTIONS_SELECTED):
"""
Use staff tools to assess available responses.
Args:
number_to_assess: the number of submissions to assess. If not provided (or 0),
will grade all available submissions.
"""
self.staff_area_page.visit()
self.staff_area_page.click_staff_toolbar_button("staff-grading")
# Get the counts before checking out a submission for assessment.
start_numbers = self.staff_area_page.available_checked_out_numbers
# Check out a submission.
self.staff_area_page.expand_staff_grading_section()
# Checked out number should increase, ungraded decrease.
ungraded = start_numbers[0]-1
checked_out = start_numbers[1]+1
self.staff_area_page.verify_available_checked_out_numbers((ungraded, checked_out))
assessed = 0
while number_to_assess == 0 or assessed < number_to_assess:
continue_after = False if number_to_assess-1 == assessed else ungraded > 0
self.staff_area_page.staff_assess(options_selected, "full-grade", continue_after)
assessed += 1
if not continue_after:
self.staff_area_page.verify_available_checked_out_numbers((ungraded, checked_out-1))
break
else:
ungraded -=1
self.staff_area_page.verify_available_checked_out_numbers((ungraded, checked_out))
def refresh_page(self):
"""
Helper method that waits for "unsaved changes" warnings to clear before refreshing the page.
"""
EmptyPromise(
lambda: self.browser.execute_script("return window.onbeforeunload === null"),
"Unsubmitted changes exist on page."
).fulfill()
self.browser.refresh()
def observe(config_file, profile=False, raise_queue_empty=True):
if profile:
try:
from pyinstrument import Profiler
except ImportError:
print "Error importing pyinstrument"
profile = False
ztf_config = ZTFConfiguration("../sims/{}".format(config_file))
# load config parameters into local variables
run_name = ztf_config.config["run_name"]
start_time = ztf_config.config["start_time"]
weather_year = ztf_config.config["weather_year"]
if weather_year == "None":
weather_year = None
survey_duration = ztf_config.config["survey_duration_days"] * u.day
block_programs = ztf_config.config["block_programs"]
observing_programs = ztf_config.build_observing_programs()
if profile:
if survey_duration > 1.0 * u.day:
print ("Don't profile long runs: 25% overhead")
profile = False
else:
profiler = Profiler()
profiler.start()
survey_start_time = Time(start_time, scale="utc", location=P48_loc)
tel = ZTFStateMachine(
current_time=survey_start_time,
historical_observability_year=weather_year,
logfile="../sims/{}_log.txt".format(run_name),
)
# set up QueueManager
Q = GreedyQueueManager(block_programs=block_programs)
for op in observing_programs:
Q.add_observing_program(op)
# initialize nightly field requests (Tom Barlow function)
Q.assign_nightly_requests(tel.current_state_dict())
# initialize sqlite history
log = ObsLogger(run_name, tel.current_time)
current_night_mjd = np.floor(tel.current_time.mjd)
while tel.current_time < (survey_start_time + survey_duration):
# check if it is a new night and reload queue with new requests
if np.floor(tel.current_time.mjd) > current_night_mjd:
log.prev_obs = None
Q.assign_nightly_requests(tel.current_state_dict())
current_night_mjd = np.floor(tel.current_time.mjd)
if tel.check_if_ready():
current_state = tel.current_state_dict()
# get coords
try:
next_obs = Q.next_obs(current_state)
# TODO: debugging check...
assert next_obs["request_id"] in Q.queue.index
except QueueEmptyError:
if not raise_queue_empty:
tel.logger.info("Queue empty! Waiting...")
log.prev_obs = None
tel.wait()
continue
else:
raise QueueEmptyError("Queue is empty")
# try to change filters, if needed
if next_obs["target_filter_id"] != current_state["current_filter_id"]:
if not tel.start_filter_change(next_obs["target_filter_id"]):
tel.logger.info("Filter change failure! Waiting...")
log.prev_obs = None
tel.wait()
continue
# try to slew to the next target
if not tel.start_slew(coord.SkyCoord(next_obs["target_ra"] * u.deg, next_obs["target_dec"] * u.deg)):
tel.set_cant_observe()
# TODO: log the failure
# "missed history": http://ops2.lsst.org/docs/current/architecture.html#output-tables
tel.logger.info(
"Failure slewing to {}, {}! Waiting...".format(
next_obs["target_ra"] * u.deg, next_obs["target_dec"] * u.deg
)
)
log.prev_obs = None
tel.wait()
continue
# try to expose
if not tel.start_exposing():
tel.set_cant_observe()
tel.logger.info("Exposure failure! Waiting...")
log.prev_obs = None
tel.wait()
continue
else:
# exposure completed successfully. now
# a) store exposure information in pointing history sqlite db
current_state = tel.current_state_dict()
log.log_pointing(current_state, next_obs)
# b) update Fields
Q.fields.mark_field_observed(next_obs, current_state)
# c) remove completed request_id from the pool and the queue
# TODO: debugging check
assert next_obs["request_id"] in Q.queue.index
Q.remove_requests(next_obs["request_id"])
else:
log.prev_obs = None
tel.set_cant_observe()
tel.wait()
if profile:
profiler.stop()
print profiler.output_text(unicode=True, color=True)
with open("../sims/{}_profile.txt".format(run_name), "w") as f:
f.write(profiler.output_text())
def main():
usage = ("usage: pyinstrument [options] scriptfile [arg] ...")
parser = OptionParser(usage=usage)
parser.allow_interspersed_args = False
parser.add_option('', '--setprofile',
dest='setprofile', action='store_true',
help='run in setprofile mode, instead of signal mode', default=False)
parser.add_option('', '--html',
dest="output_html", action='store_true',
help="output HTML instead of text", default=False)
parser.add_option('-o', '--outfile',
dest="outfile", action='store',
help="save report to <outfile>", default=None)
parser.add_option('', '--unicode',
dest='unicode', action='store_true',
help='force unicode text output')
parser.add_option('', '--no-unicode',
dest='unicode', action='store_false',
help='force ascii text output')
parser.add_option('', '--color',
dest='color', action='store_true',
help='force ansi color text output')
parser.add_option('', '--no-color',
dest='color', action='store_false',
help='force no color text output')
if not sys.argv[1:]:
parser.print_help()
sys.exit(2)
(options, args) = parser.parse_args()
sys.argv[:] = args
if len(args) > 0:
progname = args[0]
sys.path.insert(0, os.path.dirname(progname))
with open(progname, 'rb') as fp:
code = compile(fp.read(), progname, 'exec')
globs = {
'__file__': progname,
'__name__': '__main__',
'__package__': None,
}
try:
profiler = Profiler(use_signal=not options.setprofile)
except SignalUnavailableError:
profiler = Profiler(use_signal=False)
profiler.start()
try:
exec_(code, globs, None)
except IOError as e:
import errno
if e.errno == errno.EINTR:
print(
'Failed to run program due to interrupted system system call.\n'
'This happens because pyinstrument is sending OS signals to the running\n'
'process to interrupt it. If your program has long-running syscalls this\n'
'can cause a problem.\n'
'\n'
'You can avoid this error by running in \'setprofile\' mode. Do this by\n'
'passing \'--setprofile\' when calling pyinstrument at the command-line.\n'
'\n'
'For more information, see\n'
'https://github.com/joerick/pyinstrument/issues/16\n'
)
raise
except (SystemExit, KeyboardInterrupt):
pass
profiler.stop()
if options.outfile:
f = codecs.open(options.outfile, 'w', 'utf-8')
else:
f = sys.stdout
unicode_override = options.unicode != None
color_override = options.color != None
unicode = options.unicode if unicode_override else file_supports_unicode(f)
color = options.color if color_override else file_supports_color(f)
if options.output_html:
f.write(profiler.output_html())
else:
f.write(profiler.output_text(unicode=unicode, color=color))
f.close()
else:
parser.print_usage()
return parser
def observe(run_name=run_name, start_time='2018-01-01 04:00:00',
weather_year=None, survey_duration=1 * u.hour):
if profile:
profiler = Profiler()
profiler.start()
survey_start_time = Time(start_time, scale='utc', location=P48_loc)
tel = ZTFStateMachine(
current_time=survey_start_time,
historical_observability_year=weather_year)
# set up QueueManager
Q = GreedyQueueManager()
# set up Observing Programs
#CollabOP = CollaborationObservingProgram()
# Q.add_observing_program(CollabOP)
MSIPOP = MSIPObservingProgram(
Q.fields.select_field_ids(dec_range=[-30, 90], grid_id=0))
MSIPOP.observing_time_fraction = 1.0
Q.add_observing_program(MSIPOP)
#CaltechOP = CaltechObservingProgram()
# Q.add_observing_program(CaltechOP)
# initialize nightly field requests (Tom Barlow function)
Q.assign_nightly_requests(tel.current_state_dict())
# temporary loading to test things
# Q.rp.add_requests(1,
# Q.fields.fields[
# Q.fields.select_fields(dec_range=[-30,90])].index, 2,
# 'no_cadence',{})
# initialize sqlite history
log = ObsLogger(run_name, tel.current_time)
log.create_pointing_log(clobber=True)
while tel.current_time < (survey_start_time + survey_duration):
# TODO: reload queue with new requests on update interval (nightly
if tel.check_if_ready():
current_state = tel.current_state_dict()
# get coords
next_obs = Q.next_obs(current_state)
# TODO: filter change, if needed
if not tel.start_slew(coord.SkyCoord(next_obs['target_ra'] * u.deg,
next_obs['target_dec'] * u.deg)):
tel.set_cant_observe()
# TODO: log the failure
# "missed history": http://ops2.lsst.org/docs/current/architecture.html#output-tables
log.prev_obs = None
tel.wait()
continue
if not tel.start_exposing():
tel.set_cant_observe()
# TODO: log the failure
log.prev_obs = None
tel.wait()
continue
else:
# exposure completed successfully. now
# a) store exposure information in pointing history sqlite db
current_state = tel.current_state_dict()
log.log_pointing(current_state, next_obs)
# b) update Fields
Q.fields.mark_field_observed(next_obs, current_state)
# c) remove completed request_id
Q.rp.remove_requests(next_obs['request_id'])
else:
tel.set_cant_observe()
tel.wait()
if profile:
profiler.stop()
print profiler.output_text(unicode=True, color=True)
from pyinstrument import Profiler import numpy as np import pyamg profiler = Profiler() profiler.start() n = int(1e3) A = pyamg.gallery.poisson((n, n), format='csr') b = np.random.rand(A.shape[0]) ml = pyamg.smoothed_aggregation_solver(A, max_coarse=10) res = [] x = ml.solve(b, accel='cg', residuals=res) print(len(res)) profiler.stop() print(profiler.output_text(unicode=True, color=True))
from pyinstrument import Profiler
from platform import platform
p = Profiler()
p.start()
def func():
fd = open('/dev/urandom', 'rb')
data = fd.read(1024*1024)
func()
# this failed on ubuntu 12.04
platform()
p.stop()
print(p.output_text())
with open('ioerror_out.html', 'w') as f:
f.write(p.output_html())
