def profile_threaded(filename):
import yappi # noqa # yappi is not a dependency
import gil_load # noqa # same
yappi.set_clock_type("cpu")
try:
gil_load.init()
gil_load.start(av_sample_interval=0.1,
output_interval=3,
output=sys.stdout)
monitoring_gil = True
except RuntimeError:
monitoring_gil = False
pass
yappi.start()
yield
yappi.stop()
if monitoring_gil:
gil_load.stop()
print("Gil was held %0.1f %% of the time" % (100 * gil_load.get()[0]))
p = yappi.get_func_stats()
p = yappi.convert2pstats(p)
p.dump_stats(filename)
yappi.clear_stats()
def profile_threaded(filename):
import yappi # noqa # yappi is not a dependency
yappi.set_clock_type("cpu")
try:
import gil_load # noqa # same
gil_load.init()
gil_load.start(av_sample_interval=0.1,
output_interval=10,
output=sys.stdout)
monitoring_gil = True
except (RuntimeError, ImportError):
monitoring_gil = False
pass
yappi.start()
yield
yappi.stop()
if monitoring_gil:
gil_load.stop()
stats = gil_load.get()
print("GIL load information: ", gil_load.format(stats))
p = yappi.get_func_stats()
p = yappi.convert2pstats(p)
p.dump_stats(filename)
yappi.clear_stats()
def gil_10_pc():
while True:
for i in range(300):
a = 1 + 1
time.sleep(1e-9)
gil_threads = []
for gil_idx in range(0):
gil_threads.append(threading.Thread(target=gil_10_pc, daemon=True))
gil_threads[-1].daemon = True
gil_threads[-1].start()
pipeline.set_state(Gst.State.PLAYING)
if gil_load_enabled:
gil_load.start()
try:
while True:
msg = pipeline.get_bus().timed_pop_filtered(
Gst.SECOND,
Gst.MessageType.EOS | Gst.MessageType.ERROR
)
if msg:
text = msg.get_structure().to_string() if msg.get_structure() else ''
msg_type = Gst.message_type_get_name(msg.type)
print(f'{msg.src.name}: [{msg_type}] {text}')
break
finally:
if gil_load_enabled:
gil_load.stop()
strax.xenon.plugins.DAQReader.save_meta_only = True
st = strax.Context(storage=[
strax.DataDirectory(out_dir + '/raw', take_only='raw_records'),
strax.DataDirectory(out_dir + '/reduced_raw', take_only='records'),
strax.DataDirectory(out_dir + '/temp_processed',
exclude=['records', 'raw_records'])
],
config=dict(input_dir=in_dir, erase=args.erase))
if args.mongo:
st.storage.append(
strax.MongoStore(mongo_uri, take_only=['events', 'event_basics']))
st.register_all(strax.xenon.plugins)
gil_load.start(av_sample_interval=0.05)
start = time.time()
for i, events in enumerate(st.get_iter(run_id, args.target,
max_workers=args.n)):
print(f"\t{i}: Found {len(events)} events")
end = time.time()
gil_load.stop()
dt = end - start
gil_pct = 100 * gil_load.get(4)[0]
print(f"Took {dt:.3f} seconds, GIL was held {gil_pct:.3f}% of the time")
def total_size(data_type, raw=False):
def compiler():
global exceptions
exceptions = False
if not CompilationManager.config.raw_opts.disable_cache:
Cache.load_cache()
if CompilationManager.config.raw_opts.file is not None:
path = Path(CompilationManager.config.raw_opts.file)
else:
path = CompilationManager.config.rial_path.joinpath(
"builtin").joinpath("start.rial")
if not path.exists():
raise FileNotFoundError(str(path))
# Clear global data that can sometimes remain (especially during test execution).
context.global_context.scope._useset.clear()
context.global_context.identified_types.clear()
# Collect all always imported paths
builtin_path = str(
CompilationManager.config.rial_path.joinpath("builtin").joinpath(
"always_imported"))
for file in [
join(builtin_path, f) for f in listdir(builtin_path)
if isfile(join(builtin_path, f))
]:
CompilationManager.request_file(file)
module_name = CompilationManager.mod_name_from_path(
CompilationManager.filename_from_path(str(file)))
CompilationManager.always_imported.append(module_name)
# Request main file
CompilationManager.request_file(str(path))
if CompilationManager.config.raw_opts.profile_gil:
import gil_load
gil_load.init()
gil_load.start()
futures = list()
with concurrent.futures.ThreadPoolExecutor() as executor:
while True:
for future in futures:
if future.done():
futures.remove(future)
try:
# This timeout is so small, it shouldn't matter.
# However it serves a crucial role in regards to Python's GIL.
# Setting this on a higher timeout obviously means that there may be some excessive time spent waiting
# for nothing to appear.
# BUT, setting it to non-blocking means that the GIL never gives any of the compilation threads priority
# as well as holding up the internal threading.Lock(s).
# So setting this timeout so small means not much time is spent waiting for nothing, but it gives other
# things the opportunity to get control of the GIL and execute.
path = CompilationManager.files_to_compile.get(timeout=0.01)
future = executor.submit(compile_file, path)
futures.append(future)
except Empty:
pass
if len(futures) == 0 and CompilationManager.files_to_compile.empty(
):
break
executor.shutdown(True)
CompilationManager.files_to_compile.join()
if CompilationManager.config.raw_opts.profile_gil:
import gil_load
gil_load.stop()
print(f"Main Thread ID: {threading.current_thread().ident}")
print(gil_load.format(gil_load.get()))
if exceptions:
return
if not CompilationManager.config.raw_opts.disable_cache:
Cache.save_cache()
execute_stage(Stage.POST_IR_GEN)
modules: Dict[str, ModuleRef] = dict()
for key, mod in CompilationManager.modules.items():
if not CompilationManager.config.raw_opts.disable_cache:
cache_path = str(
CompilationManager.get_cache_path_str(key)).replace(
".rial", ".cache")
if not Path(cache_path).exists():
CompilationManager.codegen.save_module(mod, cache_path)
with run_with_profiling(CompilationManager.filename_from_path(key),
ExecutionStep.COMPILE_MOD):
try:
modules[key] = CompilationManager.codegen.compile_ir(mod)
except Exception as e:
log_fail(f"Exception when compiling module {mod.name}")
log_fail(e)
log_fail(traceback.format_exc())
return
object_files: List[str] = list()
CompilationManager.codegen.generate_final_modules(list(modules.values()))
for path in list(modules.keys()):
mod = modules[path]
if CompilationManager.config.raw_opts.print_ir:
ir_file = str(
CompilationManager.get_output_path_str(path)).replace(
".rial", ".ll")
if check_needs_output(mod.name, ir_file):
CompilationManager.codegen.save_ir(ir_file, mod)
if CompilationManager.config.raw_opts.print_asm:
asm_file = str(
CompilationManager.get_cache_path_str(path)).replace(
".rial", ".asm")
if check_needs_output(mod.name, asm_file):
CompilationManager.codegen.save_assembly(asm_file, mod)
if not CompilationManager.config.raw_opts.use_object_files:
llvm_bitcode_file = str(
CompilationManager.get_output_path_str(path)).replace(
".rial", ".o")
if check_needs_output(mod.name, llvm_bitcode_file):
CompilationManager.codegen.save_llvm_bitcode(
llvm_bitcode_file, mod)
object_files.append(llvm_bitcode_file)
else:
object_file = str(
CompilationManager.get_output_path_str(path)).replace(
".rial", ".o")
if check_needs_output(mod.name, object_file):
CompilationManager.codegen.save_object(object_file, mod)
object_files.append(object_file)
with run_with_profiling(CompilationManager.config.project_name,
ExecutionStep.LINK_EXE):
exe_path = str(
CompilationManager.config.bin_path.joinpath(
f"{CompilationManager.config.project_name}{Platform.get_exe_file_extension()}"
))
Linker.link_files(
object_files, exe_path,
CompilationManager.config.raw_opts.print_link_command,
CompilationManager.config.raw_opts.strip)