def before(cmd, **_):
""" Phase for initialization the collect module
Arguments:
cmd(string): binary file to profile
Returns:
tuple: (return code, status message, updated kwargs)
"""
pwd = os.path.dirname(os.path.abspath(__file__))
if not os.path.isfile("{}/{}".format(pwd, _lib_name)):
print(
"Missing compiled dynamic library 'lib{}'. Compiling from sources: "
.format(os.path.splitext(_lib_name)[0]),
end='')
result = syscalls.init()
if result:
log.failed()
error_msg = 'Build of the library failed with error code: '
error_msg += str(result)
return CollectStatus.ERROR, error_msg, {}
else:
log.done()
print("Checking if binary contains debugging information: ", end='')
if not syscalls.check_debug_symbols(cmd):
log.failed()
error_msg = "Binary does not contain debug info section.\n"
error_msg += "Please recompile your project with debug options (gcc -g | g++ -g)"
return CollectStatus.ERROR, error_msg, {}
log.done()
print("Finished preprocessing step!\n")
return CollectStatus.OK, '', {}
def after(**kwargs):
""" Handles the trace collector output and transforms it into resources
The output dictionary is updated with:
- profile: the performance profile contents created from the collector output
:param kwargs: dictionary containing the configuration settings for the collector
:returns: tuple (int as a status code, nonzero values for errors,
string as a status message, mainly for error states,
dict of kwargs and new values)
"""
log.cprint('Starting the post-processing phase... ', 'white')
# Get the trace log path
try:
resources = list(systemtap.trace_to_profile(**kwargs))
# Update the profile dictionary
kwargs['profile'] = {
'global': {
'timestamp':
sum(res['amount'] for res in resources) / _MICRO_TO_SECONDS,
'resources': resources
}
}
log.done()
result = _COLLECTOR_STATUS[systemtap.Status.OK]
return result[0], result[1], dict(kwargs)
except (CalledProcessError, exceptions.TraceStackException) as exception:
log.failed()
return _COLLECTOR_STATUS[systemtap.Status.EXCEPT], str(
exception), dict(kwargs)
def before(**kwargs):
""" Assembles the SystemTap script according to input parameters and collection strategy
The output dictionary is updated with:
- timestamp: current timestamp that is used for saved files
- cmd, cmd_dir, cmd_base: absolute path to the command, its directory and the command base name
- script_path: path to the generated script file
- log_path: path to the collection log
- output_path: path to the collection output
:param kwargs: dictionary containing the configuration settings for the collector
:returns: tuple (int as a status code, nonzero values for errors,
string as a status message, mainly for error states,
dict of kwargs and new values)
"""
try:
log.cprint('Starting the pre-processing phase... ', 'white')
# Update the configuration dictionary with some additional values
kwargs['timestamp'] = time.strftime("%Y-%m-%d-%H-%M-%S",
time.localtime())
_, kwargs['cmd_dir'], kwargs['cmd_base'] = utils.get_path_dir_file(
kwargs['cmd'])
kwargs['script_path'] = _create_collector_file('script', '.stp',
**kwargs)
kwargs['log_path'] = _create_collector_file('log', **kwargs)
kwargs['output_path'] = _create_collector_file('record', **kwargs)
# Validate collection parameters
kwargs = _validate_input(**kwargs)
# Extract and / or post process the collect configuration
kwargs = strategy.extract_configuration(**kwargs)
if not kwargs['func'] and not kwargs['static'] and not kwargs[
'dynamic']:
return CollectStatus.ERROR, (
'No function, static or dynamic probes to be profiled '
'(due to invalid specification, failed extraction or '
'filtering)'), dict(kwargs)
# Assemble script according to the parameters
stap_script.assemble_system_tap_script(**kwargs)
log.done()
result = _COLLECTOR_STATUS[systemtap.Status.OK]
return result[0], result[1], dict(kwargs)
except (OSError, ValueError, CalledProcessError, UnicodeError,
exceptions.InvalidBinaryException) as exception:
log.failed()
return _COLLECTOR_STATUS[systemtap.Status.EXCEPT][0], str(
exception), dict(kwargs)
def load_generator_specifications():
"""Collects from configuration file all of the workload specifications and constructs a mapping
of 'id' -> GeneratorSpec, which contains the constructor and parameters used for construction.
The specifications are specified by :ckey:`generators.workload` as follows:
generators:
workload:
- id: name_of_generator
type: integer
min_range: 10
max_range: 20
:return: map of ids to generator specifications
"""
specifications_from_config = config.gather_key_recursively(
'generators.workload')
spec_map = {}
warnings = []
print("Loading workload generator specifications ", end='')
for spec in specifications_from_config:
if 'id' not in spec.keys() or 'type' not in spec.keys():
warnings.append(
"incorrect workload specification: missing 'id' or 'type'")
print("F", end='')
continue
generator_module = "perun.workload.{}_generator".format(
spec['type'].lower())
constructor_name = "{}Generator".format(spec['type'].title())
try:
constructor = getattr(utils.get_module(generator_module),
constructor_name)
spec_map[spec['id']] = GeneratorSpec(constructor, spec)
print(".", end='')
except (ImportError, AttributeError):
warnings.append(
"incorrect workload generator '{}': '{}' is not valid workload type"
.format(spec['id'], spec['type']))
print("F", end='')
# Print the warnings and badge
if len(warnings):
log.failed()
print("\n".join(warnings))
else:
log.done()
return spec_map
def collect(cmd, args, workload, **_):
""" Phase for collection of the profile data
Arguments:
cmd(string): binary file to profile
args(string): executing arguments
workload(string): file that has to be provided to binary
Returns:
tuple: (return code, status message, updated kwargs)
"""
print("Collecting data: ", end='')
result, collector_errors = syscalls.run(cmd, args, workload)
if result:
log.failed()
error_msg = 'Execution of binary failed with error code: '
error_msg += str(result) + "\n"
error_msg += collector_errors
return CollectStatus.ERROR, error_msg, {}
log.done()
print("Finished collection of the raw data!\n")
return CollectStatus.OK, '', {}
def systemtap_collect(script_path, log_path, output_path, cmd, args, **kwargs):
"""Collects performance data using the system tap wrapper, assembled script and
external command. This function serves as a interface to the system tap collector.
:param str script_path: path to the assembled system tap script file
:param str log_path: path to the collection log file
:param str output_path: path to the collection output file
:param str cmd: the external command that contains / invokes the profiled executable
:param list args: the arguments supplied to the external command
:param kwargs: additional collector configuration
:return tuple: containing the collection status, path to the output file of the collector
"""
# Perform the cleanup
if kwargs['cleanup']:
_stap_cleanup()
# Create the output and log file for collection
with open(log_path, 'w') as logfile:
# Start the SystemTap process
log.cprint('Starting the SystemTap process... ', 'white')
stap_pgid = set()
try:
stap_runner, code = start_systemtap_in_background(
script_path, output_path, logfile, **kwargs)
if code != Status.OK:
return code, None
stap_pgid.add(os.getpgid(stap_runner.pid))
log.done()
# Run the command that is supposed to be profiled
log.cprint(
'SystemTap up and running, execute the profiling target... ',
'white')
run_profiled_command(cmd, args, **kwargs)
log.done()
# Terminate SystemTap process after the file was fully written
log.cprint(
'Data collection complete, terminating the SystemTap process... ',
'white')
# _wait_for_fully_written(output_path)
_wait_for_fully_written(output_path)
kill_systemtap_in_background(stap_pgid)
log.done()
return Status.OK, output_path
# Timeout was reached, inform the user but continue normally
except exceptions.HardTimeoutException as e:
kill_systemtap_in_background(stap_pgid)
log.cprintln('', 'white')
log.warn(e.msg)
log.warn('The profile creation might fail or be inaccurate.')
return Status.OK, output_path
# Critical error during profiling or collection interrupted
# make sure we terminate the collector and remove module
except (Exception, KeyboardInterrupt):
if not stap_pgid:
stap_pgid = None
# Clean only our mess
_stap_cleanup(stap_pgid, output_path)
raise
def after(cmd, sampling=DEFAULT_SAMPLING, **kwargs):
""" Phase after the collection for minor postprocessing
that needs to be done after collect
Arguments:
cmd(string): binary file to profile
sampling(int): sampling of the collection of the data
kwargs(dict): profile's header
Returns:
tuple: (return code, message, updated kwargs)
Case studies:
--sampling=0.1 --no-func=f2 --no-source=s --all
-> run memory collector with 0.1s sampling,
excluding allocations in "f2" function and in "s" source file,
including allocators and unreachable records in call trace
--no-func=f1 --no-source=s --all
-> run memory collector with 0.001s sampling,
excluding allocations in "f1" function and in "s" source file,
including allocators and unreachable records in call trace
--no-func=f1 --sampling=1.0
-> run memory collector with 1.0s sampling,
excluding allocations in "f1" function,
excluding allocators and unreachable records in call trace
"""
include_all = kwargs.get('all', False)
exclude_funcs = kwargs.get('no_func', None)
exclude_sources = kwargs.get('no_source', None)
print("Generating profile: ", end='')
try:
profile = parser.parse_log(_tmp_log_filename, cmd, sampling)
except IndexError as i_err:
log.failed()
return CollectStatus.ERROR, 'Info missing in log file: {}'.format(
str(i_err)), {}
except ValueError as v_err:
log.failed()
return CollectStatus.ERROR, 'Wrong format of log file: {}'.format(
str(v_err)), {}
log.done()
if not include_all:
print("Filtering traces: ", end='')
filters.remove_allocators(profile)
filters.trace_filter(profile, function=['?'], source=['unreachable'])
log.done()
if exclude_funcs or exclude_sources:
print("Excluding functions and sources: ", end='')
filters.allocation_filter(profile,
function=[exclude_funcs],
source=[exclude_sources])
log.done()
print("Clearing records without assigned UID from profile: ", end='')
filters.remove_uidless_records_from(profile)
log.done()
print("")
return CollectStatus.OK, '', {'profile': profile}