def main(self, argv): TauInstallation.check_env_compat() args = self._parse_args(argv) store = arguments.parse_storage_flag(args)[0] if hasattr(args, "forced_makefile"): self._parse_tau_makefile(args) self.logger.debug('Arguments after parsing TAU Makefile: %s', args) compilers = self.parse_compiler_flags(args) data = {attr: getattr(args, attr) for attr in self.model.attributes if hasattr(args, attr)} for comp in compilers.itervalues(): record = Compiler.controller(store).register(comp) data[comp.info.role.keyword] = record.eid return super(TargetCreateCommand, self)._create_record(store, data)
def main(self, argv): args = self._parse_args(argv) profile_tools = getattr(args, 'profile_tools', None) trace_tools = getattr(args, 'trace_tools', None) data_files = [] trial_numbers = [] for num in getattr(args, 'trial_numbers', []) + getattr( args, 'data_files', []): if os.path.exists(num): data_files.append(num) else: try: trial_numbers.append(int(num)) except ValueError: self.parser.error("Invalid trial number: %s" % num) tau = TauInstallation.get_minimal() dataset = {} if not (data_files or trial_numbers): expr = Project.selected().experiment() for fmt, path in expr.trials()[0].get_data_files().iteritems(): dataset[fmt] = [path] elif trial_numbers: expr = Project.selected().experiment() for trial in expr.trials(trial_numbers): for fmt, path in trial.get_data_files().iteritems(): dataset.setdefault(fmt, []).append(path) for path in data_files: fmt = tau.get_data_format(path) dataset.setdefault(fmt, []).append(path) return tau.show_data_files(dataset, profile_tools, trace_tools)
def main(self, argv): args = self._parse_args(argv) profile_tools = getattr(args, 'profile_tools', None) trace_tools = getattr(args, 'trace_tools', None) data_files = [] trial_numbers = [] for num in getattr(args, 'trial_numbers', []) + getattr(args, 'data_files', []): if os.path.exists(num): data_files.append(num) else: try: trial_numbers.append(int(num)) except ValueError: self.parser.error("Invalid trial number: %s" % num) tau = TauInstallation.get_minimal() dataset = {} if not (data_files or trial_numbers): expr = Project.selected().experiment() for fmt, path in expr.trials()[0].get_data_files().iteritems(): dataset[fmt] = [path] elif trial_numbers: expr = Project.selected().experiment() for trial in expr.trials(trial_numbers): for fmt, path in trial.get_data_files().iteritems(): dataset.setdefault(fmt, []).append(path) for path in data_files: fmt = tau.get_data_format(path) dataset.setdefault(fmt, []).append(path) return tau.show_data_files(dataset, profile_tools, trace_tools)
def main(self, argv): TauInstallation.check_env_compat() args = self._parse_args(argv) store = arguments.parse_storage_flag(args)[0] compilers = target_create_cmd.parse_compiler_flags(args) data = {attr: getattr(args, attr) for attr in self.model.attributes if hasattr(args, attr)} for keyword, comp in compilers.iteritems(): self.logger.debug("%s=%s (%s)", keyword, comp.absolute_path, comp.info.short_descr) record = Compiler.controller(store).register(comp) data[comp.info.role.keyword] = record.eid key_attr = self.model.key_attribute try: data[key_attr] = getattr(args, 'copy_%s' % key_attr) except AttributeError: pass key = getattr(args, key_attr) return self._copy_record(store, data, key)
def main(self, argv): TauInstallation.check_env_compat() args = self._parse_args(argv) store = arguments.parse_storage_flag(args)[0] if hasattr(args, "forced_makefile"): self._parse_tau_makefile(args) self.logger.debug('Arguments after parsing TAU Makefile: %s', args) compilers = self.parse_compiler_flags(args) data = { attr: getattr(args, attr) for attr in self.model.attributes if hasattr(args, attr) } for comp in compilers.itervalues(): record = Compiler.controller(store).register(comp) data[comp.info.role.keyword] = record.eid return super(TargetCreateCommand, self)._create_record(store, data)
def export(self, dest): """Export experiment trial data. Args: dest (str): Path to directory to contain exported data. Raises: ConfigurationError: This trial has no data. """ expr = self.populate('experiment') if self.get('data_size', 0) <= 0: raise ConfigurationError( "Trial %s of experiment '%s' has no data" % (self['number'], expr['name'])) data = self.get_data_files() stem = '%s.trial%d' % (expr['name'], self['number']) for fmt, path in data.iteritems(): if fmt == 'tau': export_file = os.path.join(dest, stem + '.ppk') tau = TauInstallation.get_minimal() tau.create_ppk_file(export_file, path) elif fmt == 'merged': export_file = os.path.join(dest, stem + '.xml.gz') util.create_archive('gz', export_file, [path]) elif fmt == 'cubex': export_file = os.path.join(dest, stem + '.cubex') LOGGER.info("Writing '%s'...", export_file) util.copy_file(path, export_file) elif fmt == 'slog2': export_file = os.path.join(dest, stem + '.slog2') LOGGER.info("Writing '%s'...", export_file) util.copy_file(path, export_file) elif fmt == 'otf2': export_file = os.path.join(dest, stem + '.tgz') expr_dir, trial_dir = os.path.split(os.path.dirname(path)) items = [ os.path.join(trial_dir, item) for item in 'traces', 'traces.def', 'traces.otf2' ] util.create_archive('tgz', export_file, items, expr_dir) elif fmt != 'none': raise InternalError("Unhandled data file format '%s'" % fmt)
def _postprocess_slog2(self): slog2 = os.path.join(self.prefix, 'tau.slog2') if os.path.exists(slog2): return tau = TauInstallation.get_minimal() merged_trc = os.path.join(self.prefix, 'tau.trc') merged_edf = os.path.join(self.prefix, 'tau.edf') if not os.path.exists(merged_trc) or not os.path.exists(merged_edf): tau.merge_tau_trace_files(self.prefix) tau.tau_trace_to_slog2(merged_trc, merged_edf, slog2) trc_files = glob.glob(os.path.join(self.prefix, '*.trc')) edf_files = glob.glob(os.path.join(self.prefix, '*.edf')) count_trc_edf = len(trc_files) + len(edf_files) LOGGER.info('Cleaning up TAU trace files...') with ProgressIndicator("", total_size=count_trc_edf) as progress_bar: count = 0 for path in trc_files + edf_files: os.remove(path) count += 1 progress_bar.update(count)
def _postprocess_slog2(self): slog2 = os.path.join(self.prefix, 'tau.slog2') if os.path.exists(slog2): return tau = TauInstallation.get_minimal() merged_trc = os.path.join(self.prefix, 'tau.trc') merged_edf = os.path.join(self.prefix, 'tau.edf') if not os.path.exists(merged_trc) or not os.path.exists(merged_edf): tau.merge_tau_trace_files(self.prefix) tau.tau_trace_to_slog2(merged_trc, merged_edf, slog2) trc_files = glob.glob(os.path.join(self.prefix, '*.trc')) edf_files = glob.glob(os.path.join(self.prefix, '*.edf')) count_trc_edf = len(trc_files) + len(edf_files) LOGGER.info('Cleaning up TAU trace files...') with ProgressIndicator(count_trc_edf) as progress_bar: count = 0 for path in trc_files + edf_files: os.remove(path) count += 1 progress_bar.update(count)
def export(self, dest): """Export experiment trial data. Args: dest (str): Path to directory to contain exported data. Raises: ConfigurationError: This trial has no data. """ expr = self.populate('experiment') if self.get('data_size', 0) <= 0: raise ConfigurationError("Trial %s of experiment '%s' has no data" % (self['number'], expr['name'])) data = self.get_data_files() stem = '%s.trial%d' % (expr['name'], self['number']) for fmt, path in data.iteritems(): if fmt == 'tau': export_file = os.path.join(dest, stem+'.ppk') tau = TauInstallation.get_minimal() tau.create_ppk_file(export_file, path) elif fmt == 'merged': export_file = os.path.join(dest, stem+'.xml.gz') util.create_archive('gz', export_file, [path]) elif fmt == 'cubex': export_file = os.path.join(dest, stem+'.cubex') LOGGER.info("Writing '%s'...", export_file) util.copy_file(path, export_file) elif fmt == 'slog2': export_file = os.path.join(dest, stem+'.slog2') LOGGER.info("Writing '%s'...", export_file) util.copy_file(path, export_file) elif fmt == 'otf2': export_file = os.path.join(dest, stem+'.tgz') expr_dir, trial_dir = os.path.split(os.path.dirname(path)) items = [os.path.join(trial_dir, item) for item in 'traces', 'traces.def', 'traces.otf2'] util.create_archive('tgz', export_file, items, expr_dir) elif fmt != 'none': raise InternalError("Unhandled data file format '%s'" % fmt)
def configure(self): """Sets up the Experiment for a new trial. Installs or configures TAU and all its dependencies. After calling this function, the experiment is ready to operate on the user's application. Returns: TauInstallation: Object handle for the TAU installation. """ from taucmdr.cf.software.tau_installation import TauInstallation LOGGER.debug("Configuring experiment %s", self['name']) with fasteners.InterProcessLock( os.path.join(PROJECT_STORAGE.prefix, '.lock')): populated = self.populate(defaults=True) target = populated['target'] application = populated['application'] measurement = populated['measurement'] baseline = measurement.get_or_default('baseline') tau = TauInstallation(\ target.sources(), target_arch=target.architecture(), target_os=target.operating_system(), compilers=target.compilers(), # Use a minimal configuration for the baseline measurement minimal=baseline, # TAU feature suppport application_linkage=application.get_or_default('linkage'), openmp_support=application.get_or_default('openmp'), pthreads_support=application.get_or_default('pthreads'), tbb_support=application.get_or_default('tbb'), mpi_support=application.get_or_default('mpi'), mpi_libraries=target.get('mpi_libraries', []), cuda_support=application.get_or_default('cuda'), cuda_prefix=target.get('cuda_toolkit', None), opencl_support=application.get_or_default('opencl'), opencl_prefix=target.get('opencl', None), shmem_support=application.get_or_default('shmem'), shmem_libraries=target.get('shmem_libraries', []), mpc_support=application.get_or_default('mpc'), # Instrumentation methods and options source_inst=measurement.get_or_default('source_inst'), compiler_inst=measurement.get_or_default('compiler_inst'), keep_inst_files=measurement.get_or_default('keep_inst_files'), reuse_inst_files=measurement.get_or_default('reuse_inst_files'), select_file=application.get('select_file', None), # Measurement methods and options baseline=baseline, profile=measurement.get_or_default('profile'), trace=measurement.get_or_default('trace'), sample=measurement.get_or_default('sample'), metrics=measurement.get_or_default('metrics'), measure_io=measurement.get_or_default('io'), measure_mpi=measurement.get_or_default('mpi'), measure_openmp=measurement.get_or_default('openmp'), measure_opencl=measurement.get_or_default('opencl'), measure_cuda=measurement.get_or_default('cuda'), measure_shmem=measurement.get_or_default('shmem'), measure_heap_usage=measurement.get_or_default('heap_usage'), measure_system_load=measurement.get_or_default('system_load'), measure_memory_alloc=measurement.get_or_default('memory_alloc'), measure_comm_matrix=measurement.get_or_default('comm_matrix'), measure_callsite=measurement.get_or_default('callsite'), callpath_depth=measurement.get_or_default('callpath'), throttle=measurement.get_or_default('throttle'), metadata_merge=measurement.get_or_default('metadata_merge'), throttle_per_call=measurement.get_or_default('throttle_per_call'), throttle_num_calls=measurement.get_or_default('throttle_num_calls'), forced_makefile=target.get('forced_makefile', None)) tau.install() if not baseline: self.controller(self.storage).update( {'tau_makefile': os.path.basename(tau.get_makefile())}, self.eid) return tau
def managed_rewrite(self, rewrite_package, executable, inst_file): from taucmdr.cf.software.tau_installation import TauInstallation with fasteners.InterProcessLock(os.path.join(PROJECT_STORAGE.prefix, '.lock')): populated = self.populate(defaults=True) target = populated['target'] application = populated['application'] measurement = populated['measurement'] if rewrite_package == 'maqao' or rewrite_package == 'pebil': source_inst = "automatic" dyninst = False else: source_inst = 'never' dyninst = True tau = TauInstallation(\ target.sources(), target_arch=target.architecture(), target_os=target.operating_system(), compilers=target.compilers(), # TAU feature suppport application_linkage=application.get_or_default('linkage'), openmp_support=application.get_or_default('openmp'), pthreads_support=application.get_or_default('pthreads'), tbb_support=application.get_or_default('tbb'), mpi_support=application.get_or_default('mpi'), mpi_libraries=target.get('mpi_libraries', []), caf_support=application.get_or_default('caf'), cuda_support=application.get_or_default('cuda'), cuda_prefix=target.get('cuda_toolkit', None), opencl_support=application.get_or_default('opencl'), opencl_prefix=target.get('opencl', None), shmem_support=application.get_or_default('shmem'), shmem_libraries=target.get('shmem_libraries', []), mpc_support=application.get_or_default('mpc'), max_threads=application.get('max_threads', None), # Instrumentation methods and options source_inst=source_inst, compiler_inst=measurement.get_or_default('compiler_inst'), keep_inst_files=measurement.get_or_default('keep_inst_files'), reuse_inst_files=measurement.get_or_default('reuse_inst_files'), select_file=measurement.get('select_file', None), # Measurement methods and options profile=measurement.get_or_default('profile'), trace=measurement.get_or_default('trace'), sample=measurement.get_or_default('sample'), metrics=measurement.get_or_default('metrics'), measure_io=measurement.get_or_default('io'), measure_mpi=measurement.get_or_default('mpi'), measure_openmp=measurement.get_or_default('openmp'), measure_opencl=measurement.get_or_default('opencl'), measure_cuda=measurement.get_or_default('cuda'), measure_shmem=measurement.get_or_default('shmem'), measure_heap_usage=measurement.get_or_default('heap_usage'), measure_system_load=measurement.get_or_default('system_load'), measure_memory_alloc=measurement.get_or_default('memory_alloc'), measure_comm_matrix=measurement.get_or_default('comm_matrix'), measure_callsite=measurement.get_or_default('callsite'), callpath_depth=measurement.get_or_default('callpath'), throttle=measurement.get_or_default('throttle'), metadata_merge=measurement.get_or_default('metadata_merge'), throttle_per_call=measurement.get_or_default('throttle_per_call'), throttle_num_calls=measurement.get_or_default('throttle_num_calls'), sampling_period=measurement.get_or_default('sampling_period'), track_memory_footprint=measurement.get_or_default('track_memory_footprint'), update_nightly=measurement.get_or_default('update_nightly'), ptts=measurement.get_or_default('ptts'), ptts_post=measurement.get_or_default('ptts_post'), ptts_sample_flags=measurement.get_or_default('ptts_sample_flags'), ptts_restart=measurement.get_or_default('ptts_restart'), ptts_start=measurement.get_or_default('ptts_start'), ptts_stop=measurement.get_or_default('ptts_stop'), ptts_report_flags=measurement.get_or_default('ptts_report_flags'), forced_makefile=target.get('forced_makefile', None), mpit=measurement.get_or_default('mpit'), unwind_depth=measurement.get_or_default('unwind_depth'), dyninst=dyninst) tau.install() tau.rewrite(rewrite_package, executable, inst_file) return tau
def configure(self): """Sets up the Experiment for a new trial. Installs or configures TAU and all its dependencies. After calling this function, the experiment is ready to operate on the user's application. Returns: TauInstallation: Object handle for the TAU installation. """ from taucmdr.cf.software.tau_installation import TauInstallation LOGGER.debug("Configuring experiment %s", self['name']) with fasteners.InterProcessLock(os.path.join(PROJECT_STORAGE.prefix, '.lock')): populated = self.populate(defaults=True) target = populated['target'] application = populated['application'] measurement = populated['measurement'] baseline = measurement.get_or_default('baseline') tau = TauInstallation(\ target.sources(), target_arch=target.architecture(), target_os=target.operating_system(), compilers=target.compilers(), # Use a minimal configuration for the baseline measurement minimal=baseline, # TAU feature suppport application_linkage=application.get_or_default('linkage'), openmp_support=application.get_or_default('openmp'), pthreads_support=application.get_or_default('pthreads'), tbb_support=application.get_or_default('tbb'), mpi_support=application.get_or_default('mpi'), mpi_libraries=target.get('mpi_libraries', []), cuda_support=application.get_or_default('cuda'), cuda_prefix=target.get('cuda_toolkit', None), opencl_support=application.get_or_default('opencl'), opencl_prefix=target.get('opencl', None), shmem_support=application.get_or_default('shmem'), shmem_libraries=target.get('shmem_libraries', []), mpc_support=application.get_or_default('mpc'), # Instrumentation methods and options source_inst=measurement.get_or_default('source_inst'), compiler_inst=measurement.get_or_default('compiler_inst'), keep_inst_files=measurement.get_or_default('keep_inst_files'), reuse_inst_files=measurement.get_or_default('reuse_inst_files'), select_file=application.get('select_file', None), # Measurement methods and options baseline=baseline, profile=measurement.get_or_default('profile'), trace=measurement.get_or_default('trace'), sample=measurement.get_or_default('sample'), metrics=measurement.get_or_default('metrics'), measure_io=measurement.get_or_default('io'), measure_mpi=measurement.get_or_default('mpi'), measure_openmp=measurement.get_or_default('openmp'), measure_opencl=measurement.get_or_default('opencl'), measure_cuda=measurement.get_or_default('cuda'), measure_shmem=measurement.get_or_default('shmem'), measure_heap_usage=measurement.get_or_default('heap_usage'), measure_system_load=measurement.get_or_default('system_load'), measure_memory_alloc=measurement.get_or_default('memory_alloc'), measure_comm_matrix=measurement.get_or_default('comm_matrix'), measure_callsite=measurement.get_or_default('callsite'), callpath_depth=measurement.get_or_default('callpath'), throttle=measurement.get_or_default('throttle'), metadata_merge=measurement.get_or_default('metadata_merge'), throttle_per_call=measurement.get_or_default('throttle_per_call'), throttle_num_calls=measurement.get_or_default('throttle_num_calls'), forced_makefile=target.get('forced_makefile', None)) tau.install() if not baseline: self.controller(self.storage).update({'tau_makefile': os.path.basename(tau.get_makefile())}, self.eid) return tau