def __init__(self, *args, **kwargs): super(RunTracker, self).__init__(*args, **kwargs) run_timestamp = time.time() cmd_line = ' '.join(['pants'] + sys.argv[1:]) # run_id is safe for use in paths. millis = int((run_timestamp * 1000) % 1000) run_id = 'pants_run_{}_{}_{}'.format( time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime(run_timestamp)), millis, uuid.uuid4().hex) info_dir = os.path.join(self.get_options().pants_workdir, self.options_scope) self.run_info_dir = os.path.join(info_dir, run_id) self.run_info = RunInfo(os.path.join(self.run_info_dir, 'info')) self.run_info.add_basic_info(run_id, run_timestamp) self.run_info.add_info('cmd_line', cmd_line) # Create a 'latest' symlink, after we add_infos, so we're guaranteed that the file exists. link_to_latest = os.path.join(os.path.dirname(self.run_info_dir), 'latest') relative_symlink(self.run_info_dir, link_to_latest) # Time spent in a workunit, including its children. self.cumulative_timings = AggregatedTimings(os.path.join(self.run_info_dir, 'cumulative_timings')) # Time spent in a workunit, not including its children. self.self_timings = AggregatedTimings(os.path.join(self.run_info_dir, 'self_timings')) # Hit/miss stats for the artifact cache. self.artifact_cache_stats = \ ArtifactCacheStats(os.path.join(self.run_info_dir, 'artifact_cache_stats')) # Number of threads for foreground work. self._num_foreground_workers = self.get_options().num_foreground_workers # Number of threads for background work. self._num_background_workers = self.get_options().num_background_workers # We report to this Report. self.report = None # self._threadlocal.current_workunit contains the current workunit for the calling thread. # Note that multiple threads may share a name (e.g., all the threads in a pool). self._threadlocal = threading.local() # For main thread work. Created on start(). self._main_root_workunit = None # For background work. Created lazily if needed. self._background_worker_pool = None self._background_root_workunit = None # Trigger subproc pool init while our memory image is still clean (see SubprocPool docstring). SubprocPool.set_num_processes(self._num_foreground_workers) SubprocPool.foreground() self._aborted = False
def start(self, all_options, run_start_time=None): """Start tracking this pants run.""" if self.run_info: raise AssertionError( "RunTracker.start must not be called multiple times.") # Initialize the run. info_dir = os.path.join(self.options.pants_workdir, self.options_scope) self.run_info_dir = os.path.join(info_dir, self.run_id) self.run_info = RunInfo(os.path.join(self.run_info_dir, "info")) self.run_info.add_basic_info(self.run_id, self._run_timestamp) self.run_info.add_info("cmd_line", self._cmd_line) if self.options.parent_build_id: self.run_info.add_info("parent_build_id", self.options.parent_build_id) # Create a 'latest' symlink, after we add_infos, so we're guaranteed that the file exists. link_to_latest = os.path.join(os.path.dirname(self.run_info_dir), "latest") relative_symlink(self.run_info_dir, link_to_latest) # Time spent in a workunit, including its children. self.cumulative_timings = AggregatedTimings( os.path.join(self.run_info_dir, "cumulative_timings")) # Time spent in a workunit, not including its children. self.self_timings = AggregatedTimings( os.path.join(self.run_info_dir, "self_timings")) # Daemon stats. self.pantsd_stats = PantsDaemonStats() self._all_options = all_options self.report = Report() # Set up the JsonReporter for V2 stats. if self._stats_version == 2: json_reporter_settings = JsonReporter.Settings( log_level=Report.INFO) self.json_reporter = JsonReporter(self, json_reporter_settings) self.report.add_reporter("json", self.json_reporter) self.report.open() # And create the workunit. self._main_root_workunit = WorkUnit(run_info_dir=self.run_info_dir, parent=None, name=RunTracker.DEFAULT_ROOT_NAME, cmd=None) self.register_thread(self._main_root_workunit) # Set the true start time in the case of e.g. the daemon. self._main_root_workunit.start(run_start_time) self.report.start_workunit(self._main_root_workunit)
def initialize(self, all_options): """Create run_info and relevant directories, and return the run id. Must be called before `start`. """ if self.run_info: raise AssertionError( 'RunTracker.initialize must not be called multiple times.') # Initialize the run. # Select a globally unique ID for the run, that sorts by time. millis = int((self._run_timestamp * 1000) % 1000) # run_uuid is used as a part of run_id and also as a trace_id for Zipkin tracing run_uuid = uuid.uuid4().hex run_id = 'pants_run_{}_{}_{}'.format( time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime(self._run_timestamp)), millis, run_uuid) info_dir = os.path.join(self.get_options().pants_workdir, self.options_scope) self.run_info_dir = os.path.join(info_dir, run_id) self.run_info = RunInfo(os.path.join(self.run_info_dir, 'info')) self.run_info.add_basic_info(run_id, self._run_timestamp) self.run_info.add_info('cmd_line', self._cmd_line) # Create a 'latest' symlink, after we add_infos, so we're guaranteed that the file exists. link_to_latest = os.path.join(os.path.dirname(self.run_info_dir), 'latest') relative_symlink(self.run_info_dir, link_to_latest) # Time spent in a workunit, including its children. self.cumulative_timings = AggregatedTimings( os.path.join(self.run_info_dir, 'cumulative_timings')) # Time spent in a workunit, not including its children. self.self_timings = AggregatedTimings( os.path.join(self.run_info_dir, 'self_timings')) # Hit/miss stats for the artifact cache. self.artifact_cache_stats = ArtifactCacheStats( os.path.join(self.run_info_dir, 'artifact_cache_stats')) # Daemon stats. self.pantsd_stats = PantsDaemonStats() self._all_options = all_options return (run_id, run_uuid)
def start(self, all_options: Options, run_start_time: float) -> None: """Start tracking this pants run.""" if self.run_info: raise AssertionError( "RunTracker.start must not be called multiple times.") # Initialize the run. info_dir = os.path.join(self.options.pants_workdir, self.options_scope) self.run_info_dir = os.path.join(info_dir, self.run_id) self.run_info = RunInfo(os.path.join(self.run_info_dir, "info")) self.run_info.add_basic_info(self.run_id, self._run_timestamp) self.run_info.add_info("cmd_line", self._cmd_line) # Create a 'latest' symlink, after we add_infos, so we're guaranteed that the file exists. link_to_latest = os.path.join(os.path.dirname(self.run_info_dir), "latest") relative_symlink(self.run_info_dir, link_to_latest) # Time spent in a workunit, including its children. self.cumulative_timings = AggregatedTimings( os.path.join(self.run_info_dir, "cumulative_timings")) # Time spent in a workunit, not including its children. self.self_timings = AggregatedTimings( os.path.join(self.run_info_dir, "self_timings")) # pantsd stats. self._pantsd_metrics: Dict[str, int] = dict() self._all_options = all_options self.report = Report() self.report.open() # And create the workunit. self._main_root_workunit = WorkUnit(run_info_dir=self.run_info_dir, parent=None, name=RunTracker.DEFAULT_ROOT_NAME, cmd=None) self.register_thread(self._main_root_workunit) # Set the true start time in the case of e.g. the daemon. self._main_root_workunit.start(run_start_time) self.report.start_workunit(self._main_root_workunit) goal_names: Tuple[str, ...] = tuple(all_options.goals) self._v2_goal_rule_names = goal_names self.run_logs_file = Path(self.run_info_dir, "logs") self.native.set_per_run_log_path(str(self.run_logs_file))
def get_critical_path_timings(self): """Get the cumulative timings of each goal and all of the goals it (transitively) depended on.""" setup_workunit = WorkUnitLabel.SETUP.lower() transitive_dependencies = dict() for goal_info in self._sorted_goal_infos: deps = transitive_dependencies.setdefault(goal_info.goal.name, set()) for dep in goal_info.goal_dependencies: deps.add(dep.name) deps.update(transitive_dependencies.get(dep.name)) # Add setup workunit as a dep manually, as its unaccounted for, otherwise. deps.add(setup_workunit) raw_timings = dict() for entry in self.cumulative_timings.get_all(): raw_timings[entry["label"]] = entry["timing"] critical_path_timings = AggregatedTimings() def add_to_timings(goal, dep): tracking_label = get_label(goal) timing_label = get_label(dep) critical_path_timings.add_timing(tracking_label, raw_timings.get(timing_label, 0.0)) def get_label(dep): return f"{RunTracker.DEFAULT_ROOT_NAME}:{dep}" # Add setup workunit to critical_path_timings manually, as its unaccounted for, otherwise. add_to_timings(setup_workunit, setup_workunit) for goal, deps in transitive_dependencies.items(): add_to_timings(goal, goal) for dep in deps: add_to_timings(goal, dep) return critical_path_timings
def initialize(self, all_options): """Create run_info and relevant directories, and return the run id. Must be called before `start`. """ if self.run_info: raise AssertionError( 'RunTracker.initialize must not be called multiple times.') # Initialize the run. info_dir = os.path.join(self.get_options().pants_workdir, self.options_scope) self.run_info_dir = os.path.join(info_dir, self.run_id) self.run_info = RunInfo(os.path.join(self.run_info_dir, 'info')) self.run_info.add_basic_info(self.run_id, self._run_timestamp) self.run_info.add_info('cmd_line', self._cmd_line) if self.get_options().parent_build_id: self.run_info.add_info('parent_build_id', self.get_options().parent_build_id) # Create a 'latest' symlink, after we add_infos, so we're guaranteed that the file exists. link_to_latest = os.path.join(os.path.dirname(self.run_info_dir), 'latest') relative_symlink(self.run_info_dir, link_to_latest) # Time spent in a workunit, including its children. self.cumulative_timings = AggregatedTimings( os.path.join(self.run_info_dir, 'cumulative_timings')) # Time spent in a workunit, not including its children. self.self_timings = AggregatedTimings( os.path.join(self.run_info_dir, 'self_timings')) # Hit/miss stats for the artifact cache. self.artifact_cache_stats = ArtifactCacheStats( os.path.join(self.run_info_dir, 'artifact_cache_stats')) # Daemon stats. self.pantsd_stats = PantsDaemonStats() self._all_options = all_options return (self.run_id, self.run_uuid)
def get_critical_path_timings(self): """ Get the cumulative timings of each goal and all of the goals it (transitively) depended on. """ transitive_dependencies = dict() for goal_info in self._sorted_goal_infos: deps = transitive_dependencies.setdefault(goal_info.goal.name, set()) for dep in goal_info.goal_dependencies: deps.add(dep.name) deps.update(transitive_dependencies.get(dep.name)) raw_timings = dict() for entry in self.cumulative_timings.get_all(): raw_timings[entry["label"]] = entry["timing"] timings = AggregatedTimings() for goal, deps in transitive_dependencies.items(): label = "{}:{}".format(RunTracker.DEFAULT_ROOT_NAME, goal) timings.add_timing(label, raw_timings.get(label, 0.0)) for dep in deps: dep_label = "{}:{}".format(RunTracker.DEFAULT_ROOT_NAME, dep) timings.add_timing(label, raw_timings.get(dep_label, 0.0)) return timings
def __init__(self, info_dir, stats_upload_url=None, stats_upload_timeout=2, num_foreground_workers=8, num_background_workers=8): self.run_timestamp = time.time( ) # A double, so we get subsecond precision for ids. cmd_line = ' '.join(['./pants'] + sys.argv[1:]) # run_id is safe for use in paths. millis = (self.run_timestamp * 1000) % 1000 run_id = 'pants_run_%s_%d' % \ (time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime(self.run_timestamp)), millis) self.run_info_dir = os.path.join(info_dir, run_id) self.run_info = RunInfo(os.path.join(self.run_info_dir, 'info')) self.run_info.add_basic_info(run_id, self.run_timestamp) self.run_info.add_info('cmd_line', cmd_line) self.stats_url = stats_upload_url self.stats_timeout = stats_upload_timeout # Create a 'latest' symlink, after we add_infos, so we're guaranteed that the file exists. link_to_latest = os.path.join(os.path.dirname(self.run_info_dir), 'latest') try: if os.path.lexists(link_to_latest): os.unlink(link_to_latest) os.symlink(self.run_info_dir, link_to_latest) except OSError as e: # Another run may beat us to deletion or creation. if not (e.errno == errno.EEXIST or e.errno == errno.ENOENT): raise # Time spent in a workunit, including its children. self.cumulative_timings = AggregatedTimings( os.path.join(self.run_info_dir, 'cumulative_timings')) # Time spent in a workunit, not including its children. self.self_timings = AggregatedTimings( os.path.join(self.run_info_dir, 'self_timings')) # Hit/miss stats for the artifact cache. self.artifact_cache_stats = \ ArtifactCacheStats(os.path.join(self.run_info_dir, 'artifact_cache_stats')) # Number of threads for foreground work. self._num_foreground_workers = num_foreground_workers # Number of threads for background work. self._num_background_workers = num_background_workers # We report to this Report. self.report = None # self._threadlocal.current_workunit contains the current workunit for the calling thread. # Note that multiple threads may share a name (e.g., all the threads in a pool). self._threadlocal = threading.local() # For main thread work. Created on start(). self._main_root_workunit = None # For concurrent foreground work. Created lazily if needed. # Associated with the main thread's root workunit. self._foreground_worker_pool = None # For background work. Created lazily if needed. self._background_worker_pool = None self._background_root_workunit = None # Trigger subproc pool init while our memory image is still clean (see SubprocPool docstring) SubprocPool.foreground() self._aborted = False