def __init__(self, *args, **kwargs): """ :API: public """ super(RunTracker, self).__init__(*args, **kwargs) self._run_timestamp = time.time() self._cmd_line = ' '.join(['pants'] + sys.argv[1:]) # Initialized in `initialize()`. self.run_info_dir = None self.run_info = None self.cumulative_timings = None self.self_timings = None self.artifact_cache_stats = None self.pantsd_stats = None # Initialized in `start()`. self.report = None self._main_root_workunit = None # A lock to ensure that adding to stats at the end of a workunit # operates thread-safely. self._stats_lock = threading.Lock() # Log of success/failure/aborted for each workunit. self.outcomes = {} # Number of threads for foreground work. self._num_foreground_workers = self.get_options().num_foreground_workers # Number of threads for background work. self._num_background_workers = self.get_options().num_background_workers # self._threadlocal.current_workunit contains the current workunit for the calling thread. # Note that multiple threads may share a name (e.g., all the threads in a pool). self._threadlocal = threading.local() # For background work. Created lazily if needed. self._background_worker_pool = None self._background_root_workunit = None # Trigger subproc pool init while our memory image is still clean (see SubprocPool docstring). SubprocPool.set_num_processes(self._num_foreground_workers) SubprocPool.foreground() self._aborted = False # Data will be organized first by target and then scope. # Eg: # { # 'target/address:name': { # 'running_scope': { # 'run_duration': 356.09 # }, # 'GLOBAL': { # 'target_type': 'pants.test' # } # } # } self._target_to_data = {}
def __init__(self, *args, **kwargs): """ :API: public """ super(RunTracker, self).__init__(*args, **kwargs) self._run_timestamp = time.time() self._cmd_line = ' '.join(['pants'] + sys.argv[1:]) # Initialized in `initialize()`. self.run_info_dir = None self.run_info = None self.cumulative_timings = None self.self_timings = None self.artifact_cache_stats = None # Initialized in `start()`. self.report = None self._main_root_workunit = None # A lock to ensure that adding to stats at the end of a workunit # operates thread-safely. self._stats_lock = threading.Lock() # Log of success/failure/aborted for each workunit. self.outcomes = {} # Number of threads for foreground work. self._num_foreground_workers = self.get_options().num_foreground_workers # Number of threads for background work. self._num_background_workers = self.get_options().num_background_workers # self._threadlocal.current_workunit contains the current workunit for the calling thread. # Note that multiple threads may share a name (e.g., all the threads in a pool). self._threadlocal = threading.local() # For background work. Created lazily if needed. self._background_worker_pool = None self._background_root_workunit = None # Trigger subproc pool init while our memory image is still clean (see SubprocPool docstring). SubprocPool.set_num_processes(self._num_foreground_workers) SubprocPool.foreground() self._aborted = False # Data will be organized first by target and then scope. # Eg: # { # 'target/address:name': { # 'running_scope': { # 'run_duration': 356.09 # }, # 'GLOBAL': { # 'target_type': 'pants.test' # } # } # } self._target_to_data = {}
def __init__(self, *args, **kwargs): """ :API: public """ super().__init__(*args, **kwargs) self._run_timestamp = time.time() self._cmd_line = " ".join(["pants"] + sys.argv[1:]) self._v2_goal_rule_names: Tuple[str, ...] = tuple() self.run_uuid = uuid.uuid4().hex # Select a globally unique ID for the run, that sorts by time. millis = int((self._run_timestamp * 1000) % 1000) # run_uuid is used as a part of run_id and also as a trace_id for Zipkin tracing str_time = time.strftime("%Y_%m_%d_%H_%M_%S", time.localtime(self._run_timestamp)) self.run_id = f"pants_run_{str_time}_{millis}_{self.run_uuid}" # Initialized in `initialize()`. self.run_info_dir = None self.run_info = None self.cumulative_timings = None self.self_timings = None # Initialized in `start()`. self.report = None self._main_root_workunit = None self._all_options = None # A lock to ensure that adding to stats at the end of a workunit # operates thread-safely. self._stats_lock = threading.Lock() # Log of success/failure/aborted for each workunit. self.outcomes = {} # Number of threads for foreground work. self._num_foreground_workers = self.options.num_foreground_workers # Number of threads for background work. self._num_background_workers = self.options.num_background_workers # self._threadlocal.current_workunit contains the current workunit for the calling thread. # Note that multiple threads may share a name (e.g., all the threads in a pool). self._threadlocal = threading.local() # For background work. Created lazily if needed. self._background_root_workunit = None # Trigger subproc pool init while our memory image is still clean (see SubprocPool docstring). SubprocPool.set_num_processes(self._num_foreground_workers) SubprocPool.foreground() self._aborted = False self._end_memoized_result: Optional[ExitCode] = None self.native = Native() self.run_logs_file: Optional[Path] = None
def __init__(self, *args, **kwargs): super(RunTracker, self).__init__(*args, **kwargs) run_timestamp = time.time() cmd_line = ' '.join(['pants'] + sys.argv[1:]) # run_id is safe for use in paths. millis = int((run_timestamp * 1000) % 1000) run_id = 'pants_run_{}_{}_{}'.format( time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime(run_timestamp)), millis, uuid.uuid4().hex) info_dir = os.path.join(self.get_options().pants_workdir, self.options_scope) self.run_info_dir = os.path.join(info_dir, run_id) self.run_info = RunInfo(os.path.join(self.run_info_dir, 'info')) self.run_info.add_basic_info(run_id, run_timestamp) self.run_info.add_info('cmd_line', cmd_line) # Create a 'latest' symlink, after we add_infos, so we're guaranteed that the file exists. link_to_latest = os.path.join(os.path.dirname(self.run_info_dir), 'latest') relative_symlink(self.run_info_dir, link_to_latest) # Time spent in a workunit, including its children. self.cumulative_timings = AggregatedTimings(os.path.join(self.run_info_dir, 'cumulative_timings')) # Time spent in a workunit, not including its children. self.self_timings = AggregatedTimings(os.path.join(self.run_info_dir, 'self_timings')) # Hit/miss stats for the artifact cache. self.artifact_cache_stats = \ ArtifactCacheStats(os.path.join(self.run_info_dir, 'artifact_cache_stats')) # Number of threads for foreground work. self._num_foreground_workers = self.get_options().num_foreground_workers # Number of threads for background work. self._num_background_workers = self.get_options().num_background_workers # We report to this Report. self.report = None # self._threadlocal.current_workunit contains the current workunit for the calling thread. # Note that multiple threads may share a name (e.g., all the threads in a pool). self._threadlocal = threading.local() # For main thread work. Created on start(). self._main_root_workunit = None # For background work. Created lazily if needed. self._background_worker_pool = None self._background_root_workunit = None # Trigger subproc pool init while our memory image is still clean (see SubprocPool docstring). SubprocPool.set_num_processes(self._num_foreground_workers) SubprocPool.foreground() self._aborted = False
def subproc_map(self, f, items): """Map function `f` over `items` in subprocesses and return the result. :param f: A multiproc-friendly (importable) work function. :param args: A iterable of pickleable arguments to f. """ try: # Pool.map (and async_map().get() w/o timeout) can miss SIGINT. # See: http://stackoverflow.com/a/1408476, http://bugs.python.org/issue8844 # Instead, we map_async(...), wait *with a timeout* until ready, then .get() # NB: in 2.x, wait() with timeout wakes up often to check, burning CPU. Oh well. res = SubprocPool.foreground().map_async(f, items) while not res.ready(): res.wait(60) # Repeatedly wait for up to a minute. if not res.ready(): self.log.debug('subproc_map result still not ready...') return res.get() except KeyboardInterrupt: SubprocPool.shutdown(True) raise
def subproc_map(self, f, items): """Map function `f` over `items` in subprocesses and return the result. :param f: A multiproc-friendly (importable) work function. :param items: A iterable of pickleable arguments to f. """ try: # Pool.map (and async_map().get() w/o timeout) can miss SIGINT. # See: http://stackoverflow.com/a/1408476, http://bugs.python.org/issue8844 # Instead, we map_async(...), wait *with a timeout* until ready, then .get() # NB: in 2.x, wait() with timeout wakes up often to check, burning CPU. Oh well. res = SubprocPool.foreground().map_async(f, items) while not res.ready(): res.wait(60) # Repeatedly wait for up to a minute. if not res.ready(): self.log.debug('subproc_map result still not ready...') return res.get() except KeyboardInterrupt: SubprocPool.shutdown(True) raise
def __init__(self, *args, **kwargs): """ :API: public """ super().__init__(*args, **kwargs) self._run_timestamp = time.time() self._cmd_line = ' '.join(['pants'] + sys.argv[1:]) self._sorted_goal_infos = tuple() self._v2_goal_rule_names = tuple() self.run_uuid = uuid.uuid4().hex # Select a globally unique ID for the run, that sorts by time. millis = int((self._run_timestamp * 1000) % 1000) # run_uuid is used as a part of run_id and also as a trace_id for Zipkin tracing str_time = time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime(self._run_timestamp)) self.run_id = f'pants_run_{str_time}_{millis}_{self.run_uuid}' # Initialized in `initialize()`. self.run_info_dir = None self.run_info = None self.cumulative_timings = None self.self_timings = None self.artifact_cache_stats = None self.pantsd_stats = None # Initialized in `start()`. self.report = None self.json_reporter = None self._main_root_workunit = None self._all_options = None # A lock to ensure that adding to stats at the end of a workunit # operates thread-safely. self._stats_lock = threading.Lock() # Log of success/failure/aborted for each workunit. self.outcomes = {} # Number of threads for foreground work. self._num_foreground_workers = self.get_options( ).num_foreground_workers # Number of threads for background work. self._num_background_workers = self.get_options( ).num_background_workers # self._threadlocal.current_workunit contains the current workunit for the calling thread. # Note that multiple threads may share a name (e.g., all the threads in a pool). self._threadlocal = threading.local() # A logger facade that logs into this RunTracker. self._logger = RunTrackerLogger(self) # For background work. Created lazily if needed. self._background_worker_pool = None self._background_root_workunit = None # Trigger subproc pool init while our memory image is still clean (see SubprocPool docstring). SubprocPool.set_num_processes(self._num_foreground_workers) SubprocPool.foreground() self._aborted = False # Data will be organized first by target and then scope. # Eg: # { # 'target/address:name': { # 'running_scope': { # 'run_duration': 356.09 # }, # 'GLOBAL': { # 'target_type': 'pants.test' # } # } # } self._target_to_data = {} self._end_memoized_result = None
def __init__(self, info_dir, stats_upload_url=None, stats_upload_timeout=2, num_foreground_workers=8, num_background_workers=8): self.run_timestamp = time.time() # A double, so we get subsecond precision for ids. cmd_line = ' '.join(['./pants'] + sys.argv[1:]) # run_id is safe for use in paths. millis = (self.run_timestamp * 1000) % 1000 run_id = 'pants_run_%s_%d' % \ (time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime(self.run_timestamp)), millis) self.run_info_dir = os.path.join(info_dir, run_id) self.run_info = RunInfo(os.path.join(self.run_info_dir, 'info')) self.run_info.add_basic_info(run_id, self.run_timestamp) self.run_info.add_info('cmd_line', cmd_line) self.stats_url = stats_upload_url self.stats_timeout = stats_upload_timeout # Create a 'latest' symlink, after we add_infos, so we're guaranteed that the file exists. link_to_latest = os.path.join(os.path.dirname(self.run_info_dir), 'latest') try: if os.path.lexists(link_to_latest): os.unlink(link_to_latest) os.symlink(self.run_info_dir, link_to_latest) except OSError as e: # Another run may beat us to deletion or creation. if not (e.errno == errno.EEXIST or e.errno == errno.ENOENT): raise # Time spent in a workunit, including its children. self.cumulative_timings = AggregatedTimings(os.path.join(self.run_info_dir, 'cumulative_timings')) # Time spent in a workunit, not including its children. self.self_timings = AggregatedTimings(os.path.join(self.run_info_dir, 'self_timings')) # Hit/miss stats for the artifact cache. self.artifact_cache_stats = \ ArtifactCacheStats(os.path.join(self.run_info_dir, 'artifact_cache_stats')) # Number of threads for foreground work. self._num_foreground_workers = num_foreground_workers # Number of threads for background work. self._num_background_workers = num_background_workers # We report to this Report. self.report = None # self._threadlocal.current_workunit contains the current workunit for the calling thread. # Note that multiple threads may share a name (e.g., all the threads in a pool). self._threadlocal = threading.local() # For main thread work. Created on start(). self._main_root_workunit = None # For concurrent foreground work. Created lazily if needed. # Associated with the main thread's root workunit. self._foreground_worker_pool = None # For background work. Created lazily if needed. self._background_worker_pool = None self._background_root_workunit = None # Trigger subproc pool init while our memory image is still clean (see SubprocPool docstring) SubprocPool.foreground() self._aborted = False
def __init__(self, info_dir, stats_upload_url=None, stats_upload_timeout=2, num_foreground_workers=8, num_background_workers=8): self.run_timestamp = time.time( ) # A double, so we get subsecond precision for ids. cmd_line = ' '.join(['./pants'] + sys.argv[1:]) # run_id is safe for use in paths. millis = (self.run_timestamp * 1000) % 1000 run_id = 'pants_run_%s_%d' % \ (time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime(self.run_timestamp)), millis) self.run_info_dir = os.path.join(info_dir, run_id) self.run_info = RunInfo(os.path.join(self.run_info_dir, 'info')) self.run_info.add_basic_info(run_id, self.run_timestamp) self.run_info.add_info('cmd_line', cmd_line) self.stats_url = stats_upload_url self.stats_timeout = stats_upload_timeout # Create a 'latest' symlink, after we add_infos, so we're guaranteed that the file exists. link_to_latest = os.path.join(os.path.dirname(self.run_info_dir), 'latest') try: if os.path.lexists(link_to_latest): os.unlink(link_to_latest) os.symlink(self.run_info_dir, link_to_latest) except OSError as e: # Another run may beat us to deletion or creation. if not (e.errno == errno.EEXIST or e.errno == errno.ENOENT): raise # Time spent in a workunit, including its children. self.cumulative_timings = AggregatedTimings( os.path.join(self.run_info_dir, 'cumulative_timings')) # Time spent in a workunit, not including its children. self.self_timings = AggregatedTimings( os.path.join(self.run_info_dir, 'self_timings')) # Hit/miss stats for the artifact cache. self.artifact_cache_stats = \ ArtifactCacheStats(os.path.join(self.run_info_dir, 'artifact_cache_stats')) # Number of threads for foreground work. self._num_foreground_workers = num_foreground_workers # Number of threads for background work. self._num_background_workers = num_background_workers # We report to this Report. self.report = None # self._threadlocal.current_workunit contains the current workunit for the calling thread. # Note that multiple threads may share a name (e.g., all the threads in a pool). self._threadlocal = threading.local() # For main thread work. Created on start(). self._main_root_workunit = None # For concurrent foreground work. Created lazily if needed. # Associated with the main thread's root workunit. self._foreground_worker_pool = None # For background work. Created lazily if needed. self._background_worker_pool = None self._background_root_workunit = None # Trigger subproc pool init while our memory image is still clean (see SubprocPool docstring) SubprocPool.foreground() self._aborted = False