def do(self): self._reporter.start_cache_refresh() with self._reporter.tag_timer_context('refresh conduit cache'): for key in self._conduits: conduit = self._conduits[key] abdt_tryloop.critical_tryloop( conduit.refresh_cache_on_cycle, abdt_errident.CONDUIT_REFRESH, conduit.describe()) with self._reporter.tag_timer_context('refresh git watcher'): abdt_tryloop.critical_tryloop( self._url_watcher.refresh, abdt_errident.GIT_SNOOP, '') self._reporter.finish_cache_refresh() return True
def do(self): self._reporter.start_cache_refresh() with self._reporter.tag_timer_context('refresh conduit cache'): for key in self._conduits: conduit = self._conduits[key] abdt_tryloop.critical_tryloop( conduit.refresh_cache_on_cycle, "conduit-refresh", conduit.describe()) with self._reporter.tag_timer_context('refresh git watcher'): abdt_tryloop.critical_tryloop( self._url_watcher.refresh, 'git-snoop', '') self._reporter.finish_cache_refresh() return True
def do(repo_configs, sys_admin_emails, sleep_secs, is_no_loop, external_report_command, mail_sender, max_workers, overrun_secs): conduit_manager = _ConduitManager() fs_accessor = abdt_fs.make_default_accessor() url_watcher_wrapper = phlurl_watcher.FileCacheWatcherWrapper( fs_accessor.layout.urlwatcher_cache_path) # decide max workers based on number of CPUs if no value is specified if max_workers == 0: max_workers = determine_max_workers_default() repo_list = [] for name, config in repo_configs: repo_list.append( _ArcydManagedRepository(name, config, conduit_manager, url_watcher_wrapper, sys_admin_emails, mail_sender)) # if we always overrun half our workers then the loop is sustainable, if we # overrun more than that then we'll be lagging too far behind. In the event # that we only have one worker then we can't overrun any. max_overrun_workers = max_workers // 2 pool = phlmp_cyclingpool.CyclingPool(repo_list, max_workers, max_overrun_workers) cycle_timer = phlsys_timer.Timer() cycle_timer.start() exit_code = None while exit_code is None: # This timer needs to be separate from the cycle timer. The cycle timer # must be reset every time it is reported. The sleep timer makes sure # that each run of the loop takes a minimum amount of time. sleep_timer = phlsys_timer.Timer() sleep_timer.start() # refresh git snoops with abdt_logging.remote_io_read_event_context('refresh-git-snoop', ''): abdt_tryloop.critical_tryloop(url_watcher_wrapper.watcher.refresh, abdt_errident.GIT_SNOOP, '') with abdt_logging.remote_io_read_event_context('refresh-conduit', ''): conduit_manager.refresh_conduits() with abdt_logging.misc_operation_event_context( 'process-repos', '{} workers, {} repos'.format(max_workers, len(repo_list))): if max_workers > 1: for i, res in pool.cycle_results(overrun_secs=overrun_secs): repo = repo_list[i] repo.merge_from_worker(res) else: for r in repo_list: r() # important to do this before stopping arcyd and as soon as possible # after doing fetches url_watcher_wrapper.save() # report cycle stats report = { "cycle_time_secs": cycle_timer.restart(), "overrun_jobs": pool.num_active_jobs, } _LOGGER.debug("cycle-stats: {}".format(report)) if external_report_command: report_json = json.dumps(report) full_path = os.path.abspath(external_report_command) with abdt_logging.misc_operation_event_context( 'external-report-command', external_report_command): try: phlsys_subprocess.run(full_path, stdin=report_json) except phlsys_subprocess.CalledProcessError as e: _LOGGER.error( "External command: {} failed with exception: " "{}.".format(external_report_command, type(e).__name__)) _LOGGER.error( "VERBOSE MESSAGE: CycleReportJson:{}".format(e)) if is_no_loop: exit_code = abdi_processexitcodes.ExitCodes.ec_exit elif os.path.isfile(fs_accessor.layout.killfile): exit_code = abdi_processexitcodes.ExitCodes.ec_exit if phlsys_fs.read_text_file(fs_accessor.layout.killfile): _LOGGER.info("Killfile observed, reason given: {}".format( phlsys_fs.read_text_file(fs_accessor.layout.killfile))) else: _LOGGER.info("Killfile observed, arcyd will stop") os.remove(fs_accessor.layout.killfile) elif os.path.isfile(fs_accessor.layout.reloadfile): _LOGGER.info("Reloadfile observed, arcyd will reload") exit_code = abdi_processexitcodes.ExitCodes.ec_reload os.remove(fs_accessor.layout.reloadfile) # sleep to pad out the cycle secs_to_sleep = float(sleep_secs) - float(sleep_timer.duration) if secs_to_sleep > 0 and exit_code is None: with abdt_logging.misc_operation_event_context( 'sleep', secs_to_sleep): time.sleep(secs_to_sleep) # finish any jobs that overran for i, res in pool.finish_results(): repo = repo_list[i] repo.merge_from_worker(res) # important to do this before stopping arcyd and as soon as # possible after doing fetches url_watcher_wrapper.save() return exit_code
def refresh_conduits(self): for conduit, cache in self._conduits_caches.itervalues(): abdt_tryloop.critical_tryloop(cache.refresh_active_reviews, abdt_errident.CONDUIT_REFRESH, conduit.describe())
def do( repo_configs, sys_admin_emails, kill_file, sleep_secs, is_no_loop, external_report_command, mail_sender, max_workers, overrun_secs): conduit_manager = _ConduitManager() fs_accessor = abdt_fs.make_default_accessor() url_watcher_wrapper = phlurl_watcher.FileCacheWatcherWrapper( fs_accessor.layout.urlwatcher_cache_path) finished = False # decide max workers based on number of CPUs if no value is specified if max_workers is None: try: # use the same default as multiprocessing.Pool max_workers = multiprocessing.cpu_count() _LOGGER.debug( "max_workers unspecified, defaulted to cpu_count: {}".format( max_workers)) except NotImplementedError: _LOGGER.warning( "multiprocessing.cpu_count() not supported, disabling " "multiprocessing. Specify max workers explicitly to enable.") max_workers = 0 repo_list = [] for name, config in repo_configs: repo_list.append( _ArcydManagedRepository( name, config, conduit_manager, url_watcher_wrapper, sys_admin_emails, mail_sender)) # if we always overrun half our workers then the loop is sustainable, if we # overrun more than that then we'll be lagging too far behind. In the event # that we only have one worker then we can't overrun any. max_overrun_workers = max_workers // 2 pool = phlmp_cyclingpool.CyclingPool( repo_list, max_workers, max_overrun_workers) cycle_timer = phlsys_timer.Timer() cycle_timer.start() while not finished: # This timer needs to be separate from the cycle timer. The cycle timer # must be reset every time it is reported. The sleep timer makes sure # that each run of the loop takes a minimum amount of time. sleep_timer = phlsys_timer.Timer() sleep_timer.start() # refresh git snoops abdt_tryloop.critical_tryloop( url_watcher_wrapper.watcher.refresh, abdt_errident.GIT_SNOOP, '') conduit_manager.refresh_conduits() if max_workers: for i, res in pool.cycle_results(overrun_secs=overrun_secs): repo = repo_list[i] repo.merge_from_worker(res) else: for r in repo_list: r() # important to do this before stopping arcyd and as soon as possible # after doing fetches url_watcher_wrapper.save() # report cycle stats if external_report_command: report = { "cycle_time_secs": cycle_timer.restart(), "overrun_jobs": pool.num_active_jobs, } report_json = json.dumps(report) full_path = os.path.abspath(external_report_command) try: phlsys_subprocess.run(full_path, stdin=report_json) except phlsys_subprocess.CalledProcessError as e: _LOGGER.error("CycleReportJson: {}".format(e)) # look for killfile if os.path.isfile(kill_file): # finish any jobs that overran for i, res in pool.finish_results(): repo = repo_list[i] repo.merge_from_worker(res) # important to do this before stopping arcyd and as soon as # possible after doing fetches url_watcher_wrapper.save() os.remove(kill_file) finished = True break if is_no_loop: finished = True break # sleep to pad out the cycle secs_to_sleep = float(sleep_secs) - float(sleep_timer.duration) if secs_to_sleep > 0: time.sleep(secs_to_sleep)
def refresh_conduits(self): for conduit, cache in self._conduits_caches.itervalues(): abdt_tryloop.critical_tryloop( cache.refresh_active_reviews, abdt_errident.CONDUIT_REFRESH, conduit.describe())
def do( repo_configs, sys_admin_emails, sleep_secs, is_no_loop, external_report_command, mail_sender, max_workers, overrun_secs): conduit_manager = _ConduitManager() fs_accessor = abdt_fs.make_default_accessor() url_watcher_wrapper = phlurl_watcher.FileCacheWatcherWrapper( fs_accessor.layout.urlwatcher_cache_path) # decide max workers based on number of CPUs if no value is specified if max_workers == 0: max_workers = determine_max_workers_default() repo_list = [] for name, config in repo_configs: repo_list.append( _ArcydManagedRepository( name, config, conduit_manager, url_watcher_wrapper, sys_admin_emails, mail_sender)) # if we always overrun half our workers then the loop is sustainable, if we # overrun more than that then we'll be lagging too far behind. In the event # that we only have one worker then we can't overrun any. max_overrun_workers = max_workers // 2 pool = phlmp_cyclingpool.CyclingPool( repo_list, max_workers, max_overrun_workers) cycle_timer = phlsys_timer.Timer() cycle_timer.start() exit_code = None while exit_code is None: # This timer needs to be separate from the cycle timer. The cycle timer # must be reset every time it is reported. The sleep timer makes sure # that each run of the loop takes a minimum amount of time. sleep_timer = phlsys_timer.Timer() sleep_timer.start() # refresh git snoops with abdt_logging.remote_io_read_event_context( 'refresh-git-snoop', ''): abdt_tryloop.critical_tryloop( url_watcher_wrapper.watcher.refresh, abdt_errident.GIT_SNOOP, '') with abdt_logging.remote_io_read_event_context('refresh-conduit', ''): conduit_manager.refresh_conduits() with abdt_logging.misc_operation_event_context( 'process-repos', '{} workers, {} repos'.format(max_workers, len(repo_list))): if max_workers > 1: for i, res in pool.cycle_results(overrun_secs=overrun_secs): repo = repo_list[i] repo.merge_from_worker(res) else: for r in repo_list: r() # important to do this before stopping arcyd and as soon as possible # after doing fetches url_watcher_wrapper.save() # report cycle stats report = { "cycle_time_secs": cycle_timer.restart(), "overrun_jobs": pool.num_active_jobs, } _LOGGER.debug("cycle-stats: {}".format(report)) if external_report_command: report_json = json.dumps(report) full_path = os.path.abspath(external_report_command) with abdt_logging.misc_operation_event_context( 'external-report-command', external_report_command): try: phlsys_subprocess.run(full_path, stdin=report_json) except phlsys_subprocess.CalledProcessError as e: _LOGGER.error( "External command: {} failed with exception: " "{}.".format( external_report_command, type(e).__name__)) _LOGGER.error("VERBOSE MESSAGE: CycleReportJson:{}".format( e)) if is_no_loop: exit_code = abdi_processexitcodes.ExitCodes.ec_exit elif os.path.isfile(fs_accessor.layout.killfile): exit_code = abdi_processexitcodes.ExitCodes.ec_exit if phlsys_fs.read_text_file(fs_accessor.layout.killfile): _LOGGER.info("Killfile observed, reason given: {}".format( phlsys_fs.read_text_file(fs_accessor.layout.killfile))) else: _LOGGER.info("Killfile observed, arcyd will stop") os.remove(fs_accessor.layout.killfile) elif os.path.isfile(fs_accessor.layout.reloadfile): _LOGGER.info("Reloadfile observed, arcyd will reload") exit_code = abdi_processexitcodes.ExitCodes.ec_reload os.remove(fs_accessor.layout.reloadfile) # sleep to pad out the cycle secs_to_sleep = float(sleep_secs) - float(sleep_timer.duration) if secs_to_sleep > 0 and exit_code is None: with abdt_logging.misc_operation_event_context( 'sleep', secs_to_sleep): time.sleep(secs_to_sleep) # finish any jobs that overran for i, res in pool.finish_results(): repo = repo_list[i] repo.merge_from_worker(res) # important to do this before stopping arcyd and as soon as # possible after doing fetches url_watcher_wrapper.save() return exit_code