def __init__(self): LOG.info('Health Manager starting.') self.seq = 0 self.amp_repo = repo.AmphoraRepository() self.listener_repo = repo.ListenerRepository() self.amp_health_repo = repo.AmphoraHealthRepository() self.lb_repo = repo.LoadBalancerRepository() self.health_executor = futurist.ThreadPoolExecutor( max_workers=CONF.health_manager.health_update_threads) self.stats_executor = futurist.ThreadPoolExecutor( max_workers=CONF.health_manager.stats_update_threads) self.bigips = [bigip for bigip in self.initialize_bigips()] # Cache reachability of every bigip self.bigip_status = {bigip.hostname: False for bigip in self.bigips} self._active_bigip = None self._last_failover_check = 0 self._last_cleanup_check = 0 # Create RPC Client topic = cfg.CONF.oslo_messaging.topic self.target = messaging.Target( namespace=o_const.RPC_NAMESPACE_CONTROLLER_AGENT, topic=topic, version="1.0", fanout=False) self.client = rpc.get_client(self.target) if cfg.CONF.f5_agent.prometheus: prometheus_port = CONF.f5_agent.prometheus_port LOG.info('Starting Prometheus HTTP server on port {}'.format( prometheus_port)) prometheus.start_http_server(prometheus_port)
def _do_collection(self, metrics, timestamp): def _get_result(metric): try: return self._collect(metric, timestamp) except collector.NoDataCollected: LOG.info(self._log_prefix + 'No data collected ' 'for metric {metric} at timestamp {ts}'.format( metric=metric, ts=timestamp)) return metric, None except Exception as e: LOG.exception( self._log_prefix + 'Error while collecting' ' metric {metric} at timestamp {ts}: {e}. Exiting.'.format( metric=metric, ts=timestamp, e=e)) # FIXME(peschk_l): here we just exit, and the # collection will be retried during the next collect # cycle. In the future, we should implement a retrying # system in workers sys.exit(1) with futurist.ThreadPoolExecutor( max_workers=CONF.orchestrator.max_threads) as tpool: futs = [tpool.submit(_get_result, metric) for metric in metrics] LOG.debug(self._log_prefix + 'Collecting {} metrics.'.format(len(metrics))) results = [r.result() for r in waiters.wait_for_all(futs).done] LOG.debug(self._log_prefix + 'Collecting {} metrics took {}s ' 'total, with {}s average'.format( tpool.statistics.executed, tpool.statistics.runtime, tpool.statistics.average_runtime)) return dict(filter(lambda x: x[1] is not None, results))
def test_execution_concurrency_scale_up(self): self.await_runtime_available(self.runtime_id) self._create_function(name='test_python_sleep.py') def _create_execution(): resp, body = self.client.create_execution(self.function_id) return resp, body futs = [] with futurist.ThreadPoolExecutor(max_workers=10) as executor: for _ in range(6): fut = executor.submit(_create_execution) futs.append(fut) for f in futures.as_completed(futs): # Wait until we get the response resp, body = f.result() self.assertEqual(201, resp.status) self.addCleanup(self.client.delete_resource, 'executions', body['id'], ignore_notfound=True) self.assertEqual('success', body['status']) resp, body = self.admin_client.get_function_workers(self.function_id) self.assertEqual(200, resp.status) self.assertEqual(2, len(body['workers']))
def get_pool(size): import futurist if eventletutils.is_monkey_patched('thread'): return futurist.GreenThreadPoolExecutor(size) return futurist.ThreadPoolExecutor(size)
def __init__(self, exchange, topic, tasks, executor=None, threads_count=None, url=None, transport=None, transport_options=None, retry_options=None, advertiser_factory=None): self._topic = topic self._executor = executor self._owns_executor = False if self._executor is None: self._executor = futurist.ThreadPoolExecutor( max_workers=threads_count) self._owns_executor = True self._endpoints = tuple(self._derive_endpoints(tasks)) self._exchange = exchange self._server = server.Server(topic, exchange, self._executor, self._endpoints, url=url, transport=transport, transport_options=transport_options, retry_options=retry_options) self._advertiser = None self._advertiser_lock = threading.Lock() if advertiser_factory is not None: if not six.callable(advertiser_factory): raise ValueError("Provided factory used to build worker" " advertisers must be callable") self._advertiser_factory = advertiser_factory else: self._advertiser_factory = None
def __init__(self, exchange, topic, tasks, executor=None, threads_count=None, url=None, transport=None, transport_options=None, retry_options=None): self._topic = topic self._executor = executor self._owns_executor = False if self._executor is None: self._executor = futurist.ThreadPoolExecutor( max_workers=threads_count) self._owns_executor = True self._endpoints = self._derive_endpoints(tasks) self._exchange = exchange self._server = server.Server(topic, exchange, self._executor, self._endpoints, url=url, transport=transport, transport_options=transport_options, retry_options=retry_options)
def execute_flow(flow): """ Create all necessary prerequisites like task database and thread pool and execute TaskFlow flow. :param flow: TaskFlow flow instance """ backend = backends.fetch({ 'connection': 'sqlite:///' + TASK_DATABASE_FILE, 'isolation_level': 'SERIALIZABLE' }) executor = futurist.ThreadPoolExecutor(max_workers=MAX_WORKERS) conn = backend.get_connection() logbook, flow_detail = _ensure_db_initialized(conn, flow) engine = engines.load(flow, flow_detail=flow_detail, backend=backend, book=logbook, engine='parallel', executor=executor) engine.compile() _workaround_reverted_reset(flow_detail) with MetadataSavingListener(engine, flow_detail): try: engine.run() except exceptions.WrappedFailure as wf: for failure in wf: if failure.exc_info is not None: traceback.print_exception(*failure.exc_info) else: print failure
def _default_executor_factory(self): max_simultaneous_jobs = self._max_simultaneous_jobs if max_simultaneous_jobs <= 0: max_workers = tu.get_optimal_thread_count() else: max_workers = max_simultaneous_jobs return futurist.ThreadPoolExecutor(max_workers=max_workers)
def setup(self): bind_port = self.port if self.exposed: bind_addr = '0.0.0.0' else: bind_addr = 'localhost' try: keyfile = self.ssl_config.private_key.path except AttributeError: keyfile = None try: certfile = self.ssl_config.cert.path except AttributeError: certfile = None executor = futurist.ThreadPoolExecutor(max_workers=self.max_workers) server = make_server(bind_addr, bind_port, self.wsgi_app, executor, certfile=certfile, keyfile=keyfile) if keyfile or certfile: server_base = 'https' else: server_base = 'http' server_host, server_port = server.server_address for pat, ok_methods, _maybe_handler in getattr(self.wsgi_app, 'urls', []): LOG.info("Will match %s requests that match pattern" " '%s' on port %s on %s://%s for app: %s (dispatching" " into a worker pool/executor of size %s)", ", ".join(sorted(ok_methods)), pat.pattern, server_port, server_base, server_host, reflection.get_class_name(self.wsgi_app), self.max_workers) self.server = server self.executor = executor self._server_base = server_base self._server_port = server_port
def _job(self): generators = { "icmp": self.gen_periodic_ping, "http": self.gen_periodic_http_ping } period_tasks = {} for task in self.tasks: task_data = task.values()[0] period_ = task_data["settings"]["period"] protocol = task_data["protocol"] period_tasks.setdefault(period_, []) if protocol in generators: period_tasks[period_].append(generators[protocol](task)) else: LOG.warning("Allowed protocols are: %s" % generators.keys()) pool = futurist.ThreadPoolExecutor(max_workers=len(period_tasks)) with pool: min_period = min(period_tasks) min_lag = float(min_period) / len(period_tasks[min_period]) lag = min(min_lag / len(period_tasks), 1) LOG.info(period_tasks) for period, callables in period_tasks.iteritems(): pool.submit(self._job_per_period(callables, period)) self.death.wait(lag)
def call_functions_parallel(*worker_defs): """Call specified functions in parallel. :param *worker_defs: Each positional argument can be either of a function to be called or a tuple which consists of a function, a list of positional arguments) and keyword arguments (optional). If you need to pass arguments, you need to pass a tuple. Example usages are like: call_functions_parallel(func1, func2, func3) call_functions_parallel(func1, (func2, [1, 2])) call_functions_parallel((func1, [], {'a': 1}), (func2, [], {'a': 2, 'b': 10})) :returns: a tuple of values returned from individual functions. None is returned if a corresponding function does not return. It is better to return values other than None from individual functions. """ # TODO(amotoki): Needs to figure out what max_workers can be specified. # According to e0ne, the apache default configuration in devstack allows # only 10 threads. What happens if max_worker=11 is specified? max_workers = len(worker_defs) # Prepare a list with enough length. futures = [None] * len(worker_defs) with futurist.ThreadPoolExecutor(max_workers=max_workers) as e: for index, func_def in enumerate(worker_defs): if callable(func_def): func_def = [func_def] args = func_def[1] if len(func_def) > 1 else [] kwargs = func_def[2] if len(func_def) > 2 else {} func = functools.partial(func_def[0], *args, **kwargs) futures[index] = e.submit(fn=func) return tuple(f.result() for f in futures)
class ProxyExecutor(object): KIND_TO_FACTORY = { 'threaded': (lambda: futurist.ThreadPoolExecutor(max_workers=1)), 'synchronous': lambda: futurist.SynchronousExecutor(), } # Provide a few common aliases... KIND_TO_FACTORY['thread'] = KIND_TO_FACTORY['threaded'] KIND_TO_FACTORY['threading'] = KIND_TO_FACTORY['threaded'] KIND_TO_FACTORY['sync'] = KIND_TO_FACTORY['synchronous'] DEFAULT_KIND = 'threaded' def __init__(self, driver_name, default_executor_factory): self.default_executor_factory = default_executor_factory self.driver_name = driver_name self.started = False self.executor = None self.internally_owned = True @classmethod def build(cls, driver_name, options): default_executor_fact = cls.KIND_TO_FACTORY[cls.DEFAULT_KIND] if 'executor' in options: executor_kind = options['executor'] try: default_executor_fact = cls.KIND_TO_FACTORY[executor_kind] except KeyError: executors_known = sorted(list(cls.KIND_TO_FACTORY)) raise tooz.ToozError("Unknown executor" " '%s' provided, accepted values" " are %s" % (executor_kind, executors_known)) return cls(driver_name, default_executor_fact) def start(self): if self.started: return self.executor = self.default_executor_factory() self.started = True def stop(self): executor = self.executor self.executor = None if executor is not None: executor.shutdown() self.started = False def submit(self, cb, *args, **kwargs): if not self.started: raise tooz.ToozError("%s driver asynchronous executor" " has not been started" % self.driver_name) try: return self.executor.submit(cb, *args, **kwargs) except RuntimeError: raise tooz.ToozError("%s driver asynchronous executor has" " been shutdown" % self.driver_name)
def __init__(self, bigip_url, auth=None): self.task_watcher = futurist.ThreadPoolExecutor(max_workers=1) verify = CONF.f5_agent.bigip_verify super(AS3RestClient, self).__init__(bigip_url, verify, auth) if CONF.f5_agent.prometheus: self.hooks['response'].append(self.metric_response_hook) self.hooks['response'].append(self.error_response_hook) self.hooks['response'].append(self.task_watcher_hook)
def mt_test(): startTime = timeit.default_timer() with futurist.ThreadPoolExecutor(max_workers=8) as executors: futures = [executors.submit(compute, queue) for _ in range(8)] results = [f.result() for f in futures] print('result : {}'.format(queue.get())) resource_info(startTime)
def _executor(self): if CONF.taskflow_executor.engine_mode != 'parallel': yield None else: max_workers = CONF.taskflow_executor.max_workers try: yield futurist.GreenThreadPoolExecutor(max_workers=max_workers) except RuntimeError: yield futurist.ThreadPoolExecutor(max_workers=max_workers)
def create(cls, callback_after_job=None): with cls._lock: if not cls._self: self = cls() cls._self = self self._worker = futurist.ThreadPoolExecutor() self._death = threading.Event() self._worker.submit(cls._self._periodic_worker) self._force_update = False self._callback_after_job = callback_after_job or (lambda: True)
def run(self): """Run""" LOG.debug("{}".format(self.__class__.__name__)) # Look for templates to translate from within a thread executor = futurist.ThreadPoolExecutor() while self.running: fut = executor.submit(self.__check_for_templates) fut.result() executor.shutdown()
def _fetch_an_executor(): if CONF.taskflow_executor.engine_mode != 'parallel': return None else: max_workers = CONF.taskflow_executor.max_workers try: return futurist.GreenThreadPoolExecutor( max_workers=max_workers) except RuntimeError: # NOTE(harlowja): I guess eventlet isn't being made # useable, well just use native threads then (or try to). return futurist.ThreadPoolExecutor(max_workers=max_workers)
def __init__(self): super(StatusManager, self).__init__(bigip_urls=CONF.f5_agent.bigip_urls, enable_verify=CONF.f5_agent.bigip_verify, enable_token=CONF.f5_agent.bigip_token) self.amphora_id = None self.seq = 0 LOG.info('Health Manager Sender starting.') self.amp_repo = repo.AmphoraRepository() self.amp_health_repo = repo.AmphoraHealthRepository() self.lb_repo = repo.LoadBalancerRepository() self.health_executor = futurist.ThreadPoolExecutor( max_workers=CONF.health_manager.health_update_threads) self.stats_executor = futurist.ThreadPoolExecutor( max_workers=CONF.health_manager.stats_update_threads) if cfg.CONF.f5_agent.prometheus: prometheus_port = CONF.f5_agent.prometheus_port LOG.info('Starting Prometheus HTTP server on port {}'.format( prometheus_port)) prometheus.start_http_server(prometheus_port)
def main(): """ check_and_reject로 큐의 크기 제한. """ with futurist.ThreadPoolExecutor( max_workers=8, check_and_reject=rejection.reject_when_reached(2)) as executors: futures = [executors.submit(compute) for _ in range(20)] print(' statistics : {}'.format(executors.statistics)) results = [f.result() for f in futures] print(' statistics : {}'.format(executors.statistics)) print('result : {}'.format(results))
def run(self): """Run""" # The server listens for messages and calls the # appropriate methods. It also deletes messages once # processed. if self.conf.messaging_server.debug: LOG.debug("%s" % self.__class__.__name__) # Listen for messages within a thread executor = futurist.ThreadPoolExecutor() while self.running: fut = executor.submit(self.__check_for_messages) fut.result() executor.shutdown()
def get_data(self): try: adcs = api.f5wafaas.adc_list(self.request) except Exception as e: exceptions.handle(self.request, _('Unable to retrieve instances.')) # In case of exception when calling nova.server_list # don't call api.network return [] images = [] image_map = {} flavors = [] full_flavors = {} def _task_get_flavors(): # Gather our flavors to correlate our instances to them try: tmp_flavors = api.nova.flavor_list(self.request) flavors.extend(tmp_flavors) full_flavors.update([(str(flavor.id), flavor) for flavor in flavors]) except Exception: exceptions.handle(self.request, ignore=True) def _task_get_images(): # Gather our images to correlate our instances to them try: # TODO(gabriel): Handle pagination. tmp_images = api.glance.image_list_detailed(self.request)[0] images.extend(tmp_images) image_map.update([(str(image.id), image) for image in images]) except Exception: exceptions.handle(self.request, ignore=True) with futurist.ThreadPoolExecutor(max_workers=3) as e: e.submit(fn=_task_get_flavors) e.submit(fn=_task_get_images) for n in adcs: try: image_id = n.compute['imageRef'] flavor_id = n.compute['flavorRef'] n.image_name = image_map[image_id].name n.full_flavor = full_flavors[flavor_id] except Exception as e: exceptions.handle(self.request, _("Unable to retrieve instance's image or flavor. %s" % e.message)) return adcs
def main(): if len(sys.argv) == 2: tbl = [] with open(sys.argv[1], 'rb') as fh: reader = csv.reader(fh) for row in reader: tbl.append([float(r) if r else 0.0 for r in row]) else: # Make some random table out of thin air... tbl = [] cols = random.randint(1, 100) rows = random.randint(1, 100) for _i in compat_range(0, rows): row = [] for _j in compat_range(0, cols): row.append(random.random()) tbl.append(row) # Generate the work to be done. f = make_flow(tbl) # Now run it (using the specified executor)... try: executor = futurist.GreenThreadPoolExecutor(max_workers=5) except RuntimeError: # No eventlet currently active, use real threads instead. executor = futurist.ThreadPoolExecutor(max_workers=5) try: e = engines.load(f, engine='parallel', executor=executor) for st in e.run_iter(): print(st) finally: executor.shutdown() # Find the old rows and put them into place... # # TODO(harlowja): probably easier just to sort instead of search... computed_tbl = [] for i in compat_range(0, len(tbl)): for t in f: if t.index == i: computed_tbl.append(e.storage.get(t.name)) # Do some basic validation (which causes the return code of this process # to be different if things were not as expected...) if len(computed_tbl) != len(tbl): return 1 else: return 0
def test_send_periodically(self, mock_send): p = pusher.Pusher("", period=0.1) p._death = threading.Event() def stop(): time.sleep(0.55) p._death.set() e = futurist.ThreadPoolExecutor() e.submit(stop) p._send_periodically() self.assertEqual(5, mock_send.call_count) e.shutdown()
def __init__(self, resource, object_class, resource_push_api): self._resource = resource self._obj_class = object_class self._resource_push_api = resource_push_api self._resources_to_push = {} # NOTE(annp): uWSGI seems not happy with eventlet.GreenPool. # So switching to ThreadPool self._worker_pool = futurist.ThreadPoolExecutor() self.fts = [] self._semantic_warned = False for event in (events.AFTER_CREATE, events.AFTER_UPDATE, events.AFTER_DELETE): registry.subscribe(self.handle_event, resource, event)
def main(): # the behavior is specific to GreenThreadPoolExecutor threadpool = futurist.ThreadPoolExecutor(max_workers=4) rrwlock = ReentrantReadWriteLock() futures = [] futures.append(threadpool.submit(get_write, rrwlock)) futures.append(threadpool.submit(get_read, rrwlock)) # Get the results and verify only one of the calls succeeded # assert that the other call is still pending results = waiters.wait_for_any(futures) # these statements will be printed print(results[0].pop().result == True) print(len(results[1]))
def test_ngs_basic_dlm_ops(self): pool = futurist.ThreadPoolExecutor() self.addCleanup(pool.shutdown) fts = [] for i in range(CONF.ngs.port_dlm_concurrency): fts.append( pool.submit(self._test_ngs_basic_ops, port_name='{base}_{ind}'.format( base=CONF.ngs.port_name, ind=i))) executed = futurist.waiters.wait_for_all(fts) self.assertFalse(executed.not_done) # TODO(pas-ha) improve test error reporting here for ft in executed.done: self.assertIsNone(ft.exception())
def _fetch_server(self, task_classes): endpoints = [] for cls in task_classes: endpoints.append(endpoint.Endpoint(cls)) server = worker_server.Server( TEST_TOPIC, TEST_EXCHANGE, futurist.ThreadPoolExecutor(max_workers=1), endpoints, transport='memory', transport_options={ 'polling_interval': POLLING_INTERVAL, }) server_thread = threading_utils.daemon_thread(server.start) return (server, server_thread)
def connect(self, timeout=10.0): def try_clean(): # Attempt to do the needed cleanup if post-connection setup does # not succeed (maybe the connection is lost right after it is # obtained). try: self.close() except k_exceptions.KazooException: LOG.exception("Failed cleaning-up after post-connection" " initialization failed") try: if timeout is not None: timeout = float(timeout) self._client.start(timeout=timeout) self._closing = False except (self._client.handler.timeout_exception, k_exceptions.KazooException): excp.raise_with_cause(excp.JobFailure, "Failed to connect to zookeeper") try: if self._conf.get('check_compatible', True): kazoo_utils.check_compatible(self._client, self.MIN_ZK_VERSION) if self._worker is None and self._emit_notifications: self._worker = futurist.ThreadPoolExecutor(max_workers=1) self._client.ensure_path(self.path) self._client.ensure_path(self.trash_path) if self._job_watcher is None: self._job_watcher = watchers.ChildrenWatch( self._client, self.path, func=self._on_job_posting, allow_session_lost=True) self._connected = True except excp.IncompatibleVersion: with excutils.save_and_reraise_exception(): try_clean() except (self._client.handler.timeout_exception, k_exceptions.KazooException): exc_type, exc, exc_tb = sys.exc_info() try: try_clean() excp.raise_with_cause(excp.JobFailure, "Failed to do post-connection" " initialization", cause=exc) finally: del (exc_type, exc, exc_tb)
def redeploy(self, config, old_clients): new_clients = config["clients"] old_idx = {c["host"]: c for c in old_clients} new_idx = {c["host"]: c for c in new_clients} for c in new_clients: c["configured"] = False unregister = [ "%s:%s/api/v1/unregister" % (h, old_idx[h]["port"]) for h in old_idx if h not in new_idx ] with futurist.ThreadPoolExecutor(max_workers=10) as e: e.map(requests.post, unregister) return new_clients